108 lines
3.1 KiB
Python
108 lines
3.1 KiB
Python
#!/bin/python3
|
|
|
|
#required modules
|
|
|
|
import json
|
|
import subprocess
|
|
|
|
#defining user / key location variables
|
|
|
|
ssh_key = "/key/location"
|
|
ssh_user= "theguy"
|
|
|
|
#create dictionary defining servers, their ip and ssh port from json
|
|
|
|
try:
|
|
with open("/root/servers.json") as f:
|
|
servers = json.load(f)
|
|
except FileNotFoundError:
|
|
raise RuntimeError("servers json file not found")
|
|
|
|
|
|
#load previous state of servers, to avoid constant alert spam
|
|
try:
|
|
with open("/var/www/porygon/status.json") as f:
|
|
#creating a previous status dictionary
|
|
previous_status = json.load(f)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
#creating the current status dictionary
|
|
status = {}
|
|
|
|
for server, info in servers.items():
|
|
ip = info["ip"]
|
|
port = info["port"]
|
|
result = subprocess.run(
|
|
["ping", "-c", "2", "-W", "1", ip],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL
|
|
)
|
|
|
|
#if the ping returns successful the online status is true, else false
|
|
if result.returncode == 0:
|
|
status[server] = True
|
|
else:
|
|
status[server] = False
|
|
|
|
#if the server is now offline, and its last state was online send an email alert
|
|
if status[server] == False:
|
|
if previous_status.get(server) == True:
|
|
message = f"""To: person@example.com
|
|
Subject: {server} is offline
|
|
|
|
{server} at {ip} looks to be offline and is no longer responding to pings.
|
|
"""
|
|
|
|
subprocess.run(
|
|
["msmtp", "123@hehehe.net"],
|
|
input=message,
|
|
text=True
|
|
)
|
|
|
|
#if the server is online run the stats collector python script over ssh
|
|
#copy back the stats.json it generates back via scp to the relevant dir for apache to host
|
|
if status[server] == True:
|
|
try:
|
|
subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-o", "StrictHostKeyChecking=accept-new",
|
|
"-i", ssh_key,
|
|
"-p", str(port),
|
|
f"{ssh_user}@{ip}",
|
|
f"/home/{ssh_user}/collector.py"
|
|
],
|
|
timeout=5
|
|
)
|
|
subprocess.run(
|
|
[
|
|
"scp",
|
|
"-o", "StrictHostKeyChecking=accept-new",
|
|
"-i", ssh_key,
|
|
"-P", str(port),
|
|
f"{ssh_user}@{ip}:/tmp/stats.json",
|
|
f"/var/www/porygon/{server}/stats.json"
|
|
],
|
|
timeout=5
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
print(f"{server} ssh connection timed out")
|
|
|
|
#if the server is now online, but it was last offline send an email saying the alert has been cleared
|
|
if previous_status.get(server) == False:
|
|
message = f"""To: fakest@email.com
|
|
Subject: {server} is back online
|
|
|
|
{server} at {ip} looks to be back online and is now responding to pings once more.
|
|
"""
|
|
|
|
subprocess.run(
|
|
["msmtp", "real@this.time"],
|
|
input=message,
|
|
text=True
|
|
)
|
|
|
|
#dump the online/offline status from the status dictionary
|
|
with open("/tmp/status.json", "w") as f:
|
|
json.dump(status, f, indent=4) |