update monitor

monitor_logic.py

@@ -2,6 +2,9 @@ import paramiko
 import requests
 import time
 import re
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
 
 def get_ssh_data(host, port, user, password):
     try:
@@ -57,3 +60,64 @@ def check_web_service(url):
             return {"status": "error", "code": response.status_code}
     except Exception:
         return {"status": "offline"}
+
+def check_testarena_backend_status():
+    url = "http://asf-server.duckdns.org:8080/api/system/status"
+    try:
+        start_time = time.time()
+        response = requests.get(url, timeout=5)
+        latency = int((time.time() - start_time) * 1000)
+        if response.status_code == 200:
+            data = response.json()
+            # {"testarena-app":"online","testarena-worker":"online","nginx":"online"}
+            is_online = all(v == "online" for v in data.values())
+            if is_online:
+                return {"status": "online", "latency": f"{latency}ms", "details": data}
+            else:
+                return {"status": "partial", "details": data}
+        else:
+            return {"status": "error", "code": response.status_code}
+    except Exception:
+        return {"status": "offline"}
+
+def recover_testarena_backend(host, port, user, password, attempt):
+    try:
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        ssh.connect(host, port=port, username=user, password=password, timeout=10)
+        
+        if attempt == 1:
+            # system restart
+            cmd = "echo 'ASF' | sudo -S /home/asf/testarena_backend/restart_services.sh"
+        else:
+            # system redeploy
+            cmd = "echo 'ASF' | sudo -S /home/asf/testarena_backend/deploy.sh"
+            
+        stdin, stdout, stderr = ssh.exec_command(cmd)
+        exit_status = stdout.channel.recv_exit_status()
+        ssh.close()
+        return exit_status == 0
+    except Exception as e:
+        print(f"Recovery failed: {e}")
+        return False
+
+def send_email_notification(smtp_user, smtp_pass, service_name, status):
+    try:
+        msg = MIMEMultipart()
+        msg['From'] = smtp_user
+        msg['To'] = smtp_user # Send to self as requested
+        msg['Subject'] = f"ALERT: Service {service_name} is {status}"
+        
+        body = f"The service {service_name} is currently {status}. Please check the system."
+        msg.attach(MIMEText(body, 'plain'))
+        
+        server = smtplib.SMTP('smtp.gmail.com', 587) # Assuming Gmail based on the password format
+        server.starttls()
+        server.login(smtp_user, smtp_pass)
+        text = msg.as_string()
+        server.sendmail(smtp_user, smtp_user, text)
+        server.quit()
+        return True
+    except Exception as e:
+        print(f"Email failed: {e}")
+        return False