From 16bbc930a8e8b276ba57e6f533c6e16204a0202c Mon Sep 17 00:00:00 2001 From: mahmamdouh Date: Sun, 4 Jan 2026 16:22:27 +0100 Subject: [PATCH] update monitor --- __pycache__/main.cpython-313.pyc | Bin 0 -> 5759 bytes __pycache__/monitor_logic.cpython-313.pyc | Bin 0 -> 6394 bytes main.py | 92 ++++++++++++++++------ monitor.html | 43 +++++++--- monitor_logic.py | 64 +++++++++++++++ 5 files changed, 163 insertions(+), 36 deletions(-) create mode 100644 __pycache__/main.cpython-313.pyc create mode 100644 __pycache__/monitor_logic.cpython-313.pyc diff --git a/__pycache__/main.cpython-313.pyc b/__pycache__/main.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1779b78055ac6ac62e85b27111bbec5f7bee882 GIT binary patch literal 5759 zcmb6dTWlN0aqoD1i4Q#}iIPZBM_H~!JxDo<;zyl_{LpKGkKIQCVA}M z(X!d5Rg$(bf&x)c1a$+n)u+-HZNJi=e&pRWDGJh5K=mS{De518RkB_5g#w*D@<=I) z>+}fD&d$uv&d%)Z%-(H>qYlCIhZnvo9&1GCFQlVCmI`uviALyE#37D~pfH6gmf&=R z4jZsR!wnH8Y{W(lXCkJs8JjiS7_ndr$eJS7a2>AGXy%A5Y{&Mn13SV_>WTO+v>w z+k&whadysel;WIR{ZX=(2HeUuVmH@>+ZHS{_vx~=7Dl+Z=27qnd2k2Uf<0U-_Hu6A zdDjtbTzhqdkL$o)oCmX(SbS+?TYrm+K>K$b|h|fa@Qo z$}ogDBR3Faa5p#Df$#xX<>|6jyCF9;N|pOuu%qfrd=R83cgs~|LRIPEUDCaGkv>$F zzHgUw-$H;JId%>q-i^pkF-H1v|3d%F6Z_}uhS+fTbEDh?#QTfJMPuObO(GVcRP#w* zR>r1JzfP(4lVVbs5#)@NmIYwhIE7cl7@^A0b)Gmg!%d3ucv4vAu`op>k`f<}#wA5a zUkn&k`=X#kWqBzY=M`RUj4cVV3(;lad{h?jMKLC*zA{M>WCb*&`RI8*c0owTqtI8f zvf7S?m~>IV(R~uCTPAWsiWig7w4{g$F$Ts-X|Tcq(+P^05>%=`U{D==GAS)bB`hwA zX<6^41Z7EzSG!9B9~ZEkGl$63Voaripvf8+!2(6XmsL}l4n!u+r-WQ9FDHh`Is_by zXJZ%QX*nq2MG?J;kZJ&?YMwq3ogU*j)iB1L%(dc-I7G%M=l96}0^WDk1dOP19)_GQ z4>7@z={iPUQxsB_+-vo6*`{^!ut1$7{?_Z$7?G(Pr7?Ww1~ha zF-wU=QcMd0Mx`?`_=~EgvcY87k+7Yk*$mkdF|x(=#|p>`@W1`05upQ!g`?I7zj>XH zW-5nGl>kULnrHO>ys3)mM>d+*j&(wxx9i+d&`+Mz@KP=r5V}acYM3(GQTqRx%URX| zdK;=(Qx$90vAfPig;1KFCp$oUTC{3IeNyUOV`?A zG-sHg=9mdeKeSjh$8a>!#Tkw*BNPWePr?y;Pg}gY)_GqQV|5JdV~9>&_uV|fqRDR| zWP^C$bG~j}w{H2qH5=!QyH~>h2Uc-V9|dW`qeEMm(Y-fk8m;WoeYTtD%(^-)0yBFe(67%_Yas<&bz48H?F-oc|9KPAm}b?Fz^ah$?ph~F8SMDa5bIKD^TR+GPHwEySs;_Ui7ci&f^zyC?0!#RWWoN><^faV8v&vNx- zfpgR@N`pqBcTpM%Wj?5nnHkoxT3aj9W=3jM9n;IXT{@dIow@*;`TUqS>pt8c^HcQX zYY17;DSB#2gQFTW1Ei+j(BStqctnHE7BqDS{y*@e`TO;0&4`CHCT-ajW!Id=iAHFB zUKVx1$(go{S*FgG+XR(B4;s2qf&y(5hYblT&@zQtP>IRgr<$ZR$p>1(yGa4%CV2^R zBV&ppq%w+_USunE#bq`jVYZsM*zt1t!S+=|*&GuH27@=rv7QU;m#e4|;;fvF#ROSS zWRuCu!Q9AQMPWv$SYQC3Pz215vV7SnIhbqQHz;>M$&rxHN!_r&);M-#_{eaG&xj%U zvJ6YpN|&sRyTDpZf{V!;jspac;O3eqrL+ju46E?+1y;aV!pB%~5L^`SsAgWioQ{c- zYE`7DvV@@^!X!y*bRf42&(<;#CQ~> zpE!MDg2Tj>Q14`**ob2X=~%hR*=bF_f|8;M+X%~u`ShZo8Z%f-E10BI)hH(gA)`7T z8=IZrW~1Ef*z6pKNq$rfX=z!tin0VjP4S9qowyVeAl^_ET2)gf7R7=Lt*Es&QilQ| zn^a_#l2wDoRCtYPu0%yOSK_Ktf=ryj>W^xM$}=6zTvl6BWvA*ztsEuZ#iYzX0)P3t zkl${i6|`CB+@^=D!5g0Ll4tOmXYl6>zd8Kwso$U7c;cy|=jl?*(^pPz{SV%@33)oN zME=y`U!}IZ2TIn+={)LwBK}d+qodzjl0mWbODy z!|v1@Rv!(ZwcxYjYSF?izLCpV^_%+D|6pNbXcvmac}pIgXxJhjpM^j6P6sb}7Y)SukG)#V$$gFi{VmRcV#`a-3)P`)j+)f+7J9=XTnE-J6PRE&z=IPH_b~>NA zm|y@dyHzqbv*_}7jJS^fW%{|fu@c{<{w zzH=yi7`;J-eax>%0efqZ3j3M2hK2!uyORp{nBMlWfWJeB2s}iEdrj}$cM$M*-Bh@r zdADt>6FTpWQsE)fdkOmH6s^bh?S?9pJa2KzNQq`_egj%X0(2l>Nwk#@sJlrz$7 z_^93pdOm745=uJ*IK07dwN$d1J;gsM;>PerJm1g|65UT_RQM!1fONm1dV0LR)a&@RAVhI_kqQruu{CB}V?ac{H@ z7284|GsFLsiN(*e-IPj&_N)Xh6y=q$fK*mOQl9p3)uOvcvoJtU?@l`t{$ItblE684Xgl#ABJY1DcT1FO)Y*B_%ivE%uqI^Ezh)7tT0>Q-XD#ULpd7F znTLbgPkaP&VSrYdB}rCPhTNnBgQ__JcS1;;m^e>utn`AA0JJY1GA5x`T{^vd@lTMi z9+(sos$1uzsvke#UXEpKCR|#ABvbwbf-QvL&=W*bS`=eiPSvuYmg2OersY9x=Ts6I zq({}s$K%o345~3DWns6Fk7Ia*08&G#Ch!n^Y|%0?COZ>9K)_)FNcP3Y2>3DqBpa2z zOmdTIk$)~= zu5#M|>v8W`jg9n*bugX#gW8n3fRxe2qKFxv>T9hi*$O0H;V{{&pf*j|i0A6YeA4XpPRnb5WoF&)}# F{$C|*Ct?5q literal 0 HcmV?d00001 diff --git a/__pycache__/monitor_logic.cpython-313.pyc b/__pycache__/monitor_logic.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93ac8e7f83d02e0fe225c5ac4624105dec8acc21 GIT binary patch literal 6394 zcmcgwZEV}d89s@mM2Y&aBY!1MZN{-9%c&(hj+-p8-O5hVv{~HFjMB7jr!f+3YmrFx zNTspXZbdf?GFqTX(;@~o4Ea^KzyPE1TW>(u{n>kz zXt{}#wOiMNba%YZm&bdad*0jddff=Z?|=GCYI+Akf29vru-7sh=OObJl8|IOg>W^s zVVn7E$96MiFk_}HW+Ano;vRQkhe+Epr<{-Tn1?*O3t^YULF|?s5DStMVvoc_?47lR zT({|jlOY@7Pd|40_!;Gr0eLUYpU$O?R941Df~^@v6CVBy{5Rf)_$}0drfI1erYz&F z0!X{iaP3KDW9$^SrK*{|;xi5bkavG1eG4 zuswTLb;cZ1-n-jb7Ek%_Q`dO!I+X_tx0|2+mga{K9&;T91LeI4#avT?yKR*?$uV5} zUfb3?>-DPpi9In_Z*35{4Wa2~YsBeRD;INFZzu8n@b=ad^tMYb)qRgOaa-;F&)-D2 z$Na*#G`|Pt-)XH?J+<~4K-ClDWA2y`^GMz>*Nyg|>0MSkC!(vadL>_d*aI z>-J6nzUqy7yKCu;Z3c}pooL1uYM8uD8Rs3AuvrZogwN>;R^^OB_^hn!FK9SPoJK06 zXgMSQl%ZwC{`2af_@X$Am8=+?$mV+WP!OJVB`F5=NVxAovG3X5>4E<5grDnsHq_H6 zrjp^l$g_hH+Dd-mf*AZ!7N=B092^)B`^E0TmxB3c60#vi^m#p!NW-iVV@|;tIc=r! zyrwDx5ktvj#RFrJq;fH$=F(~L#TUhkiCINWWRhY=k@Xx_;&(R6bF^fXpu9aearmPrT8bp+%F) zC##Rngr=%W!XO^yl9Grgv`j`;lZ3^JoFtBB+8Qji2A7=6JIbvGS6fF)ts^UE-aAlkJ+W}|FW%-`t-F3Xe7)@# z4=sM+dkxmiwcvV9SZnF11a?=NcAIg>r-Gxwy^b6%_ve52Ay4xLVmbF( zOXscTw#CM)XFuhb2KPS@<8uFNoq?J!^bYu9ubkL_B8=V-yB}>}-aqVpRA4^fY>@vz z;2=)aPIzdD5LF6*W3jc#{y{SW{wknrek1iAgQ3X1|uG*)a)-YRC zh*h*Dt&bkF^#c|prXTFH_*Z3O_9+UMJA%X>IR|Gj)WCVn_>klnvP;ge9T=LwM-LaR z1?D#eA{`ExA8drv!7T{`Hp=qdXp`4Hleg(Iy<@{vZD1;8ZhuH~6ja2X(L*)5ai(Ph z7<3-fbNiiIAxo*jq$At}#UTgK7MuZu$IA0LMK^%U0Ct1{8Ye=P-GE-83y2C7VhNMO zY2t$%mu^V-vIJbiV#;Jy`Tzc%v5$aQAvdjOmQe``nco5r>IH|DSY==w9I*2wa~Qp=(8 zj>83KMF^}4oh703`cPTeSLxWdJiGk#%K7E7QeeF38DHz@S-P|unJB?8aIEM#_W3QL zX#?4~o=QX8VyNiv{)q2hXW;n@9gbP_O0*|>2>r3wN8>}jah|#1v_YJJy3Me;Ki@R> zr({#Vtc5ol)-&~8^p*CgU1I*HJ<5KiJ<6GT^f>55`*xbu(kCiAy{VL`BLQmWKI%BI zsrolD!o|4W+EU0s-jdTq1V3btv63sy046{oN`i?F&ppuLt)ZjATGLH*_$2>b=xCVC zADlCc?2$-B)@SFAimNVu$j~2hoDi2sp6jElsJ=$A*a$haZ+b5 zC8ukwfggujlQHmNnxo&l$`meqgyyM-fk$XGN~1A|2>U}_Qwbxd)5LL6P6Llr>B4~@ zrfnXfQItmJ)GR&Ei7TbYt2#=j9w6lmYfU=tzYzF)(Ebtl>wkdgWmIuL^76#G-T%$- z2=LG7mC>sY7mj_>)?R9lE+72e(cc{X?P$3@T5gLL9{Y>GtXz1vpiz z=i`q}{`neyI3V#G^253R?1xK?$u9sIiB%oo3393f=HJFUjyWu@Fg?Lbsw>7%d92s1 zr}yI>@G`ln`*`MXGXU2h6rkoQOYvfcntvtKy({1+!H0(ImQ5lu{VO3V%<|x8LAu4i za!q2oaWPdBx8ZZ3{4pglr-{L+bi(u)a!F0>m&C}NmQg4fftPa!vLo0m0hL=l9QCk1 zmp}5gM@fRf*XE%wrp%IWoC5t1tT!*t0K%2z5%EMwz%A5P>bGNh+F|OIV(Mz*c8G{G zF(-pZktPjTNnDI8mr`Jq>D%Bg+K8V1gfqRnkOz0uBDxoG4~?j9Ax_z#qTj&#=o15u z8xGRSUK)jHnXq-#2$bmbLz}p-MOD$T6HK7?gA;Mbdf{iFgG75SLqx^H4zBsXbOP2u zj?1r(tnE0rFag3k@Pw@}u_o@Vx&_ODvN&uB<(A#o)GO+0%W$b>xZHB6AW)CORL2$n zu0^@z?=142Rdrzpb9isDVbAqF*8@dg&rN>cEq~L$);XxC$&B&V@m=V~F8BBdbEDfg zeu%koh=cr>uG{FoqUX)V+ZN2}{xU=TXg^s8rCe6e+$z@#<)qyZgZ8F1lqPNSGs#Y%myJI`BTebrrx4nWpXR}6t z7Wb7J<|W=XkUxiF&f(hInX?2etLtgqidhecZmNRKP@@CBez_%iWBfLBz(@I6AJ74$ zTj;<)N!Sxu%Mkk+E$@$h_xRMABcfErwU}qclr9pdlso@JaNF~qGjj^utC)&d%=>6% zB8TD7H`4QBepgt0G7Xm=ifA5$BHWpXRk0EV45sCjn)m4$BO9L00KmiG{ijX=^3UXb z$7#V9*n2Pw??xPeO*t`#>B}}apUZiqnfGQ#=qRqt6odyq27YdjnIB z(VLP>1~w@GM9P zihsxJFD<QCq(W`jl7Qt~E0M4#Zo;Wcl?AJ|F@mwk)RFD8nw18S5(F+yv$Y zis^ls`pKkHb44+@0cZpMsJi_8NJdNM(#jY%po|iaPGvE;N;cca$Z!6HkCE$R ` -
  • -
    - - ${s.name} -
    -
    - - ${s.status === 'online' ? s.latency : 'Offline'} -
    -
    • - `).join(''); + serviceList.innerHTML = data.services.map(s => { + let statusColor = 'var(--status-red)'; + let statusIcon = 'fa-times-circle'; + let statusText = s.status; + + if (s.status === 'online') { + statusColor = 'var(--status-green)'; + statusIcon = 'fa-check-circle'; + statusText = s.latency; + } else if (s.status === 'partial') { + statusColor = 'var(--status-yellow)'; + statusIcon = 'fa-exclamation-circle'; + statusText = 'Partial'; + } + + return ` +
  • +
    + + ${s.name} +
    +
    + + ${statusText} +
    +
    • + `; + }).join(''); document.getElementById('loading').style.opacity = '0'; setTimeout(() => document.getElementById('loading').style.display = 'none', 500); @@ -537,7 +553,8 @@ 'Draw.io': 'https://app.diagrams.net/images/logo-flat.svg', 'TestArena': '/static/testarena.png', 'TBM': '/static/tbm.ico', - 'Board': 'https://excalidraw.com/favicon-32x32.png' + 'Board': 'https://excalidraw.com/favicon-32x32.png', + 'TestArena Backend': '/static/testarena.png' }; return logos[name] || ''; } diff --git a/monitor_logic.py b/monitor_logic.py index 42b1638..e79ffc2 100644 --- a/monitor_logic.py +++ b/monitor_logic.py @@ -2,6 +2,9 @@ import paramiko import requests import time import re +import smtplib +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart def get_ssh_data(host, port, user, password): try: @@ -57,3 +60,64 @@ def check_web_service(url): return {"status": "error", "code": response.status_code} except Exception: return {"status": "offline"} + +def check_testarena_backend_status(): + url = "http://asf-server.duckdns.org:8080/api/system/status" + try: + start_time = time.time() + response = requests.get(url, timeout=5) + latency = int((time.time() - start_time) * 1000) + if response.status_code == 200: + data = response.json() + # {"testarena-app":"online","testarena-worker":"online","nginx":"online"} + is_online = all(v == "online" for v in data.values()) + if is_online: + return {"status": "online", "latency": f"{latency}ms", "details": data} + else: + return {"status": "partial", "details": data} + else: + return {"status": "error", "code": response.status_code} + except Exception: + return {"status": "offline"} + +def recover_testarena_backend(host, port, user, password, attempt): + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(host, port=port, username=user, password=password, timeout=10) + + if attempt == 1: + # system restart + cmd = "echo 'ASF' | sudo -S /home/asf/testarena_backend/restart_services.sh" + else: + # system redeploy + cmd = "echo 'ASF' | sudo -S /home/asf/testarena_backend/deploy.sh" + + stdin, stdout, stderr = ssh.exec_command(cmd) + exit_status = stdout.channel.recv_exit_status() + ssh.close() + return exit_status == 0 + except Exception as e: + print(f"Recovery failed: {e}") + return False + +def send_email_notification(smtp_user, smtp_pass, service_name, status): + try: + msg = MIMEMultipart() + msg['From'] = smtp_user + msg['To'] = smtp_user # Send to self as requested + msg['Subject'] = f"ALERT: Service {service_name} is {status}" + + body = f"The service {service_name} is currently {status}. Please check the system." + msg.attach(MIMEText(body, 'plain')) + + server = smtplib.SMTP('smtp.gmail.com', 587) # Assuming Gmail based on the password format + server.starttls() + server.login(smtp_user, smtp_pass) + text = msg.as_string() + server.sendmail(smtp_user, smtp_user, text) + server.quit() + return True + except Exception as e: + print(f"Email failed: {e}") + return False