update issues of freeze

README.md

@@ -6,7 +6,15 @@ TestArena is an automated build and test execution system for ESP32 projects. It
 
 1.  **Deploy**: Run `sudo ./deploy.sh` on your Ubuntu server.
 2.  **Access**: Open `http://<server-ip>:8080/` in your browser.
-3.  **Monitor**: Use the dashboard to track test queues and view real-time logs.
+3.  **Monitor**: Use the dashboard to track test queues, view individual tasks, and check service health.
+4.  **Restart**: If services need a manual restart, use `sudo ./restart_services.sh`.
+
+## 🛠️ Key Features
+
+- **Service Robustness**: Systemd services are configured to auto-restart on failure and after reboot.
+- **Monitoring Dashboard**: Real-time status of App and Worker services, plus detailed task tracking for each queue.
+- **Task Timeouts**: Running tasks have a 1-hour timeout to prevent queue blocking.
+- **Remote Management**: A dedicated restart script for easy remote execution via SSH.
 
 ## 📚 Documentation
 

restart_services.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# TestArena Service Restart Script
+# This script restarts all components of the TestArena system.
+# Usage: sudo ./restart_services.sh
+
+if [ "$EUID" -ne 0 ]; then 
+  echo "❌ Please run as root (use sudo ./restart_services.sh)"
+  exit 1
+fi
+
+echo "🔄 Restarting TestArena Services..."
+
+echo "🌐 Restarting Nginx..."
+systemctl restart nginx
+
+echo "📱 Restarting TestArena App..."
+systemctl restart testarena-app
+
+echo "⚙️ Restarting TestArena Worker..."
+systemctl restart testarena-worker
+
+echo "✅ All services restarted!"
+systemctl status testarena-app testarena-worker nginx --no-pager

testarena-app.service

@@ -12,6 +12,8 @@ Environment="XDG_RUNTIME_DIR=/tmp"
 Environment="DATABASE_URL=sqlite:////home/asf/testarena/testarena.db"
 ExecStart=/home/asf/testarena_backend/venv/bin/uvicorn testarena_app.main:app --host 0.0.0.0 --port 8000
 Restart=always
+RestartSec=10
+StartLimitIntervalSec=0
 
 [Install]
 WantedBy=multi-user.target

testarena-worker.service

@@ -12,6 +12,8 @@ Environment="XDG_RUNTIME_DIR=/tmp"
 Environment="DATABASE_URL=sqlite:////home/asf/testarena/testarena.db"
 ExecStart=/home/asf/testarena_backend/venv/bin/python3 -m testarena_app.worker
 Restart=always
+RestartSec=10
+StartLimitIntervalSec=0
 
 [Install]
 WantedBy=multi-user.target

testarena_app/main.py

@@ -158,6 +158,26 @@ async def delete_queue(id: str, db: Session = Depends(database.get_db)):
     
     raise HTTPException(status_code=404, detail="ID not found")
 
+@app.get("/api/system/status")
+async def system_status():
+    """Check the status of system services"""
+    services = ["testarena-app", "testarena-worker", "nginx"]
+    status = {}
+    for service in services:
+        try:
+            # Use systemctl is-active for a quick check
+            res = os.system(f"systemctl is-active --quiet {service}")
+            status[service] = "online" if res == 0 else "offline"
+        except:
+            status[service] = "unknown"
+    return status
+
+@app.get("/api/queue/{id}/tasks")
+async def get_queue_tasks(id: str, db: Session = Depends(database.get_db)):
+    """Get all tasks for a specific queue"""
+    tasks = db.query(models.Task).filter(models.Task.queue_id == id).all()
+    return tasks
+
 @app.get("/")
 async def root():
     return FileResponse(os.path.join(static_dir, "index.html"))

testarena_app/static/index.html

@@ -209,6 +209,12 @@
             color: #f87171;
         }
 
+        .status-timed-out {
+            background: rgba(245, 158, 11, 0.1);
+            color: #fbbf24;
+            border: 1px solid rgba(245, 158, 11, 0.3);
+        }
+
         .btn-abort {
             background: rgba(239, 68, 68, 0.1);
             color: #f87171;
@@ -291,14 +297,24 @@
                 <div class="dot"></div>
                 <span>Connecting...</span>
             </div>
+            <div id="service-status" style="display: flex; gap: 1rem;">
+                <div class="status-badge" title="App Service">
+                    <div id="app-dot" class="dot"></div>
+                    <span>App</span>
+                </div>
+                <div class="status-badge" title="Worker Service">
+                    <div id="worker-dot" class="dot"></div>
+                    <span>Worker</span>
+                </div>
+            </div>
         </header>
 
         <div class="grid">
             <div class="card">
                 <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 1.5rem;">
                     <h2>
-                        <svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
-                            stroke-linecap="round" stroke-linejoin="round">
+                        <svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                            stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
                             <rect x="3" y="3" width="18" height="18" rx="2" ry="2" />
                             <line x1="3" y1="9" x2="21" y2="9" />
                             <line x1="9" y1="21" x2="9" y2="9" />
@@ -306,7 +322,7 @@
                         Queue Monitor
                     </h2>
                     <div style="position: relative; width: 300px;">
-                        <input type="text" id="search-input" placeholder="Search Queue ID..." 
+                        <input type="text" id="search-input" placeholder="Search Queue ID..."
                             style="width: 100%; padding: 0.6rem 1rem; border-radius: 0.75rem; background: var(--glass); border: 1px solid var(--glass-border); color: var(--text); font-family: inherit;">
                     </div>
                 </div>
@@ -323,6 +339,37 @@
                         <!-- Dynamic content -->
                     </tbody>
                 </table>
+
+                <div id="tasks-section"
+                    style="margin-top: 3rem; display: none; border-top: 1px solid var(--glass-border); padding-top: 2rem;">
+                    <div
+                        style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 1.5rem;">
+                        <h2>
+                            <svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor"
+                                stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+                                <path d="M9 11l3 3L22 4" />
+                                <path d="M21 12v7a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2h11" />
+                            </svg>
+                            Tasks for <span id="selected-queue-id"></span>
+                        </h2>
+                        <button class="btn-abort"
+                            style="background: var(--glass); color: var(--text); border-color: var(--glass-border);"
+                            onclick="hideTasks()">Close</button>
+                    </div>
+                    <table id="tasks-table">
+                        <thead>
+                            <tr>
+                                <th>Task ID</th>
+                                <th>Scenario</th>
+                                <th>Status</th>
+                                <th>Result</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            <!-- Dynamic content -->
+                        </tbody>
+                    </table>
+                </div>
             </div>
 
             <div class="card">
@@ -356,6 +403,8 @@
                 const badge = document.getElementById('connection-status');
                 badge.querySelector('.dot').classList.add('online');
                 badge.querySelector('span').textContent = 'System Online';
+
+                fetchServiceStatus();
             } catch (e) {
                 const badge = document.getElementById('connection-status');
                 badge.querySelector('.dot').classList.remove('online');
@@ -363,6 +412,22 @@
             }
         }
 
+        async function fetchServiceStatus() {
+            try {
+                const response = await fetch('/api/system/status');
+                const status = await response.json();
+
+                const appDot = document.getElementById('app-dot');
+                const workerDot = document.getElementById('worker-dot');
+
+                if (status['testarena-app'] === 'online') appDot.classList.add('online');
+                else appDot.classList.remove('online');
+
+                if (status['testarena-worker'] === 'online') workerDot.classList.add('online');
+                else workerDot.classList.remove('online');
+            } catch (e) { }
+        }
+
         function renderTable() {
             const searchTerm = document.getElementById('search-input').value.toLowerCase();
             const tbody = document.querySelector('#queue-table tbody');
@@ -375,8 +440,9 @@
                 tr.innerHTML = `
                     <td style="font-weight: 600;">${q.id}</td>
                     <td><span style="opacity: 0.8;">${q.environment}</span></td>
-                    <td><span class="status-pill status-${q.status.toLowerCase()}">${q.status}</span></td>
+                    <td><span class="status-pill status-${q.status.toLowerCase().replace(' ', '-')}">${q.status}</span></td>
                     <td style="display: flex; gap: 0.5rem;">
+                        <button class="btn-abort" style="background: rgba(99, 102, 241, 0.1); color: #818cf8; border-color: rgba(99, 102, 241, 0.2);" onclick="viewTasks('${q.id}')">Tasks</button>
                         <button class="btn-abort" onclick="abortQueue('${q.id}')">Abort</button>
                         <button class="btn-abort" style="background: rgba(239, 68, 68, 0.2); border-color: var(--danger);" onclick="deleteQueue('${q.id}')">Delete</button>
                     </td>
@@ -426,6 +492,37 @@
             }
         }
 
+        async function viewTasks(queueId) {
+            document.getElementById('tasks-section').style.display = 'block';
+            document.getElementById('selected-queue-id').textContent = queueId;
+            document.getElementById('tasks-section').scrollIntoView({ behavior: 'smooth' });
+
+            try {
+                const response = await fetch(`/api/queue/${queueId}/tasks`);
+                const tasks = await response.json();
+                const tbody = document.querySelector('#tasks-table tbody');
+                tbody.innerHTML = '';
+
+                tasks.forEach(t => {
+                    const tr = document.createElement('tr');
+                    const resultStr = t.result ? JSON.stringify(t.result).substring(0, 50) + '...' : '-';
+                    tr.innerHTML = `
+                        <td>${t.id}</td>
+                        <td title="${t.scenario_path}">${t.scenario_path.split('/').pop()}</td>
+                        <td><span class="status-pill status-${t.status.toLowerCase().replace(' ', '-')}">${t.status}</span></td>
+                        <td><small>${resultStr}</small></td>
+                    `;
+                    tbody.appendChild(tr);
+                });
+            } catch (e) {
+                addLog(`Failed to fetch tasks for ${queueId}`, 'danger');
+            }
+        }
+
+        function hideTasks() {
+            document.getElementById('tasks-section').style.display = 'none';
+        }
+
         function addLog(msg, type = 'info') {
             const logs = document.getElementById('logs');
             const entry = document.createElement('div');

testarena_app/worker.py

@@ -170,10 +170,12 @@ def run_worker():
                         task_dir = os.path.join(queue_dir, task.id)
                         os.makedirs(task_dir, exist_ok=True)
                         
-                        ret = run_command_with_logging(cmd, queue_log, cwd=repo_dir)
+                        ret = run_command_with_logging(cmd, queue_log, cwd=repo_dir, timeout=3600)
                         
                         if ret == 0:
                             task.status = "Finished"
+                        elif ret == 124:
+                            task.status = "Timed Out"
                         else:
                             task.status = "Error"