init testarena backend service

This commit is contained in:
2025-12-27 00:43:28 +01:00
parent 22d0ed24d7
commit b6d7b81649
9 changed files with 901 additions and 0 deletions

View File

@@ -0,0 +1,44 @@
#!/bin/bash
# TestArena Deployment Script
# Run this script with sudo: sudo ./deploy.sh
set -e
echo "🚀 Starting TestArena Deployment..."
# 1. Install Dependencies
echo "📦 Installing dependencies..."
apt-get update
apt-get install -y nginx python3-pip python3-venv
# 2. Set up Python Virtual Environment
echo "🐍 Setting up Python environment..."
python3 -m venv venv
source venv/bin/activate
pip install fastapi uvicorn sqlalchemy
# 3. Configure Nginx
echo "🌐 Configuring Nginx..."
cp nginx/testarena.conf /etc/nginx/sites-available/testarena
ln -sf /etc/nginx/sites-available/testarena /etc/nginx/sites-enabled/
rm -f /etc/nginx/sites-enabled/default
# 4. Create Data Directory
echo "📁 Creating data directory..."
mkdir -p /home/asf/testarena
chown -R asf:asf /home/asf/testarena
chmod -R 755 /home/asf/testarena
# 5. Restart Nginx
echo "🔄 Restarting Nginx..."
nginx -t
systemctl restart nginx
echo "✅ Deployment complete!"
echo "--------------------------------------------------"
echo "Dashboard: http://asf-server.duckdns.org:8080/"
echo "Results: http://asf-server.duckdns.org:8080/results/"
echo "--------------------------------------------------"
echo "To start the app: source venv/bin/activate && uvicorn testarena_app.main:app --host 0.0.0.0 --port 8000"
echo "To start the worker: source venv/bin/activate && python3 -m testarena_app.worker"

View File

@@ -0,0 +1,80 @@
# TestArena Deployment & Testing Guide
This guide explains how to deploy and test the TestArena backend application on your Ubuntu Server.
## 🚀 Deployment Steps
### 1. Clone the Repository
Ensure you have the code on your server in a directory like `/home/asf/testarena_pc_backend`.
### 2. Run the Deployment Script
The deployment script automates Nginx configuration and dependency installation.
```bash
sudo chmod +x deploy.sh
sudo ./deploy.sh
```
### 3. Start the Application Services
You should run these in the background or using a process manager like `pm2` or `systemd`.
**Start the API Server:**
```bash
source venv/bin/activate
uvicorn testarena_app.main:app --host 0.0.0.0 --port 8000
```
**Start the Background Worker:**
```bash
source venv/bin/activate
python3 -m testarena_app.worker
```
---
## 🧪 Testing the System
### 1. Verify Dashboard Access
Open your browser and navigate to:
`http://asf-server.duckdns.org:8080/`
You should see the modern, colorful TestArena dashboard.
### 2. Verify Results Browsing
Navigate to:
`http://asf-server.duckdns.org:8080/results/`
You should see an automatic directory listing of `/home/asf/testarena/`.
### 3. Test the Queue API
Run the following `curl` command to queue a test task:
```bash
curl -X POST http://asf-server.duckdns.org:8080/api/queue \
-H "Content-Type: application/json" \
-d '{
"test_queue_001": [
"staging",
{
"task_1": "/home/asf/scenarios/test1.py",
"task_2": "/home/asf/scenarios/test2.py"
}
]
}'
```
### 4. Verify Worker Execution
- Check the dashboard; you should see the new queue appear and its status change from `Waiting` to `Running` and then `Finished`.
- Check the filesystem:
```bash
ls -R /home/asf/testarena/test_queue_001
```
You should see `queue_status.json` and any results generated by `tpf_execution.py`.
### 5. Test Abortion
Queue another task and click the **Abort** button on the dashboard. Verify that the status changes to `Aborted` in both the dashboard and the `queue_status.json` file.
---
## 🛠️ Troubleshooting
- **Nginx Errors**: Check logs with `sudo tail -f /var/log/nginx/error.log`.
- **FastAPI Errors**: Check the terminal where `uvicorn` is running.
- **Permission Issues**: Ensure `/home/asf/testarena` is writable by the user running the app.
- **Port 8080 Blocked**: Ensure your firewall (ufw) allows traffic on port 8080: `sudo ufw allow 8080`.

View File

@@ -0,0 +1,58 @@
# TestArena Nginx Configuration
# This file should be placed in /etc/nginx/sites-available/testarena
# and symlinked to /etc/nginx/sites-enabled/testarena
server {
listen 8080;
server_name _;
# Security: Prevent directory traversal and restrict symlinks
disable_symlinks on;
# Root directory for the results (autoindex)
location /results/ {
alias /home/asf/testarena/;
# Enable autoindex with requested features
autoindex on;
autoindex_exact_size off; # Human-readable sizes
autoindex_localtime on; # Local time
# Read-only access
limit_except GET {
deny all;
}
# Prevent execution of scripts
location ~* \.(php|pl|py|sh|cgi)$ {
return 403;
}
}
# Proxy requests to the FastAPI application
location / {
proxy_pass http://127.0.0.1:8000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# WebSocket support (if needed in future)
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
# Custom error pages
error_page 404 /404.html;
location = /404.html {
root /usr/share/nginx/html;
internal;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
internal;
}
}

View File

@@ -0,0 +1,16 @@
from sqlalchemy import create_all_engines, create_engine
from sqlalchemy.orm import sessionmaker
import os
# Using SQLite for simplicity as requested
DATABASE_URL = "sqlite:///d:/ASF - course/ASF_01/ASF_tools/asf-pc-server/testarena_pc_backend/testarena_app/testarena.db"
engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False})
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()

View File

@@ -0,0 +1,139 @@
from fastapi import FastAPI, Depends, HTTPException, BackgroundTasks
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse
from sqlalchemy.orm import Session
import os
import json
import uuid
from typing import Dict, List
from . import models, database
app = FastAPI(title="TestArena API")
# Mount static files
static_dir = os.path.join(os.path.dirname(__file__), "static")
os.makedirs(static_dir, exist_ok=True)
app.mount("/static", StaticFiles(directory=static_dir), name="static")
# Base directory for data as requested
BASE_DATA_DIR = "/home/asf/testarena"
# For local development on Windows, we might need to adjust this,
# but I'll stick to the user's requirement for the final version.
if os.name == 'nt':
BASE_DATA_DIR = "d:/ASF - course/ASF_01/ASF_tools/asf-pc-server/testarena_pc_backend/testarena_data"
# Ensure base directory exists
os.makedirs(BASE_DATA_DIR, exist_ok=True)
# Initialize database
models.Base.metadata.create_all(bind=database.engine)
@app.post("/api/queue")
async def queue_task(payload: Dict, db: Session = Depends(database.get_db)):
"""
Input json contain {<queue_ID> :[environment, "<TASK_ID>" : "<path to scenario>],}
"""
try:
queue_id = list(payload.keys())[0]
data = payload[queue_id]
environment = data[0]
tasks_data = data[1] # This is a dict {"TASK_ID": "path"}
# 1. Create folder
queue_dir = os.path.join(BASE_DATA_DIR, queue_id)
os.makedirs(queue_dir, exist_ok=True)
# 2. Create queue_status.json
status_file = os.path.join(queue_dir, "queue_status.json")
queue_status = {
"queue_id": queue_id,
"status": "Waiting",
"tasks": {}
}
# 3. Save to database and prepare status file
new_queue = models.Queue(id=queue_id, environment=environment, status="Waiting")
db.add(new_queue)
for task_id, scenario_path in tasks_data.items():
new_task = models.Task(id=task_id, queue_id=queue_id, scenario_path=scenario_path, status="Waiting")
db.add(new_task)
queue_status["tasks"][task_id] = "Waiting"
with open(status_file, 'w') as f:
json.dump(queue_status, f, indent=4)
db.commit()
return {"status": "Queue OK", "queue_id": queue_id}
except Exception as e:
return {"status": "Error", "message": str(e)}
@app.get("/api/status/{id}")
async def get_status(id: str, db: Session = Depends(database.get_db)):
# Check if it's a queue ID
queue = db.query(models.Queue).filter(models.Queue.id == id).first()
if queue:
return {"id": id, "type": "queue", "status": queue.status}
# Check if it's a task ID
task = db.query(models.Task).filter(models.Task.id == id).first()
if task:
return {"id": id, "type": "task", "status": task.status}
raise HTTPException(status_code=404, detail="ID not found")
@app.post("/api/abort/{id}")
async def abort_task(id: str, db: Session = Depends(database.get_db)):
# Abort queue
queue = db.query(models.Queue).filter(models.Queue.id == id).first()
if queue:
queue.status = "Aborted"
# Abort all tasks in queue
tasks = db.query(models.Task).filter(models.Task.queue_id == id).all()
for t in tasks:
if t.status in ["Waiting", "Running"]:
t.status = "Aborted"
# Update queue_status.json
queue_dir = os.path.join(BASE_DATA_DIR, id)
status_file = os.path.join(queue_dir, "queue_status.json")
if os.path.exists(status_file):
with open(status_file, 'r') as f:
data = json.load(f)
data["status"] = "Aborted"
for tid in data["tasks"]:
if data["tasks"][tid] in ["Waiting", "Running"]:
data["tasks"][tid] = "Aborted"
with open(status_file, 'w') as f:
json.dump(data, f, indent=4)
db.commit()
return {"id": id, "status": "Aborted"}
# Abort single task
task = db.query(models.Task).filter(models.Task.id == id).first()
if task:
task.status = "Aborted"
# Update queue_status.json
queue_dir = os.path.join(BASE_DATA_DIR, task.queue_id)
status_file = os.path.join(queue_dir, "queue_status.json")
if os.path.exists(status_file):
with open(status_file, 'r') as f:
data = json.load(f)
data["tasks"][id] = "Aborted"
with open(status_file, 'w') as f:
json.dump(data, f, indent=4)
db.commit()
return {"id": id, "status": "Aborted"}
raise HTTPException(status_code=404, detail="ID not found")
@app.get("/api/queues")
async def list_queues(db: Session = Depends(database.get_db)):
queues = db.query(models.Queue).order_by(models.Queue.created_at.desc()).all()
return queues
@app.get("/")
async def root():
return FileResponse(os.path.join(static_dir, "index.html"))

View File

@@ -0,0 +1,27 @@
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, JSON
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
import datetime
Base = declarative_base()
class Queue(Base):
__tablename__ = "queues"
id = Column(String, primary_key=True, index=True)
status = Column(String, default="Waiting") # Finished, Waiting, Running, Aborted
created_at = Column(DateTime, default=datetime.datetime.utcnow)
environment = Column(String)
tasks = relationship("Task", back_populates="queue", cascade="all, delete-orphan")
class Task(Base):
__tablename__ = "tasks"
id = Column(String, primary_key=True, index=True)
queue_id = Column(String, ForeignKey("queues.id"))
scenario_path = Column(String)
status = Column(String, default="Waiting") # Finished, Waiting, Running, Aborted
result = Column(JSON, nullable=True)
queue = relationship("Queue", back_populates="tasks")

View File

@@ -0,0 +1,407 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>TestArena | Modern Dashboard</title>
<link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;600;700&display=swap" rel="stylesheet">
<style>
:root {
--primary: #6366f1;
--primary-glow: rgba(99, 102, 241, 0.5);
--secondary: #ec4899;
--accent: #8b5cf6;
--bg: #0f172a;
--card-bg: rgba(30, 41, 59, 0.7);
--text: #f8fafc;
--text-muted: #94a3b8;
--success: #10b981;
--warning: #f59e0b;
--danger: #ef4444;
--glass: rgba(255, 255, 255, 0.05);
--glass-border: rgba(255, 255, 255, 0.1);
}
* {
box-sizing: border-box;
margin: 0;
padding: 0;
}
body {
font-family: 'Outfit', sans-serif;
background: radial-gradient(circle at top right, #1e1b4b, #0f172a);
color: var(--text);
min-height: 100vh;
padding: 2rem;
overflow-x: hidden;
}
.container {
max-width: 1200px;
margin: 0 auto;
position: relative;
}
/* Decorative blobs */
.blob {
position: absolute;
width: 300px;
height: 300px;
background: var(--primary-glow);
filter: blur(100px);
border-radius: 50%;
z-index: -1;
animation: move 20s infinite alternate;
}
@keyframes move {
from {
transform: translate(-10%, -10%);
}
to {
transform: translate(20%, 20%);
}
}
header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 3rem;
padding: 1.5rem;
background: var(--glass);
backdrop-filter: blur(12px);
border: 1px solid var(--glass-border);
border-radius: 1.5rem;
}
.logo {
display: flex;
align-items: center;
gap: 0.75rem;
font-size: 1.75rem;
font-weight: 700;
background: linear-gradient(to right, var(--primary), var(--secondary));
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
}
.nav-links {
display: flex;
gap: 1.5rem;
}
.nav-links a {
color: var(--text);
text-decoration: none;
font-weight: 600;
transition: color 0.3s;
}
.nav-links a:hover {
color: var(--primary);
}
.status-badge {
padding: 0.5rem 1rem;
border-radius: 1rem;
font-size: 0.875rem;
font-weight: 600;
display: flex;
align-items: center;
gap: 0.5rem;
background: var(--glass);
border: 1px solid var(--glass-border);
}
.dot {
width: 8px;
height: 8px;
border-radius: 50%;
background: var(--warning);
box-shadow: 0 0 10px var(--warning);
}
.dot.online {
background: var(--success);
box-shadow: 0 0 10px var(--success);
}
.grid {
display: grid;
grid-template-columns: 2fr 1fr;
gap: 2rem;
}
.card {
background: var(--card-bg);
backdrop-filter: blur(16px);
border: 1px solid var(--glass-border);
border-radius: 1.5rem;
padding: 2rem;
box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.5);
}
h2 {
font-size: 1.25rem;
margin-bottom: 1.5rem;
display: flex;
align-items: center;
gap: 0.5rem;
}
table {
width: 100%;
border-collapse: separate;
border-spacing: 0 0.75rem;
}
th {
text-align: left;
color: var(--text-muted);
font-weight: 600;
padding: 0 1rem;
font-size: 0.875rem;
}
td {
padding: 1rem;
background: rgba(255, 255, 255, 0.03);
}
td:first-child {
border-radius: 1rem 0 0 1rem;
}
td:last-child {
border-radius: 0 1rem 1rem 0;
}
.status-pill {
padding: 0.25rem 0.75rem;
border-radius: 0.75rem;
font-size: 0.75rem;
font-weight: 700;
text-transform: uppercase;
}
.status-waiting {
background: rgba(148, 163, 184, 0.1);
color: #94a3b8;
}
.status-running {
background: rgba(99, 102, 241, 0.1);
color: #818cf8;
border: 1px solid rgba(99, 102, 241, 0.3);
}
.status-finished {
background: rgba(16, 185, 129, 0.1);
color: #34d399;
}
.status-aborted {
background: rgba(239, 68, 68, 0.1);
color: #f87171;
}
.btn-abort {
background: rgba(239, 68, 68, 0.1);
color: #f87171;
border: 1px solid rgba(239, 68, 68, 0.2);
padding: 0.4rem 0.8rem;
border-radius: 0.75rem;
cursor: pointer;
font-weight: 600;
transition: all 0.3s;
}
.btn-abort:hover {
background: var(--danger);
color: white;
}
.log-container {
background: #020617;
border-radius: 1rem;
padding: 1.25rem;
height: 400px;
overflow-y: auto;
font-family: 'Fira Code', monospace;
font-size: 0.8125rem;
line-height: 1.6;
border: 1px solid var(--glass-border);
}
.log-entry {
margin-bottom: 0.5rem;
display: flex;
gap: 0.75rem;
}
.log-time {
color: var(--primary);
opacity: 0.7;
}
.log-msg {
color: #cbd5e1;
}
/* Custom Scrollbar */
::-webkit-scrollbar {
width: 8px;
}
::-webkit-scrollbar-track {
background: transparent;
}
::-webkit-scrollbar-thumb {
background: var(--glass-border);
border-radius: 4px;
}
::-webkit-scrollbar-thumb:hover {
background: rgba(255, 255, 255, 0.2);
}
</style>
</head>
<body>
<div class="blob"></div>
<div class="container">
<header>
<div class="logo">
<svg width="32" height="32" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5"
stroke-linecap="round" stroke-linejoin="round">
<path d="M12 2L2 7l10 5 10-5-10-5zM2 17l10 5 10-5M2 12l10 5 10-5" />
</svg>
TestArena
</div>
<nav class="nav-links">
<a href="/">Dashboard</a>
<a href="/results/" target="_blank">Browse Results</a>
</nav>
<div id="connection-status" class="status-badge">
<div class="dot"></div>
<span>Connecting...</span>
</div>
</header>
<div class="grid">
<div class="card">
<h2>
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
stroke-linecap="round" stroke-linejoin="round">
<rect x="3" y="3" width="18" height="18" rx="2" ry="2" />
<line x1="3" y1="9" x2="21" y2="9" />
<line x1="9" y1="21" x2="9" y2="9" />
</svg>
Queue Monitor
</h2>
<table id="queue-table">
<thead>
<tr>
<th>Queue ID</th>
<th>Environment</th>
<th>Status</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
<!-- Dynamic content -->
</tbody>
</table>
</div>
<div class="card">
<h2>
<svg width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"
stroke-linecap="round" stroke-linejoin="round">
<path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z" />
</svg>
Live System Logs
</h2>
<div id="logs" class="log-container">
<div class="log-entry">
<span class="log-time">23:34:52</span>
<span class="log-msg">System initialized. Waiting for connection...</span>
</div>
</div>
</div>
</div>
</div>
<script>
async function fetchStatus() {
try {
const response = await fetch('/api/queues');
const queues = await response.json();
const tbody = document.querySelector('#queue-table tbody');
tbody.innerHTML = '';
queues.forEach(q => {
const tr = document.createElement('tr');
tr.innerHTML = `
<td style="font-weight: 600;">${q.id}</td>
<td><span style="opacity: 0.8;">${q.environment}</span></td>
<td><span class="status-pill status-${q.status.toLowerCase()}">${q.status}</span></td>
<td>
<button class="btn-abort" onclick="abortQueue('${q.id}')">Abort</button>
</td>
`;
tbody.appendChild(tr);
});
const badge = document.getElementById('connection-status');
badge.querySelector('.dot').classList.add('online');
badge.querySelector('span').textContent = 'System Online';
} catch (e) {
const badge = document.getElementById('connection-status');
badge.querySelector('.dot').classList.remove('online');
badge.querySelector('span').textContent = 'Connection Lost';
}
}
async function abortQueue(id) {
if (confirm(`Are you sure you want to abort queue ${id}?`)) {
try {
await fetch(`/api/abort/${id}`, { method: 'POST' });
addLog(`Aborted queue: ${id}`, 'danger');
fetchStatus();
} catch (e) {
addLog(`Failed to abort queue: ${id}`, 'danger');
}
}
}
function addLog(msg, type = 'info') {
const logs = document.getElementById('logs');
const entry = document.createElement('div');
entry.className = 'log-entry';
const time = new Date().toLocaleTimeString([], { hour12: false });
entry.innerHTML = `
<span class="log-time">${time}</span>
<span class="log-msg">${msg}</span>
`;
logs.appendChild(entry);
logs.scrollTop = logs.scrollHeight;
}
// Initial fetch and poll
fetchStatus();
setInterval(fetchStatus, 3000);
// Simulate some system logs
setTimeout(() => addLog("Database connection established."), 1000);
setTimeout(() => addLog("Background worker is polling for tasks..."), 2000);
</script>
</body>
</html>

View File

@@ -0,0 +1,98 @@
import time
import subprocess
import json
import os
from sqlalchemy.orm import Session
from . import models, database
# Base directory for data
BASE_DATA_DIR = "/home/asf/testarena"
if os.name == 'nt':
BASE_DATA_DIR = "d:/ASF - course/ASF_01/ASF_tools/asf-pc-server/testarena_pc_backend/testarena_data"
def update_json_status(queue_id, task_id, status, result=None):
queue_dir = os.path.join(BASE_DATA_DIR, queue_id)
status_file = os.path.join(queue_dir, "queue_status.json")
if os.path.exists(status_file):
with open(status_file, 'r') as f:
data = json.load(f)
if task_id:
data["tasks"][task_id] = status
else:
data["status"] = status
if result:
data["results"] = data.get("results", {})
data["results"][task_id] = result
with open(status_file, 'w') as f:
json.dump(data, f, indent=4)
def run_worker():
print("Worker started...")
while True:
db = database.SessionLocal()
try:
# Get next waiting queue
queue = db.query(models.Queue).filter(models.Queue.status == "Waiting").order_by(models.Queue.created_at).first()
if queue:
print(f"Processing queue: {queue.id}")
queue.status = "Running"
update_json_status(queue.id, None, "Running")
db.commit()
tasks = db.query(models.Task).filter(models.Task.queue_id == queue.id, models.Task.status == "Waiting").all()
for task in tasks:
# Check if queue was aborted mid-way
db.refresh(queue)
if queue.status == "Aborted":
break
print(f"Running task: {task.id}")
task.status = "Running"
update_json_status(queue.id, task.id, "Running")
db.commit()
try:
# Run tpf_execution.py [queue_id, scenario_path, task_id]
# Assuming tpf_execution.py is in the parent directory or accessible
script_path = "tpf_execution.py"
# For testing, let's assume it's in the same dir as the app or parent
cmd = ["python", script_path, queue.id, task.scenario_path, task.id]
result = subprocess.run(cmd, capture_output=True, text=True)
# Parse result if it returns json
try:
execution_result = json.loads(result.stdout)
except:
execution_result = {"output": result.stdout, "error": result.stderr}
task.status = "Finished"
task.result = execution_result
update_json_status(queue.id, task.id, "Finished", execution_result)
except Exception as e:
print(f"Error running task {task.id}: {e}")
task.status = "Error"
update_json_status(queue.id, task.id, "Error")
db.commit()
if queue.status != "Aborted":
queue.status = "Finished"
update_json_status(queue.id, None, "Finished")
db.commit()
time.sleep(5) # Poll every 5 seconds
except Exception as e:
print(f"Worker error: {e}")
time.sleep(10)
finally:
db.close()
if __name__ == "__main__":
run_worker()

View File

@@ -0,0 +1,32 @@
import sys
import json
import time
import random
def main():
if len(sys.argv) < 4:
print("Usage: python tpf_execution.py <queue_id> <scenario_path> <task_id>")
sys.exit(1)
queue_id = sys.argv[1]
scenario_path = sys.argv[2]
task_id = sys.argv[3]
print(f"Starting execution for Task: {task_id} in Queue: {queue_id}")
print(f"Scenario: {scenario_path}")
# Simulate work
duration = random.randint(2, 5)
time.sleep(duration)
result = {
"task_id": task_id,
"status": "Success",
"duration": duration,
"details": f"Scenario {scenario_path} executed successfully."
}
print(json.dumps(result))
if __name__ == "__main__":
main()