next_updates

This commit is contained in:
2025-12-28 05:52:49 +01:00
parent 986f3ae5b5
commit 6b254534f6
8 changed files with 766 additions and 533 deletions

View File

@@ -37,11 +37,18 @@ class Job(db.Model):
reuse_results = db.Column(db.Boolean, default=False)
results_path = db.Column(db.String(500), nullable=True)
# Phase 2 Remote Tracking
remote_queue_id = db.Column(db.String(50), nullable=True)
remote_task_ids = db.Column(db.Text, nullable=True) # JSON string: {scenario_name: task_id}
remote_results = db.Column(db.Text, nullable=True) # JSON string: {scenario_name: [status, link]}
queue_log = db.Column(db.Text, nullable=True)
def get_status_icon(self):
icons = {
'in_progress': '🟠',
'passed': '🟢',
'failed': '🔴',
'waiting': '',
'in_progress': '🔄',
'passed': '',
'failed': '',
'aborted': ''
}
return icons.get(self.status, '')

View File

@@ -5,14 +5,15 @@ from app import db
import json
import subprocess
import os
import requests
import random
import string
def generate_remote_id(length=6):
return ''.join(random.choices(string.digits, k=length))
jobs_bp = Blueprint('jobs', __name__, url_prefix='/jobs')
@jobs_bp.route('/submit')
@login_required
def submit():
return render_template('jobs/submit.html')
@jobs_bp.route('/debug/test-step2')
@login_required
def debug_test_step2():
@@ -270,7 +271,7 @@ def submit_step2():
flash('Please select at least one scenario', 'error')
return redirect(url_for('jobs.submit'))
return render_template('jobs/submit_step3.html',
return render_template('jobs/submit_review.html',
branch_name=branch_name,
scenarios=selected_scenarios,
scenario_map=scenario_map)
@@ -318,50 +319,14 @@ def submit_step2_validated():
print(f"[ERROR] Step2 - Traceback: {traceback.format_exc()}")
flash(f'Error loading scenario selection: {str(e)}', 'error')
return redirect(url_for('jobs.submit'))
@login_required
def submit_step2():
branch_name = request.form.get('branch_name')
selected_scenarios = request.form.getlist('scenarios')
if not selected_scenarios:
flash('Please select at least one scenario', 'error')
return redirect(url_for('jobs.submit'))
return render_template('jobs/submit_step3.html',
branch_name=branch_name,
scenarios=selected_scenarios)
@jobs_bp.route('/submit/step3', methods=['POST'])
@login_required
def submit_step3():
branch_name = request.form.get('branch_name')
scenarios_json = request.form.get('scenarios')
scenario_map_json = request.form.get('scenario_map')
environment = request.form.get('environment')
try:
scenarios = json.loads(scenarios_json) if scenarios_json else []
scenario_map = json.loads(scenario_map_json) if scenario_map_json else {}
except json.JSONDecodeError:
flash('Invalid scenario data', 'error')
return redirect(url_for('jobs.submit'))
return render_template('jobs/submit_step4.html',
branch_name=branch_name,
scenarios=scenarios,
scenario_map=scenario_map,
environment=environment)
@jobs_bp.route('/submit/final', methods=['POST'])
@login_required
def submit_final():
branch_name = request.form.get('branch_name')
scenarios_json = request.form.get('scenarios')
scenario_map_json = request.form.get('scenario_map')
environment = request.form.get('environment')
test_mode = request.form.get('test_mode')
keep_devbenches = request.form.get('keep_devbenches') == 'on'
reuse_results = request.form.get('reuse_results') == 'on'
environment = request.form.get('environment', 'staging')
test_mode = request.form.get('test_mode', 'simulator')
try:
scenarios = json.loads(scenarios_json) if scenarios_json else []
@@ -370,22 +335,44 @@ def submit_final():
flash('Invalid scenario data', 'error')
return redirect(url_for('jobs.submit'))
# Generate Remote IDs
remote_queue_id = generate_remote_id()
remote_task_ids = {s: generate_remote_id() for s in scenarios}
# Prepare Remote Payload
# Format: {"source": "branch", "queue_id": ["staging", {"task_id": "path"}]}
payload = {
"source": branch_name,
remote_queue_id: [
environment,
{remote_task_ids[s]: scenario_map.get(s, s) for s in scenarios}
]
}
# Trigger Remote Queue
try:
remote_url = "http://asf-server.duckdns.org:8080/api/queue"
response = requests.post(remote_url, json=payload, timeout=10)
response.raise_for_status()
print(f"[DEBUG] Remote queue triggered: {response.json()}")
except Exception as e:
print(f"[ERROR] Failed to trigger remote queue: {e}")
flash(f'Warning: Job saved but failed to trigger remote queue: {str(e)}', 'warning')
job = Job(
user_id=current_user.id,
branch_name=branch_name,
scenarios=json.dumps(scenarios), # Store as JSON string
scenarios=json.dumps(scenarios),
environment=environment,
test_mode=test_mode,
keep_devbenches=keep_devbenches,
reuse_results=reuse_results,
status='in_progress'
status='waiting', # Start with waiting (sand watch)
remote_queue_id=remote_queue_id,
remote_task_ids=json.dumps(remote_task_ids)
)
db.session.add(job)
db.session.commit()
# TODO: Start test execution in background using scenario_map
flash('Test job submitted successfully', 'success')
return redirect(url_for('dashboard.index'))
@@ -409,23 +396,96 @@ def view_job(job_id):
'submitted_at': job.submitted_at.strftime('%Y-%m-%d %H:%M:%S'),
'completed_at': job.completed_at.strftime('%Y-%m-%d %H:%M:%S') if job.completed_at else None,
'duration': job.duration,
'keep_devbenches': job.keep_devbenches,
'reuse_results': job.reuse_results,
'results_path': job.results_path
'remote_queue_id': job.remote_queue_id,
'remote_task_ids': job.remote_task_ids,
'remote_results': job.remote_results,
'queue_log': job.queue_log
})
@jobs_bp.route('/<int:job_id>/abort', methods=['POST'])
@login_required
def abort_job(job_id):
return jsonify({'error': 'Abort functionality is not implemented yet'}), 400
@jobs_bp.route('/<int:job_id>/status')
@login_required
def get_job_status(job_id):
job = Job.query.get_or_404(job_id)
if not current_user.is_admin and job.user_id != current_user.id:
return jsonify({'error': 'Access denied'}), 403
if job.status not in ['passed', 'failed', 'aborted']:
# Poll remote API
try:
# 1. Check Queue Status
status_url = f"http://asf-server.duckdns.org:8080/api/status/{job.remote_queue_id}"
q_resp = requests.get(status_url, timeout=5)
if q_resp.status_code == 200:
q_data = q_resp.json()
remote_status = q_data.get('status', '').lower()
if job.status == 'in_progress':
job.status = 'aborted'
if remote_status == 'running':
job.status = 'in_progress'
elif remote_status == 'waiting':
job.status = 'waiting'
# 2. Fetch Queue Logs if running
if remote_status == 'running':
log_url = f"http://asf-server.duckdns.org:8080/results/{job.remote_queue_id}/queue_log.txt"
l_resp = requests.get(log_url, timeout=5)
if l_resp.status_code == 200:
job.queue_log = l_resp.text
# 3. Check Task Statuses and Results
task_ids = json.loads(job.remote_task_ids) if job.remote_task_ids else {}
results = json.loads(job.remote_results) if job.remote_results else {}
all_finished = True
any_failed = False
for scenario, task_id in task_ids.items():
# If we don't have result yet, check it
if scenario not in results:
# Check task status
t_status_url = f"http://asf-server.duckdns.org:8080/api/status/{task_id}"
t_resp = requests.get(t_status_url, timeout=2)
if t_resp.status_code == 200:
t_data = t_resp.json()
if t_data.get('status') == 'Finished':
# Fetch results
res_url = f"http://asf-server.duckdns.org:8080/results/{job.remote_queue_id}/{task_id}/final_summary.json"
r_resp = requests.get(res_url, timeout=2)
if r_resp.status_code == 200:
r_data = r_resp.json()
# User says: {"SCENARIO_NAME": ["PASS/FAIL", "link"]}
# We need to find the key that matches our scenario (case insensitive maybe?)
for key, val in r_data.items():
if key.upper() == scenario.upper():
results[scenario] = val
if val[0] == 'FAIL':
any_failed = True
break
else:
all_finished = False
else:
all_finished = False
else:
all_finished = False
else:
if results[scenario][0] == 'FAIL':
any_failed = True
if all_finished and task_ids:
job.status = 'failed' if any_failed else 'passed'
job.completed_at = db.session.query(db.func.now()).scalar()
job.remote_results = json.dumps(results)
db.session.commit()
# TODO: Kill the running process
return jsonify({'success': True})
return jsonify({'error': 'Job is not in progress'}), 400
except Exception as e:
print(f"[ERROR] Status polling failed: {e}")
return jsonify({
'status': job.status,
'status_icon': job.get_status_icon(),
'remote_results': job.remote_results,
'queue_log': job.queue_log
})

View File

@@ -623,8 +623,8 @@ body {
background: #9ca3af;
cursor: not-allowed;
opacity: 0.6;
}/
* Scenario Tree Styles */
}
/* Scenario Tree Styles */
.scenario-controls {
margin: 20px 0;
padding: 15px;
@@ -754,3 +754,33 @@ input[type="checkbox"]:indeterminate::before {
color: var(--primary);
font-size: 14px;
}
/* Scenario Table & Log Styles */
#scenarioTable {
margin-top: 10px;
font-size: 13px;
}
#scenarioTable th {
padding: 10px;
background: #f8fafc;
}
#scenarioTable td {
padding: 10px;
}
#queue-log {
border: 1px solid #333;
box-shadow: inset 0 2px 10px rgba(0,0,0,0.5);
}
#scenarioSearch:focus {
border-color: var(--primary);
box-shadow: 0 0 0 3px rgba(37, 99, 235, 0.1);
}
.status-waiting {
background: #fef3c7;
color: #92400e;
}

View File

@@ -13,7 +13,8 @@
<div class="job-list">
{% if jobs %}
{% for job in jobs %}
<div class="job-item" data-job-id="{{ job.id }}" onclick="loadJobDetails({{ job.id }})" oncontextmenu="showContextMenu(event, {{ job.id }}, '{{ job.status }}')">
<div class="job-item" data-job-id="{{ job.id }}" onclick="loadJobDetails({{ job.id }})"
oncontextmenu="showContextMenu(event, {{ job.id }}, '{{ job.status }}')">
<div class="job-status-icon">{{ job.get_status_icon() }}</div>
<div class="job-info">
<h4>Job #{{ job.id }} - {{ job.branch_name }}</h4>
@@ -86,28 +87,57 @@ function abortJobFromContext() {
}
hideContextMenu();
}
let pollingInterval = null;
function loadJobDetails(jobId) {
// Clear existing polling
if (pollingInterval) {
clearInterval(pollingInterval);
pollingInterval = null;
}
// Mark job as active
document.querySelectorAll('.job-item').forEach(item => {
item.classList.remove('active');
});
document.querySelector(`[data-job-id="${jobId}"]`).classList.add('active');
const jobItem = document.querySelector(`[data-job-id="${jobId}"]`);
if (jobItem) jobItem.classList.add('active');
// Fetch job details
// Initial fetch
fetchJobDetails(jobId);
// Start polling if job is not finished
pollingInterval = setInterval(() => {
fetchJobDetails(jobId, true);
}, 5000);
}
function fetchJobDetails(jobId, isPolling = false) {
fetch(`/jobs/${jobId}`)
.then(response => response.json())
.then(job => {
const container = document.getElementById('job-details-container');
const scenarios = JSON.parse(job.scenarios || '[]');
const taskIds = JSON.parse(job.remote_task_ids || '{}');
const results = JSON.parse(job.remote_results || '{}');
container.innerHTML = `
// If polling and job is finished, stop polling
if (isPolling && ['passed', 'failed', 'aborted'].includes(job.status)) {
clearInterval(pollingInterval);
pollingInterval = null;
}
// Update job item icon in the list if it changed
const jobItem = document.querySelector(`[data-job-id="${jobId}"]`);
if (jobItem) {
// We need to fetch the icon separately or update it here
// For now, let's just update the details panel
}
let html = `
<div class="detail-row">
<div class="detail-label">Job ID:</div>
<div class="detail-value">#${job.id}</div>
</div>
<div class="detail-row">
<div class="detail-label">Submitter:</div>
<div class="detail-value">${job.submitter}</div>
<div class="detail-value">#${job.id} (Remote: ${job.remote_queue_id || 'N/A'})</div>
</div>
<div class="detail-row">
<div class="detail-label">Branch:</div>
@@ -115,48 +145,100 @@ function loadJobDetails(jobId) {
</div>
<div class="detail-row">
<div class="detail-label">Status:</div>
<div class="detail-value">
<div class="detail-value" id="job-status-badge">
<span class="status-badge status-${job.status}">${job.status.replace('_', ' ').toUpperCase()}</span>
</div>
</div>
<div class="detail-row">
<div class="detail-label">Environment:</div>
<div class="detail-value">${job.environment}</div>
<div style="margin-top: 30px;">
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 15px;">
<h4 style="color: var(--dark);">Scenario Summary</h4>
<input type="text" id="scenarioSearch" placeholder="Search scenarios..." onkeyup="filterScenarios()" class="form-control" style="width: 200px; padding: 6px 12px;">
</div>
<div class="detail-row">
<div class="detail-label">Test Mode:</div>
<div class="detail-value">${job.test_mode}</div>
</div>
<div class="detail-row">
<div class="detail-label">Submitted:</div>
<div class="detail-value">${job.submitted_at}</div>
</div>
${job.completed_at ? `
<div class="detail-row">
<div class="detail-label">Completed:</div>
<div class="detail-value">${job.completed_at}</div>
</div>
<div class="detail-row">
<div class="detail-label">Duration:</div>
<div class="detail-value">${job.duration ? Math.floor(job.duration / 60) + 'm ' + (job.duration % 60) + 's' : 'N/A'}</div>
</div>
` : ''}
<div class="detail-row">
<div class="detail-label">Scenarios:</div>
<div class="detail-value">${Array.isArray(scenarios) ? scenarios.join(', ') : scenarios}</div>
</div>
${job.status === 'in_progress' ? `
<div style="margin-top: 20px;">
<button class="btn btn-danger" onclick="abortJob(${job.id})">Abort Job</button>
</div>
` : ''}
${job.results_path ? `
<div style="margin-top: 20px;">
<a href="${job.results_path}" class="btn btn-primary" target="_blank">View Results</a>
</div>
` : ''}
<table class="user-table" id="scenarioTable">
<thead>
<tr>
<th>Scenario Name</th>
<th>Task ID</th>
<th>Status</th>
</tr>
</thead>
<tbody>
`;
scenarios.forEach(s => {
const taskId = taskIds[s] || 'N/A';
const result = results[s]; // [status, link]
let statusHtml = '⌛ Waiting';
if (result) {
const icon = result[0] === 'PASS' ? '✅' : '';
const color = result[0] === 'PASS' ? 'var(--success)' : 'var(--danger)';
statusHtml = `<a href="${result[1]}" target="_blank" style="text-decoration: none; color: ${color}; font-weight: 600;">${icon} ${result[0]}</a>`;
} else if (job.status === 'in_progress') {
statusHtml = '🔄 Running';
}
html += `
<tr>
<td>${s}</td>
<td><small>${taskId}</small></td>
<td>${statusHtml}</td>
</tr>
`;
});
html += `
</tbody>
</table>
</div>
<div style="margin-top: 30px;">
<h4 style="color: var(--dark); margin-bottom: 15px;">Queue Logging</h4>
<div id="queue-log" style="background: #1e1e1e; color: #d4d4d4; padding: 15px; border-radius: 8px; font-family: 'Consolas', monospace; font-size: 12px; max-height: 400px; overflow-y: auto; white-space: pre-wrap;">${job.queue_log || 'Waiting for logs...'}</div>
</div>
<div style="margin-top: 30px; display: flex; gap: 10px;">
<button class="btn btn-danger" onclick="abortJob(${job.id})" style="width: auto;">Abort Job</button>
</div>
`;
container.innerHTML = html;
// Scroll log to bottom if it's being updated
const logElement = document.getElementById('queue-log');
if (logElement) logElement.scrollTop = logElement.scrollHeight;
// If in progress, trigger status update on backend
if (!['passed', 'failed', 'aborted'].includes(job.status)) {
fetch(`/jobs/${jobId}/status`)
.then(r => r.json())
.then(data => {
// Update icon in list
const iconEl = document.querySelector(`[data-job-id="${jobId}"] .job-status-icon`);
if (iconEl) iconEl.textContent = data.status_icon;
});
}
});
}
function filterScenarios() {
const input = document.getElementById('scenarioSearch');
const filter = input.value.toUpperCase();
const table = document.getElementById('scenarioTable');
const tr = table.getElementsByTagName('tr');
for (let i = 1; i < tr.length; i++) {
const td = tr[i].getElementsByTagName('td')[0];
if (td) {
const txtValue = td.textContent || td.innerText;
if (txtValue.toUpperCase().indexOf(filter) > -1) {
tr[i].style.display = "";
} else {
tr[i].style.display = "none";
}
}
}
}
function abortJob(jobId) {

View File

@@ -17,14 +17,6 @@
</div>
<div class="step">
<div class="step-number">3</div>
<div class="step-label">Environment</div>
</div>
<div class="step">
<div class="step-number">4</div>
<div class="step-label">Test Mode</div>
</div>
<div class="step">
<div class="step-number">5</div>
<div class="step-label">Review</div>
</div>
</div>
@@ -47,7 +39,8 @@
<div class="form-actions">
<a href="{{ url_for('dashboard.index') }}" class="btn" style="background: #6b7280; color: white;">Cancel</a>
<button type="submit" id="validateBtn" class="btn btn-primary">Validate Branch</button>
<button type="button" id="nextBtn" class="btn btn-primary" style="display: none;" onclick="proceedToStep2()">Next</button>
<button type="button" id="nextBtn" class="btn btn-primary" style="display: none;"
onclick="proceedToStep2()">Next</button>
</div>
</form>
</div>

View File

@@ -0,0 +1,57 @@
{% extends "base.html" %}
{% block title %}Review & Submit - ASF TestArena{% endblock %}
{% block content %}
<div class="submit-container">
<h2 style="margin-bottom: 30px; color: var(--dark);">Review & Submit</h2>
<div class="step-indicator">
<div class="step completed">
<div class="step-number"></div>
<div class="step-label">Branch</div>
</div>
<div class="step completed">
<div class="step-number"></div>
<div class="step-label">Scenarios</div>
</div>
<div class="step active">
<div class="step-number">3</div>
<div class="step-label">Review</div>
</div>
</div>
<div style="background: white; border: 1px solid var(--border); border-radius: 8px; padding: 20px; margin-bottom: 30px;">
<div class="detail-row">
<div class="detail-label">Branch:</div>
<div class="detail-value">{{ branch_name }}</div>
</div>
<div class="detail-row">
<div class="detail-label">Scenarios:</div>
<div class="detail-value">
<strong>{{ scenarios|length }} scenarios selected</strong>
<ul style="margin-top: 10px; padding-left: 20px; max-height: 200px; overflow-y: auto;">
{% for scenario in scenarios %}
<li style="font-size: 13px; color: #4b5563; margin-bottom: 4px;">{{ scenario }}</li>
{% endfor %}
</ul>
</div>
</div>
</div>
<form method="POST" action="{{ url_for('jobs.submit_final') }}">
<input type="hidden" name="branch_name" value="{{ branch_name }}">
<input type="hidden" name="scenarios" value="{{ scenarios|tojson }}">
<input type="hidden" name="scenario_map" value="{{ scenario_map|tojson }}">
<!-- Default values for deactivated sections -->
<input type="hidden" name="environment" value="staging">
<input type="hidden" name="test_mode" value="simulator">
<div class="form-actions">
<button type="button" class="btn" style="background: #6b7280; color: white;" onclick="history.back()">Back</button>
<button type="submit" class="btn btn-success" style="padding: 12px 40px;">Submit Job</button>
</div>
</form>
</div>
{% endblock %}

View File

@@ -17,14 +17,6 @@
</div>
<div class="step">
<div class="step-number">3</div>
<div class="step-label">Environment</div>
</div>
<div class="step">
<div class="step-number">4</div>
<div class="step-label">Test Mode</div>
</div>
<div class="step">
<div class="step-number">5</div>
<div class="step-label">Review</div>
</div>
</div>
@@ -39,8 +31,10 @@
<input type="hidden" name="selected_scenarios" id="selectedScenariosInput">
<div class="scenario-controls">
<button type="button" class="btn btn-sm" onclick="selectAll()" style="background: var(--success); color: white; margin-right: 10px;">Select All</button>
<button type="button" class="btn btn-sm" onclick="deselectAll()" style="background: var(--danger); color: white;">Deselect All</button>
<button type="button" class="btn btn-sm" onclick="selectAll()"
style="background: var(--success); color: white; margin-right: 10px;">Select All</button>
<button type="button" class="btn btn-sm" onclick="deselectAll()"
style="background: var(--danger); color: white;">Deselect All</button>
<span id="selectionCount" style="margin-left: 20px; font-weight: 600;">0 scenarios selected</span>
</div>
@@ -50,8 +44,10 @@
<div class="tree-layer">
<div class="tree-node layer-node" onclick="toggleLayer('{{ layer_name }}')">
<span class="tree-toggle" id="toggle-{{ layer_name }}"></span>
<input type="checkbox" class="layer-checkbox" id="layer-{{ layer_name }}" onchange="toggleLayerSelection('{{ layer_name }}')">
<label for="layer-{{ layer_name }}" class="tree-label layer-label">{{ layer_name.replace('_', ' ').title() }}</label>
<input type="checkbox" class="layer-checkbox" id="layer-{{ layer_name }}"
onchange="toggleLayerSelection('{{ layer_name }}')">
<label for="layer-{{ layer_name }}" class="tree-label layer-label">{{ layer_name.replace('_', '
').title() }}</label>
</div>
<div class="tree-children" id="children-{{ layer_name }}" style="display: none;">
@@ -60,18 +56,24 @@
<div class="tree-stack">
<div class="tree-node stack-node" onclick="toggleStack('{{ layer_name }}', '{{ stack_name }}')">
<span class="tree-toggle" id="toggle-{{ layer_name }}-{{ stack_name }}"></span>
<input type="checkbox" class="stack-checkbox" id="stack-{{ layer_name }}-{{ stack_name }}" onchange="toggleStackSelection('{{ layer_name }}', '{{ stack_name }}')">
<label for="stack-{{ layer_name }}-{{ stack_name }}" class="tree-label stack-label">{{ stack_name.replace('_', ' ').title() }}</label>
<input type="checkbox" class="stack-checkbox" id="stack-{{ layer_name }}-{{ stack_name }}"
onchange="toggleStackSelection('{{ layer_name }}', '{{ stack_name }}')">
<label for="stack-{{ layer_name }}-{{ stack_name }}" class="tree-label stack-label">{{
stack_name.replace('_', ' ').title() }}</label>
</div>
<div class="tree-children" id="children-{{ layer_name }}-{{ stack_name }}" style="display: none;">
<div class="tree-children" id="children-{{ layer_name }}-{{ stack_name }}"
style="display: none;">
{% if scenarios %}
{% for scenario in scenarios %}
<div class="tree-scenario">
<div class="tree-node scenario-node">
<span class="tree-spacer"></span>
<input type="checkbox" class="scenario-checkbox" id="scenario-{{ scenario }}" value="{{ scenario }}" onchange="updateSelectionCount()" data-layer="{{ layer_name }}" data-stack="{{ stack_name }}">
<label for="scenario-{{ scenario }}" class="tree-label scenario-label">{{ scenario }}</label>
<input type="checkbox" class="scenario-checkbox" id="scenario-{{ scenario }}"
value="{{ scenario }}" onchange="updateSelectionCount()"
data-layer="{{ layer_name }}" data-stack="{{ stack_name }}">
<label for="scenario-{{ scenario }}" class="tree-label scenario-label">{{ scenario
}}</label>
</div>
</div>
{% endfor %}
@@ -79,7 +81,8 @@
<div class="tree-scenario">
<div class="tree-node scenario-node">
<span class="tree-spacer"></span>
<span class="tree-label scenario-label" style="color: #9ca3af;">No scenarios found</span>
<span class="tree-label scenario-label" style="color: #9ca3af;">No scenarios
found</span>
</div>
</div>
{% endif %}

View File

@@ -7,3 +7,4 @@ gunicorn==21.2.0
python-dotenv==1.0.0
Werkzeug==3.0.1
WTForms==3.1.1
requests==2.31.0