{% extends "base.html" %} {% block title %}Background Tasks{% endblock %} {% block breadcrumb %}Background Tasks{% endblock %} {% block extra_head %} {% endblock %} {% block content %} {# ═══════════════════════════════════════════════════════════════════ AQUILIA ADMIN — BACKGROUND TASK MONITOR (v2 — Chart.js) Throughput Charts · Duration Histogram · Queue Health Worker Status · Job Table · Live Polling ═══════════════════════════════════════════════════════════════════ #}

Background Tasks

Monitor background job queue, worker status, retries, and dead-letter jobs

{# ═══ MANAGER STATUS BAR ═══ #} {% set mgr = manager %}
{% if available and mgr.running is defined and mgr.running %}
Running {% else %}
{{ 'Stopped' if available else 'Not Configured' }} {% endif %}
{% if available and mgr.num_workers is defined %} · {{ mgr.num_workers }} workers · {{ mgr.backend }} backend {% if mgr.uptime_seconds is defined and mgr.uptime_seconds > 0 %} · Uptime: {{ "%.0f"|format(mgr.uptime_seconds) }}s {% endif %} {% endif %}
{# ═══ METRIC CARDS (extended) ═══ #}
0
Total Jobs
{% if registered_tasks and registered_tasks|length > 0 %}{{ registered_tasks|length }} task{{ 's' if registered_tasks|length != 1 else '' }} registered{% elif mgr.total_enqueued is defined %}{{ mgr.total_enqueued }} enqueued{% else %}All time{% endif %}
0
Completed
{% if stats.avg_duration_ms is defined %}Avg {{ "%.1f"|format(stats.avg_duration_ms) }}ms{% else %}Successfully processed{% endif %}
0
Active
{{ pending_count }} pending
0
Failed
{{ dead_letter_count }} dead-letter
{{ success_rate }}%
Success Rate
All-time
{{ p50_ms }}
P50 (ms)
P95: {{ p95_ms }}ms · P99: {{ p99_ms }}ms
{# ═══ CHARTS ROW 1: Throughput + State Doughnut ═══ #}
Job Analytics
{# ── Throughput Line Chart ── #}
Throughput — Last 24h
{# ── Job State Doughnut ── #}
Job States
{# ═══ CHARTS ROW 2: Duration Histogram + Queue Breakdown ═══ #}
{# ── Duration Histogram ── #}
Duration Distribution
{# ── Queue Stacked Bar ── #}
Queue Health
{# ═══ QUEUE BREAKDOWN (detail cards) ═══ #} {% if queue_stats and queue_stats|length > 0 %}
Queue Breakdown
{% for queue_name, states in queue_stats.items() %}
{{ queue_name }}
{% for state, count in states.items() %}
{{ count }}
{{ state }}
{% endfor %}
{% endfor %}
{% endif %} {# ═══ REGISTERED TASK DEFINITIONS ═══ #} {% if registered_tasks and registered_tasks|length > 0 %}
Registered Tasks {{ registered_tasks|length }} task{{ 's' if registered_tasks|length != 1 else '' }} discovered
{% for t in registered_tasks %} {% endfor %}
Task Name Queue Priority Max Retries Timeout Retry Delay Tags
{{ t.name }}
{{ t.queue }} {{ t.priority }} {{ t.max_retries }} {{ t.timeout }}s {{ t.retry_delay }}s ×{{ t.retry_backoff }} {% if t.tags and t.tags|length > 0 %} {% for tag in t.tags %} {{ tag }} {% endfor %} {% else %} — {% endif %}
{% endif %} {# ═══ JOB LIST WITH FILTERS ═══ #}
Recent Jobs
{% if jobs and jobs|length > 0 %} {% for job in jobs[:50] %} {% if job.state in ('failed', 'dead') and job.result and job.result.error %} {% endif %} {% endfor %}
Job ID Task Queue Priority State Retries Duration Created
{{ job.id[:12] }}… {{ job.name or job.func_ref }} {{ job.queue }} {{ job.priority }} {% set state_colors = {'completed': 'var(--success)', 'running': 'var(--accent)', 'pending': 'var(--text-secondary)', 'failed': 'var(--danger)', 'dead': 'var(--danger)', 'retrying': 'var(--warning)', 'cancelled': 'var(--text-faint)', 'scheduled': 'var(--info)'} %} {{ job.state }} {{ job.retry_count }}/{{ job.max_retries }} {% if job.duration_ms is not none %} {{ "%.1f"|format(job.duration_ms) }}ms {% else %}—{% endif %} {{ job.created_at[:19] if job.created_at else '—' }}
{{ job.result.error_type }}: {{ job.result.error|truncate(200) }}
{% else %}

No background jobs recorded

{% if available and registered_tasks and registered_tasks|length > 0 %} {{ registered_tasks|length }} task{{ 's' if registered_tasks|length != 1 else '' }} registered. Jobs will appear here when tasks are enqueued via manager.enqueue(). {% elif available %} Jobs will appear here as your application enqueues background tasks. {% else %} Configure a TaskManager and register it with the admin site to enable monitoring. {% endif %}

{% endif %}
{# ═══ SLIDE-OUT DETAIL DRAWER ═══ #}

Detail

{# Content injected by JS #}
{% endblock %} {% block extra_js %} // ── Chart.js configuration ────────────────────────────────────── (function() { 'use strict'; if (typeof Chart === 'undefined') return; var isDark = document.documentElement.getAttribute('data-theme') !== 'light'; var gridColor = isDark ? 'rgba(255,255,255,0.06)' : 'rgba(0,0,0,0.08)'; var textColor = isDark ? 'rgba(255,255,255,0.5)' : 'rgba(0,0,0,0.5)'; Chart.defaults.color = textColor; Chart.defaults.borderColor = gridColor; Chart.defaults.font.family = "'Outfit', system-ui, sans-serif"; Chart.defaults.font.size = 11; Chart.defaults.plugins.legend.labels.boxWidth = 10; Chart.defaults.plugins.legend.labels.padding = 12; var charts = {{ charts | tojson }}; // ── Throughput Line Chart ──────────────────────────────────── var tp = charts.throughput || {}; if (tp.labels && tp.labels.length) { new Chart(document.getElementById('chart-throughput'), { type: 'line', data: { labels: tp.labels, datasets: [ { label: 'Completed', data: tp.completed || [], borderColor: '#22c55e', backgroundColor: 'rgba(34,197,94,0.08)', borderWidth: 2, fill: true, tension: 0.35, pointRadius: 0, pointHoverRadius: 4, }, { label: 'Failed', data: tp.failed || [], borderColor: '#ef4444', backgroundColor: 'rgba(239,68,68,0.06)', borderWidth: 2, fill: true, tension: 0.35, pointRadius: 0, pointHoverRadius: 4, }, ], }, options: { responsive: true, maintainAspectRatio: false, interaction: { intersect: false, mode: 'index' }, scales: { x: { grid: { display: false }, ticks: { maxTicksLimit: 8 } }, y: { beginAtZero: true, grid: { color: gridColor }, ticks: { stepSize: 1 } }, }, plugins: { legend: { position: 'top', align: 'end' } }, }, }); } // ── Job State Doughnut ─────────────────────────────────────── var sd = charts.state_doughnut || {}; if (sd.labels && sd.labels.length) { var stateColorMap = { 'completed': '#22c55e', 'running': '#3b82f6', 'pending': '#a1a1aa', 'failed': '#ef4444', 'dead': '#991b1b', 'retrying': '#f59e0b', 'cancelled': '#6b7280', 'scheduled': '#8b5cf6', }; var stateColors = sd.labels.map(function(l) { return stateColorMap[l] || '#6b7280'; }); new Chart(document.getElementById('chart-state-doughnut'), { type: 'doughnut', data: { labels: sd.labels, datasets: [{ data: sd.values, backgroundColor: stateColors, borderWidth: 0, hoverOffset: 6, }], }, options: { responsive: true, maintainAspectRatio: false, cutout: '68%', plugins: { legend: { position: 'bottom', labels: { padding: 10 } }, }, }, }); } // ── Duration Histogram ────────────────────────────────────── var dh = charts.duration_histogram || {}; if (dh.labels && dh.labels.length) { new Chart(document.getElementById('chart-duration'), { type: 'bar', data: { labels: dh.labels, datasets: [{ label: 'Jobs', data: dh.values, backgroundColor: [ '#22c55e', '#4ade80', '#86efac', '#fbbf24', '#f59e0b', '#f97316', '#ef4444', '#dc2626', ], borderWidth: 0, borderRadius: 4, }], }, options: { responsive: true, maintainAspectRatio: false, scales: { x: { grid: { display: false } }, y: { beginAtZero: true, grid: { color: gridColor }, ticks: { stepSize: 1 } }, }, plugins: { legend: { display: false } }, }, }); } // ── Queue Stacked Bar ─────────────────────────────────────── var qb = charts.queue_breakdown || {}; if (qb.labels && qb.labels.length) { new Chart(document.getElementById('chart-queues'), { type: 'bar', data: { labels: qb.labels, datasets: [ { label: 'Pending', data: qb.pending || [], backgroundColor: '#a1a1aa', borderRadius: 2 }, { label: 'Running', data: qb.running || [], backgroundColor: '#3b82f6', borderRadius: 2 }, { label: 'Completed', data: qb.completed || [], backgroundColor: '#22c55e', borderRadius: 2 }, { label: 'Failed', data: qb.failed || [], backgroundColor: '#ef4444', borderRadius: 2 }, ], }, options: { responsive: true, maintainAspectRatio: false, scales: { x: { stacked: true, grid: { display: false } }, y: { stacked: true, beginAtZero: true, grid: { color: gridColor }, ticks: { stepSize: 1 } }, }, plugins: { legend: { position: 'top', align: 'end' } }, }, }); } // ── Job table filter ──────────────────────────────────────── window.filterJobs = function(state) { var rows = document.querySelectorAll('.job-row'); rows.forEach(function(row) { if (state === 'all') { row.style.display = ''; } else { row.style.display = row.getAttribute('data-state') === state ? '' : 'none'; } }); document.querySelectorAll('.aq-filter-btn').forEach(function(btn) { btn.classList.toggle('active', btn.getAttribute('data-filter') === state); }); }; // ── Auto-refresh ──────────────────────────────────────────── var refreshInterval = 10000; function scheduleRefresh() { setTimeout(function() { fetch('{{ url_prefix }}/tasks/api/') .then(function(r) { return r.ok ? r.json() : null; }) .then(function(data) { if (data) { var cards = document.querySelectorAll('.dash-stat-value[data-count]'); cards.forEach(function(el) { var target = parseInt(el.getAttribute('data-count'), 10); if (!isNaN(target)) el.textContent = target; }); } }) .catch(function() {}) .finally(scheduleRefresh); }, refreshInterval); } scheduleRefresh(); })(); // ═══════════════════════════════════════════════════════════════════ // SLIDE-OUT DETAIL DRAWER // ═══════════════════════════════════════════════════════════════════ (function() { 'use strict'; var overlay = document.getElementById('aq-drawer-overlay'); var drawer = document.getElementById('aq-drawer'); var title = document.getElementById('drawer-title'); var subtitle = document.getElementById('drawer-subtitle'); var icon = document.getElementById('drawer-icon'); var body = document.getElementById('drawer-body'); function esc(s) { var d = document.createElement('div'); d.textContent = s; return d.innerHTML; } function formatTs(iso) { if (!iso) return '—'; try { var d = new Date(iso); return d.toLocaleString('en-GB', { day:'2-digit', month:'short', year:'numeric', hour:'2-digit', minute:'2-digit', second:'2-digit', hour12:false }); } catch(e) { return iso.substring(0, 19); } } function relativeTime(iso) { if (!iso) return ''; try { var ms = Date.now() - new Date(iso).getTime(); if (ms < 60000) return Math.round(ms / 1000) + 's ago'; if (ms < 3600000) return Math.round(ms / 60000) + 'm ago'; if (ms < 86400000) return Math.round(ms / 3600000) + 'h ago'; return Math.round(ms / 86400000) + 'd ago'; } catch(e) { return ''; } } function priorityColor(p) { if (p === 'CRITICAL') return 'var(--danger)'; if (p === 'HIGH') return 'var(--warning)'; return 'var(--text-secondary)'; } function priorityBg(p) { if (p === 'CRITICAL') return 'rgba(239,68,68,0.12)'; if (p === 'HIGH') return 'rgba(245,158,11,0.12)'; return 'var(--bg-surface)'; } var stateColorMap = { 'completed': 'var(--success)', 'running': 'var(--accent)', 'pending': 'var(--text-secondary)', 'failed': 'var(--danger)', 'dead': 'var(--danger)', 'retrying': 'var(--warning)', 'cancelled': 'var(--text-faint)', 'scheduled': '#8b5cf6' }; function stateColor(s) { return stateColorMap[s] || 'var(--text-secondary)'; } function makeField(label, value, opts) { opts = opts || {}; var cls = opts.mono ? ' mono' : ''; var full = opts.full ? ' style="grid-column:1/-1;"' : ''; return '
' + '
' + esc(label) + '
' + '
' + (value || '—') + '
'; } function makeBadge(text, color, bg) { return '' + esc(text) + ''; } function makeTags(tagStr) { if (!tagStr) return 'No tags'; return tagStr.split(', ').map(function(t) { return '' + esc(t.trim()) + ''; }).join(''); } function openDrawer() { overlay.classList.add('open'); drawer.classList.add('open'); document.body.style.overflow = 'hidden'; } window.closeDrawer = function() { overlay.classList.remove('open'); drawer.classList.remove('open'); document.body.style.overflow = ''; }; // Close on Escape key document.addEventListener('keydown', function(e) { if (e.key === 'Escape' && drawer.classList.contains('open')) { window.closeDrawer(); } }); // ── Open Task Drawer (Task Definitions) ────────────────────── window.openTaskDrawer = function(row) { var d = row.dataset; title.textContent = d.taskName; subtitle.textContent = 'Registered Task Definition'; icon.style.background = 'rgba(34,197,94,0.12)'; icon.style.color = 'var(--accent)'; icon.innerHTML = ''; var html = ''; // ── Configuration section ── html += '
'; html += '
Configuration
'; html += '
'; html += makeField('Task Name', '' + esc(d.taskName) + '', {full: true}); html += makeField('Queue', esc(d.taskQueue)); html += makeField('Priority', makeBadge(d.taskPriority, priorityColor(d.taskPriority), priorityBg(d.taskPriority))); html += makeField('Timeout', esc(d.taskTimeout) + 's'); html += makeField('Max Retries', esc(d.taskMaxRetries)); html += '
'; // ── Retry Policy section ── html += '
'; html += '
Retry Policy
'; html += '
'; html += makeField('Base Delay', esc(d.taskRetryDelay) + 's'); html += makeField('Backoff', '×' + esc(d.taskRetryBackoff)); html += makeField('Max Retries', esc(d.taskMaxRetries) + ' attempts'); html += makeField('Strategy', 'Exponential + Jitter'); html += '
'; // Retry formula html += '
'; html += '
Delay Formula
'; html += 'delay = ' + esc(d.taskRetryDelay) + ' × ' + esc(d.taskRetryBackoff) + 'attempt ± 25% jitter'; html += '
'; // ── Scheduling section ── html += '
'; html += '
Scheduling
'; html += '
'; html += makeField('Dispatch', makeBadge(d.taskDispatch, d.taskDispatch === 'periodic' ? '#8b5cf6' : 'var(--text-secondary)', d.taskDispatch === 'periodic' ? 'rgba(139,92,246,0.12)' : 'var(--bg-surface)')); html += makeField('Schedule', esc(d.taskSchedule)); html += '
'; // ── Tags section ── html += '
'; html += '
Tags
'; html += '
' + makeTags(d.taskTags) + '
'; html += '
'; // ── Usage example ── html += '
'; html += '
Quick Dispatch
'; html += '
'; var shortName = d.taskName.split(':').pop() || d.taskName; html += '# On-demand dispatch\n'; html += 'await ' + esc(shortName) + '.delay(*args)\n\n'; html += '# Or via manager\n'; html += 'await manager.enqueue(' + esc(shortName) + ', *args)'; html += '
'; body.innerHTML = html; openDrawer(); }; // ── Open Job Drawer (Recent Jobs) ─────────────────────────── window.openJobDrawer = function(row) { var d = row.dataset; title.textContent = d.jobName || d.jobFuncRef; subtitle.textContent = d.jobId; var st = d.jobState; var isFailed = (st === 'failed' || st === 'dead'); icon.style.background = isFailed ? 'rgba(239,68,68,0.12)' : (st === 'completed' ? 'rgba(34,197,94,0.12)' : (st === 'running' ? 'rgba(59,130,246,0.12)' : 'var(--bg-surface)')); icon.style.color = stateColor(st); icon.innerHTML = isFailed ? '' : (st === 'completed' ? '' : (st === 'running' ? '' : '')); var html = ''; // ── Identity section ── html += '
'; html += '
Identity
'; html += '
'; html += makeField('Job ID', esc(d.jobId), {full: true, mono: true}); html += makeField('Task Name', '' + esc(d.jobName || d.jobFuncRef) + '', {full: true}); html += makeField('Function Ref', esc(d.jobFuncRef), {full: true, mono: true}); html += makeField('Fingerprint', esc(d.jobFingerprint), {mono: true}); html += makeField('State', makeBadge(st, stateColor(st), stateColor(st).replace(')', ',0.12)').replace('var(', 'rgba(').replace('--success', '34,197,94').replace('--accent', '34,197,94').replace('--danger', '239,68,68').replace('--warning', '245,158,11').replace('--text-secondary', '161,161,170').replace('--text-faint', '107,114,128'))); html += '
'; // ── Configuration section ── html += '
'; html += '
Configuration
'; html += '
'; html += makeField('Queue', esc(d.jobQueue)); html += makeField('Priority', makeBadge(d.jobPriority, priorityColor(d.jobPriority), priorityBg(d.jobPriority))); html += makeField('Timeout', esc(d.jobTimeout) + 's'); html += makeField('Retries', esc(d.jobRetryCount) + ' / ' + esc(d.jobMaxRetries)); html += makeField('Terminal', d.jobTerminal === 'True' ? '✓ Yes' : '✗ No'); html += makeField('Can Retry', d.jobCanRetry === 'True' ? '✓ Yes' : '✗ No'); html += '
'; // ── Timeline section ── html += '
'; html += '
Timeline
'; html += '
'; // Created html += '
'; html += '
'; html += 'Created'; html += '' + relativeTime(d.jobCreated) + ''; html += '
'; html += '
' + formatTs(d.jobCreated) + '
'; html += '
'; // Scheduled (if any) if (d.jobScheduled) { html += '
'; html += '
'; html += 'Scheduled'; html += '' + relativeTime(d.jobScheduled) + ''; html += '
'; html += '
' + formatTs(d.jobScheduled) + '
'; html += '
'; } // Started html += '
'; html += '
'; html += 'Started'; if (d.jobStarted) { html += '' + relativeTime(d.jobStarted) + ''; } html += '
'; html += '
' + formatTs(d.jobStarted) + '
'; html += '
'; // Completed / Failed html += '
'; html += '
'; html += '' + (isFailed ? 'Failed' : 'Completed') + ''; if (d.jobCompleted) { html += '' + relativeTime(d.jobCompleted) + ''; } html += '
'; html += '
' + formatTs(d.jobCompleted) + '
'; html += '
'; html += '
'; // Duration bar if (d.jobDuration) { var dur = parseFloat(d.jobDuration); var durColor = dur < 100 ? '#22c55e' : (dur < 1000 ? '#f59e0b' : '#ef4444'); html += '
'; html += '
Duration
'; html += '
'; html += '
'; html += '' + dur.toFixed(1) + 'ms'; html += '
'; } html += '
'; // ── Tags section ── html += '
'; html += '
Tags
'; html += '
' + makeTags(d.jobTags) + '
'; html += '
'; // ── Error section (if failed) ── if (isFailed && d.jobErrorType) { html += '
'; html += '
Error Details
'; html += '
'; html += '
Exception
'; html += '
' + esc(d.jobErrorType) + '
'; html += '
' + esc(d.jobError) + '
'; html += '
'; if (d.jobTraceback) { html += '
'; html += 'Show traceback'; html += '
' + esc(d.jobTraceback) + '
'; html += '
'; } html += '
'; } body.innerHTML = html; openDrawer(); }; })(); {% endblock %}