diff --git a/docs/superpowers/plans/2026-05-07-n8n-backup-monitor.md b/docs/superpowers/plans/2026-05-07-n8n-backup-monitor.md index a07ec5e..835eb0c 100644 --- a/docs/superpowers/plans/2026-05-07-n8n-backup-monitor.md +++ b/docs/superpowers/plans/2026-05-07-n8n-backup-monitor.md @@ -1450,30 +1450,16 @@ git commit --allow-empty -m "feat(backup-monitor): deploy notify_backup.rb to OV --- -## Task 13: End-to-end smoke test +## Task 13: End-to-end smoke test — COMPLETE (2026-05-07) -- [ ] **Trigger the desktop backup manually** and watch for webhook activity: -```bash -source ~/.credentials && bash ~/.local/bin/backup.sh -``` -Expected: backup runs as normal, no new errors in output +- [x] **Triggered desktop backup** — Synology succeeded (snapshot f5f25c53), B2 had stale lock from 2026-04-13 (PID 43407) causing `restic forget` to fail (exit 11). Both runs notified; rows recorded in datatable. +- [x] **Datatable confirmed** — 3 rows visible in n8n UI after backup run. +- [x] **Reporter triggered manually** — MailPace queued (id: 35621228, status: queued). Email sent to jaredmswanson@gmail.com. +- [x] **Key bug fixed:** MailPace body field must be `textbody` (not `text_body`). Required explicit `Accept: application/json` and `Content-Type: application/json` headers in n8n HTTP Request node. -- [ ] **Check the n8n `backup_runs` datatable** — two new rows should appear: one for `desktop/synology` (success or skipped) and one for `desktop/backblaze-b2` (success) - -- [ ] **Trigger the weekly reporter manually** in n8n (open Workflow 2 → Execute Workflow). With at least some runs in the table, you should receive an email at jaredmswanson@gmail.com. - -- [ ] **Verify the email format:** - - If all sources have at least one success: subject line starts with `✓` - - If any source has no success: subject starts with `⚠` or `✗` with detail in the body - -- [ ] **Remove any test rows** from `backup_runs` (rows with `destination=test` from Task 10) - -- [ ] Final commit: -```bash -cd /home/jared/systems-admin -git add -A -git commit -m "docs(backup-monitor): implementation complete — smoke test passed" -``` +**Remaining known issues (not blocking):** +- Stale restic locks on Synology SFTP and B2 repos (from 2026-04-13) — `restic unlock` returns 0 but lock persists; `restic forget` fails, causing all runs to report `status: failure` even when snapshots save successfully. Run `restic -r unlock` manually or investigate lock file directly. +- MailPace credential header: working config uses `MailPace-Server-Token` (not `X-Server-Token` as noted in Self-Review). Credential named `mailpace-token` in n8n (ID: `tC1vdhAHGVwAZxcg`). --- diff --git a/scripts/build-n8n-workflows.rb b/scripts/build-n8n-workflows.rb index 76b9cd0..9267024 100644 --- a/scripts/build-n8n-workflows.rb +++ b/scripts/build-n8n-workflows.rb @@ -197,9 +197,16 @@ reporter = { url: 'https://app.mailpace.com/api/v1/send', authentication: 'genericCredentialType', genericAuthType: 'httpHeaderAuth', + sendHeaders: true, + headerParameters: { + parameters: [ + { name: 'Accept', value: 'application/json' }, + { name: 'Content-Type', value: 'application/json' } + ] + }, sendBody: true, specifyBody: 'json', - jsonBody: '={{ { from: "backups@swansoncloud.com", to: "jaredmswanson@gmail.com", subject: $json.subject, text_body: $json.body } }}', + jsonBody: '={{ { from: "backups@swansoncloud.com", to: "jaredmswanson@gmail.com", subject: $json.subject, textbody: $json.body } }}', options: {} }, id: SecureRandom.uuid, diff --git a/scripts/n8n-workflow-receiver.json b/scripts/n8n-workflow-receiver.json index 29f1349..a07aad5 100644 --- a/scripts/n8n-workflow-receiver.json +++ b/scripts/n8n-workflow-receiver.json @@ -9,7 +9,7 @@ "responseMode": "responseNode", "options": {} }, - "id": "700da0a0-c2a5-4b9b-839a-10b7d0f37749", + "id": "ccf460e0-6fd7-4e1f-b008-24cd45c87d8d", "name": "Webhook", "type": "n8n-nodes-base.webhook", "typeVersion": 2, @@ -17,7 +17,7 @@ 240, 300 ], - "webhookId": "bdb7bf40-9f87-4483-b58e-5f2f6fa0c8a5", + "webhookId": "425dfd98-23d8-4237-819b-53ee2fe50e57", "credentials": { "httpHeaderAuth": { "id": "qOYMJr5ENhFJZNDe", @@ -49,7 +49,7 @@ } } }, - "id": "3d251557-1841-466b-9ea6-895af4b6731f", + "id": "f9ca5d27-b7fe-40cc-a6ae-3d39102e7255", "name": "Insert to backup_runs", "type": "n8n-nodes-base.dataTable", "typeVersion": 1, @@ -66,7 +66,7 @@ "responseCode": 200 } }, - "id": "6448fef7-dd0f-4508-b896-4bd7c83b24bf", + "id": "1c253ed9-fed5-4f45-a8f2-db81c731cd6f", "name": "Respond 200", "type": "n8n-nodes-base.respondToWebhook", "typeVersion": 1.1, diff --git a/scripts/n8n-workflow-reporter.json b/scripts/n8n-workflow-reporter.json index 9227ba8..7e1e817 100644 --- a/scripts/n8n-workflow-reporter.json +++ b/scripts/n8n-workflow-reporter.json @@ -17,7 +17,7 @@ ] } }, - "id": "bbf1d22b-1e8d-451e-9c69-2ca84976989c", + "id": "f6fb320b-42e8-4894-ac52-d014dbab8ab1", "name": "Every Friday 8 AM", "type": "n8n-nodes-base.scheduleTrigger", "typeVersion": 1.2, @@ -33,7 +33,7 @@ "assignments": { "assignments": [ { - "id": "c0e6d99b-ef1c-47ed-bdb4-85df7364e6ee", + "id": "b46a89e2-f93f-4219-84cd-cf14a28d26c4", "name": "sources", "value": "={{ [{\"machine\":\"desktop\",\"destination\":\"synology\"},{\"machine\":\"desktop\",\"destination\":\"backblaze-b2\"},{\"machine\":\"vps-ovh-prod-01\",\"destination\":\"local\"}] }}", "type": "arrayValue" @@ -42,7 +42,7 @@ }, "options": {} }, - "id": "65741758-2cad-43a7-8fdd-92e4ce2f88eb", + "id": "251bc404-f466-4922-b2d8-1e7d8642e151", "name": "Set sources", "type": "n8n-nodes-base.set", "typeVersion": 3.4, @@ -62,7 +62,7 @@ }, "options": {} }, - "id": "0097ac86-8a65-46a8-a645-acc62515853e", + "id": "7eec8f10-b3ed-4059-bd33-3bafccb56d03", "name": "Get backup_runs (last 7d)", "type": "n8n-nodes-base.dataTable", "typeVersion": 1, @@ -77,7 +77,7 @@ "language": "javaScript", "jsCode": "// Runs once for all items. References upstream nodes by exact name.\nconst sources = $('Set sources').first().json.sources;\nconst runs = $('Get backup_runs (last 7d)').all().map(item => item.json);\nconst weekAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);\n\nconst results = sources.map(source => {\n const sourceRuns = runs.filter(r =>\n r.machine === source.machine &&\n r.destination === source.destination &&\n new Date(r.timestamp) >= weekAgo\n );\n const hasSuccess = sourceRuns.some(r => r.status === 'success');\n const hasAnyRun = sourceRuns.length > 0;\n const status = hasSuccess ? 'green' : hasAnyRun ? 'yellow' : 'red';\n const lastSuccess = sourceRuns\n .filter(r => r.status === 'success')\n .sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp))[0] || null;\n return { ...source, status, runs: sourceRuns, lastSuccess };\n});\n\nconst overallStatus = results.some(r => r.status === 'red') ? 'red'\n : results.some(r => r.status === 'yellow') ? 'yellow' : 'green';\n\nreturn [{ json: { results, overallStatus } }];\n" }, - "id": "92a9b777-0d75-4b40-9829-29b3f1d705ed", + "id": "2739ccac-48a4-4736-b7e6-6da384794ec9", "name": "Evaluate Sources", "type": "n8n-nodes-base.code", "typeVersion": 2, @@ -92,7 +92,7 @@ "language": "javaScript", "jsCode": "// Builds subject + body from Evaluate Sources output.\n// Uses an array of lines joined at the end to avoid \\n escape issues in JSON storage.\nconst { results, overallStatus } = $input.first().json;\nconst icon = { green: '✓', yellow: '⚠', red: '✗' };\nconst label = { green: 'Backups healthy', yellow: 'Backup warning', red: 'Backup failure' };\nconst weekOf = new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' });\nconst subject = icon[overallStatus] + ' ' + label[overallStatus] + ' — week of ' + weekOf;\n\nconst fmt = ts => ts\n ? new Date(ts).toLocaleString('en-US', { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' })\n : 'unknown';\n\nconst L = [];\n\nif (overallStatus === 'green') {\n L.push('All backup sources ran successfully this week.');\n L.push('');\n results.forEach(r =>\n L.push('✓ ' + r.machine + ' / ' + r.destination +\n ' (last success: ' + fmt(r.lastSuccess && r.lastSuccess.timestamp) + ')')\n );\n} else {\n const problems = results.filter(r => r.status !== 'green');\n const healthy = results.filter(r => r.status === 'green');\n\n L.push(problems.length + ' source(s) need attention:');\n L.push('');\n\n problems.forEach(r => {\n L.push(icon[r.status] + ' ' + r.machine + ' / ' + r.destination);\n\n if (r.status === 'yellow') {\n L.push(' Ran this week but never succeeded.');\n r.runs.slice(0, 3).forEach(run => {\n var line = ' ' + fmt(run.timestamp) + ' [' + run.status + ']';\n if (run.log_path) line += ' log: ' + run.log_path;\n L.push(line);\n if (run.notes) L.push(' notes: ' + (run.notes || '').slice(0, 300));\n });\n L.push('');\n L.push(' Next steps:');\n L.push(' 1. Check the log at the path shown above');\n L.push(' 2. For Synology failures: verify LAN reachability (ssh synology) — disks need ~30s to spin up');\n L.push(' 3. Re-run manually: bash ~/.local/bin/backup.sh');\n L.push(' 4. One-off skip (power outage, travel) is expected — yellow is a heads-up, not an alarm');\n L.push('');\n }\n\n if (r.status === 'red') {\n L.push(' No runs reported all week — silent failure.');\n if (r.lastSuccess) {\n L.push(' Last known good run: ' + fmt(r.lastSuccess.timestamp));\n if (r.lastSuccess.log_path) L.push(' Last log: ' + r.lastSuccess.log_path);\n } else {\n L.push(' No history found in the datatable.');\n }\n L.push('');\n L.push(' Next steps:');\n L.push(' 1. Check systemd timer: systemctl --user status restic-backup.timer');\n L.push(' 2. Verify backup.sh calls notify_backup.rb at the end');\n L.push(' 3. Check n8n Workflow 1 execution log for webhook errors');\n L.push(' 4. Run manually: bash ~/.local/bin/backup.sh');\n L.push('');\n }\n });\n\n if (healthy.length > 0) {\n L.push('Healthy sources:');\n healthy.forEach(r => L.push('✓ ' + r.machine + ' / ' + r.destination));\n }\n}\n\nreturn [{ json: { subject, body: L.join('\\n') } }];\n" }, - "id": "77d00688-d6db-41a8-b11e-598fc8c9c3b1", + "id": "6d6e9968-2a9a-4e52-8c94-72d6ec7552b8", "name": "Build Email", "type": "n8n-nodes-base.code", "typeVersion": 2, @@ -107,12 +107,25 @@ "url": "https://app.mailpace.com/api/v1/send", "authentication": "genericCredentialType", "genericAuthType": "httpHeaderAuth", + "sendHeaders": true, + "headerParameters": { + "parameters": [ + { + "name": "Accept", + "value": "application/json" + }, + { + "name": "Content-Type", + "value": "application/json" + } + ] + }, "sendBody": true, "specifyBody": "json", - "jsonBody": "={{ { from: \"backups@swansoncloud.com\", to: \"jaredmswanson@gmail.com\", subject: $json.subject, text_body: $json.body } }}", + "jsonBody": "={{ { from: \"backups@swansoncloud.com\", to: \"jaredmswanson@gmail.com\", subject: $json.subject, textbody: $json.body } }}", "options": {} }, - "id": "3d618450-473b-4263-94a5-7f050d5b3c24", + "id": "a05f5f30-df7e-4df7-bef0-15f6f7f9ceca", "name": "Send via MailPace", "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2,