diff --git a/deploy/prometheus-snapshot.service b/deploy/prometheus-snapshot.service new file mode 100644 index 0000000..00c2968 --- /dev/null +++ b/deploy/prometheus-snapshot.service @@ -0,0 +1,20 @@ +[Unit] +Description=Prometheus — render + upload the static snapshot to bns/prometheus +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=prometheus +WorkingDirectory=/opt/prometheus +Environment=NODE_ENV=production +Environment=AWS_ENDPOINT_URL=https://p2pexchange.silentmode.st:9980 +ExecStart=/usr/bin/node /opt/prometheus/scripts/snapshot.mjs --upload --bucket bns/prometheus +Nice=10 + +# These harden the unit without blocking anything it actually does. +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/opt/prometheus/build /opt/prometheus/db.sqlite diff --git a/deploy/prometheus-snapshot.timer b/deploy/prometheus-snapshot.timer new file mode 100644 index 0000000..1b60eb9 --- /dev/null +++ b/deploy/prometheus-snapshot.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Prometheus snapshot — weekly upload to the Sia fallback + +[Timer] +# Weekly on Sunday 04:00 UTC, with a jitter so a cluster of timers does not +# all fire the same second if we ever run more than one. Persistent=yes catches +# a missed run after a reboot. +OnCalendar=Sun 04:00 UTC +RandomizedDelaySec=30m +Persistent=yes +Unit=prometheus-snapshot.service + +[Install] +WantedBy=timers.target diff --git a/scripts/snapshot.mjs b/scripts/snapshot.mjs new file mode 100644 index 0000000..6a39baf --- /dev/null +++ b/scripts/snapshot.mjs @@ -0,0 +1,163 @@ +#!/usr/bin/env node +/** + * Render Prometheus to static files and upload to Sia as the DR fallback. + * + * The architecture decision (see + * ../../../_coordination/sessions/prometheus-x-architecture.md §4): + * prometheus.x's `ip`+`tls` records point at the live service, but a weekly + * snapshot of the browseable surface is also published to `bns/prometheus/` + * on Sia. Resolver clients that cannot reach the service can fall back to + * the `s3` record and still read what's there — stale by up to a snapshot + * interval, but readable. + * + * Scope — the browse surface, not the dynamic surface: + * /, /browse, /publish, /boundaries, /c/ × 8, /app// × N, + * plus /static/style.css + /static/favicon.svg + /sitemap.xml + /robots.txt. + * NOT snapshotted: /submit (requires the live API), /admin/*, /api/*. + * + * Flags: + * --out write files here (default: ./build) + * --upload after writing, `aws s3 sync` to bns/prometheus/ + * --bucket target bucket (default: bns/prometheus) + * --endpoint S3 endpoint URL for the aws CLI + * --dry-run print the sync command instead of running it + * + * The sync uses the `sia-storage` AWS profile per + * [[sia-storage-credentials-location]]. + */ + +import { mkdir, writeFile, rm, copyFile } from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; +import { resolve, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawn } from 'node:child_process'; +import { layout } from '../service/layout.js'; +import { home, publish, boundaries } from '../service/pages.js'; +import { browse as browsePage } from '../service/pages.js'; +import { CATEGORIES, categoryPage, appPage } from '../service/pages-browse.js'; +import { allActiveListings } from '../service/db.js'; +import { renderSitemap, renderRobots } from '../service/seo.js'; + +const SELF_DIR = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(SELF_DIR, '..'); +const PUBLIC_DIR = resolve(ROOT, 'public'); + +const args = parseArgs(process.argv.slice(2)); +const OUT = resolve(ROOT, args.out || 'build'); +const BUCKET = args.bucket || 'bns/prometheus'; +const ENDPOINT = args.endpoint || process.env.AWS_ENDPOINT_URL_S3 || process.env.AWS_ENDPOINT_URL; + +const log = (...a) => console.log('[snapshot]', ...a); + +async function main() { + log(`output: ${OUT}`); + await rm(OUT, { recursive: true, force: true }); + await mkdir(OUT, { recursive: true }); + + const listings = allActiveListings(); + log(`rendering ${CATEGORIES.length} category pages + ${listings.length} listings …`); + + // Core pages — the "here" path doubles as the output URL. + await writePage('/', home()); + await writePage('/browse', browsePage()); + await writePage('/publish', publish()); + await writePage('/boundaries', boundaries()); + + // Category pages — fake a request object for the signature. + for (const c of CATEGORIES) { + await writePage(`/c/${c.slug}`, categoryPage({ url: `/c/${c.slug}` }, c.slug)); + } + + // App detail pages — one per active listing. + for (const r of listings) { + await writePage(`/app/${r.publisher}/${r.slug}`, + appPage({ url: `/app/${r.publisher}/${r.slug}` }, r.publisher, r.slug)); + } + + // Static assets live next to the rendered HTML at the same /static/ URL. + const staticOut = resolve(OUT, 'static'); + await mkdir(staticOut, { recursive: true }); + await copyFile(resolve(PUBLIC_DIR, 'style.css'), resolve(staticOut, 'style.css')); + await copyFile(resolve(PUBLIC_DIR, 'favicon.svg'), resolve(staticOut, 'favicon.svg')); + + // SEO files at the root of the bucket. + await writeFile(resolve(OUT, 'sitemap.xml'), renderSitemap({ + categories: CATEGORIES.map((c) => c.slug), + listings: listings.map((r) => ({ publisher: r.publisher, slug: r.slug, lastmod: r.created_at })), + }), 'utf8'); + await writeFile(resolve(OUT, 'robots.txt'), renderRobots(), 'utf8'); + + log(`wrote ${listings.length + CATEGORIES.length + 6} HTML files + static assets`); + + if (args.upload) { + await uploadToSia(); + } else { + log(`skipping upload — rerun with --upload (or --dry-run to just print the aws command)`); + } + log('done.'); +} + +/** + * Resolve the page's URL to the filesystem path the bucket serves at. + * + * / → index.html + * /browse → browse/index.html + * /c/privacy → c/privacy/index.html + * /app/foo.x/bar → app/foo.x/bar/index.html + * + * Directory-with-index-html lets `/browse` and `/browse/` both work on an + * S3-style static host — the bucket's "serve index when the path is a prefix" + * rule picks index.html for us. + */ +async function writePage(urlPath, page) { + const html = layout(page); + const rel = urlPath === '/' ? 'index.html' : urlPath.replace(/^\/+/, '').replace(/\/+$/, '') + '/index.html'; + const full = resolve(OUT, rel); + await mkdir(dirname(full), { recursive: true }); + await writeFile(full, html, 'utf8'); +} + +async function uploadToSia() { + const cmd = 'aws'; + const argv = [ + '--profile', 'sia-storage', + ...(ENDPOINT ? ['--endpoint-url', ENDPOINT] : []), + 's3', 'sync', OUT + '/', `s3://${BUCKET}/`, + '--delete', // objects dropped from the snapshot vanish from the bucket too + ]; + if (args['dry-run']) { + log(`DRY RUN: ${cmd} ${argv.join(' ')}`); + return; + } + log(`$ ${cmd} ${argv.join(' ')}`); + await spawnOk(cmd, argv); +} + +function spawnOk(cmd, argv) { + return new Promise((resolveP, rejectP) => { + const c = spawn(cmd, argv, { stdio: 'inherit' }); + c.on('error', rejectP); + c.on('close', (code) => code === 0 ? resolveP() : rejectP(new Error(`${cmd} exited ${code}`))); + }); +} + +function parseArgs(argv) { + const out = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (!a.startsWith('--')) continue; + const k = a.slice(2); + const next = argv[i + 1]; + if (next && !next.startsWith('--')) { + out[k] = next; i++; + } else { + out[k] = true; + } + } + return out; +} + +main().catch((err) => { + console.error('[snapshot] FAIL:', err?.stack || err); + process.exit(1); +}); diff --git a/service/seo.js b/service/seo.js new file mode 100644 index 0000000..fb584b1 --- /dev/null +++ b/service/seo.js @@ -0,0 +1,99 @@ +/** + * SEO artefacts — sitemap and robots. + * + * sitemap.xml enumerates every crawlable URL. The snapshot writes it to the + * bucket root; the live service serves the same file on request so a crawler + * hitting prometheus.x (via the apps.silentmode.st mirror in production) sees + * the same contents the snapshot does. + * + * robots.txt is the usual shape: allow everything except the admin and the + * raw API routes, which have no public content. + */ + +const BASE = process.env.PUBLIC_BASE || 'https://prometheus.x'; + +/** + * @param {{categories: string[], listings: Array<{publisher: string, slug: string, lastmod?: string}>}} data + */ +export function renderSitemap({ categories, listings }) { + const now = new Date().toISOString(); + const urls = [ + { loc: '/', lastmod: now, priority: '1.0' }, + { loc: '/browse', lastmod: now, priority: '0.9' }, + { loc: '/publish', lastmod: now, priority: '0.6' }, + { loc: '/boundaries', lastmod: now, priority: '0.5' }, + { loc: '/submit', lastmod: now, priority: '0.4' }, + ...categories.map((slug) => ({ + loc: `/c/${slug}`, lastmod: now, priority: '0.7', + })), + ...listings.map((l) => ({ + loc: `/app/${l.publisher}/${l.slug}`, + lastmod: l.lastmod || now, + priority: '0.8', + })), + ]; + + const body = urls.map((u) => + ` ${BASE}${u.loc}${u.lastmod}${u.priority}` + ).join('\n'); + + return ` + +${body} + +`; +} + +export function renderRobots() { + return `User-agent: * +Allow: / +Disallow: /admin +Disallow: /admin/ +Disallow: /api/ +Disallow: /.well-known/ + +Sitemap: ${BASE}/sitemap.xml +`; +} + +/** + * schema.org JSON-LD for a listing — Google renders a rich card for apps. + * Returns the raw JSON string; the template wraps it in a