prometheus: weekly s3 snapshot of the browse surface to bns/prometheus

scripts/snapshot.mjs renders every public page (home, browse, publish,
boundaries, 8 categories, every active listing) to <out>/<path>/index.html,
plus /sitemap.xml, /robots.txt, and the two static assets. Writing to a
directory next to the service AND uploading via `aws s3 sync` are two
flags on the same command: --out to pick the directory, --upload to push.

Live routes added for /sitemap.xml and /robots.txt so a crawler hitting
the live service sees the same enumeration the snapshot bakes into the
bucket. renderSitemap / renderRobots in service/seo.js are the single
rendering path both call sites use.

Scope deliberately limited to the browse surface — /submit and the API
routes depend on the live backend and were never going to work on a
static mirror. The browse surface IS what the fallback is for: readers
survive an outage, publishers can't.

systemd unit + timer under deploy/ run the snapshot weekly on Sunday
04:00 UTC with 30 min jitter and Persistent=yes so a missed run after a
reboot catches up. Unit uses the sia-storage AWS profile per
[[sia-storage-credentials-location]] and the per-process endpoint URL
baked in via Environment=.

Local dry-run against an empty DB produced 14 HTML files + sitemap +
robots + the two assets.

deploy/prometheus-snapshot.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=Prometheus — render + upload the static snapshot to bns/prometheus
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=prometheus
+WorkingDirectory=/opt/prometheus
+Environment=NODE_ENV=production
+Environment=AWS_ENDPOINT_URL=https://p2pexchange.silentmode.st:9980
+ExecStart=/usr/bin/node /opt/prometheus/scripts/snapshot.mjs --upload --bucket bns/prometheus
+Nice=10
+
+# These harden the unit without blocking anything it actually does.
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=yes
+ReadWritePaths=/opt/prometheus/build /opt/prometheus/db.sqlite

deploy/prometheus-snapshot.timer

@@ -0,0 +1,14 @@
+[Unit]
+Description=Prometheus snapshot — weekly upload to the Sia fallback
+
+[Timer]
+# Weekly on Sunday 04:00 UTC, with a jitter so a cluster of timers does not
+# all fire the same second if we ever run more than one. Persistent=yes catches
+# a missed run after a reboot.
+OnCalendar=Sun 04:00 UTC
+RandomizedDelaySec=30m
+Persistent=yes
+Unit=prometheus-snapshot.service
+
+[Install]
+WantedBy=timers.target

scripts/snapshot.mjs

@@ -0,0 +1,163 @@
+#!/usr/bin/env node
+/**
+ * Render Prometheus to static files and upload to Sia as the DR fallback.
+ *
+ * The architecture decision (see
+ * ../../../_coordination/sessions/prometheus-x-architecture.md §4):
+ * prometheus.x's `ip`+`tls` records point at the live service, but a weekly
+ * snapshot of the browseable surface is also published to `bns/prometheus/`
+ * on Sia. Resolver clients that cannot reach the service can fall back to
+ * the `s3` record and still read what's there — stale by up to a snapshot
+ * interval, but readable.
+ *
+ * Scope — the browse surface, not the dynamic surface:
+ *   /, /browse, /publish, /boundaries, /c/<slug> × 8, /app/<pub>/<slug> × N,
+ *   plus /static/style.css + /static/favicon.svg + /sitemap.xml + /robots.txt.
+ * NOT snapshotted: /submit (requires the live API), /admin/*, /api/*.
+ *
+ * Flags:
+ *   --out <dir>         write files here (default: ./build)
+ *   --upload            after writing, `aws s3 sync` to bns/prometheus/
+ *   --bucket <name>     target bucket (default: bns/prometheus)
+ *   --endpoint <url>    S3 endpoint URL for the aws CLI
+ *   --dry-run           print the sync command instead of running it
+ *
+ * The sync uses the `sia-storage` AWS profile per
+ * [[sia-storage-credentials-location]].
+ */
+
+import { mkdir, writeFile, rm, copyFile } from 'node:fs/promises';
+import { createReadStream } from 'node:fs';
+import { resolve, dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { spawn } from 'node:child_process';
+import { layout } from '../service/layout.js';
+import { home, publish, boundaries } from '../service/pages.js';
+import { browse as browsePage } from '../service/pages.js';
+import { CATEGORIES, categoryPage, appPage } from '../service/pages-browse.js';
+import { allActiveListings } from '../service/db.js';
+import { renderSitemap, renderRobots } from '../service/seo.js';
+
+const SELF_DIR = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(SELF_DIR, '..');
+const PUBLIC_DIR = resolve(ROOT, 'public');
+
+const args = parseArgs(process.argv.slice(2));
+const OUT = resolve(ROOT, args.out || 'build');
+const BUCKET = args.bucket || 'bns/prometheus';
+const ENDPOINT = args.endpoint || process.env.AWS_ENDPOINT_URL_S3 || process.env.AWS_ENDPOINT_URL;
+
+const log = (...a) => console.log('[snapshot]', ...a);
+
+async function main() {
+  log(`output: ${OUT}`);
+  await rm(OUT, { recursive: true, force: true });
+  await mkdir(OUT, { recursive: true });
+
+  const listings = allActiveListings();
+  log(`rendering ${CATEGORIES.length} category pages + ${listings.length} listings …`);
+
+  // Core pages — the "here" path doubles as the output URL.
+  await writePage('/', home());
+  await writePage('/browse', browsePage());
+  await writePage('/publish', publish());
+  await writePage('/boundaries', boundaries());
+
+  // Category pages — fake a request object for the signature.
+  for (const c of CATEGORIES) {
+    await writePage(`/c/${c.slug}`, categoryPage({ url: `/c/${c.slug}` }, c.slug));
+  }
+
+  // App detail pages — one per active listing.
+  for (const r of listings) {
+    await writePage(`/app/${r.publisher}/${r.slug}`,
+      appPage({ url: `/app/${r.publisher}/${r.slug}` }, r.publisher, r.slug));
+  }
+
+  // Static assets live next to the rendered HTML at the same /static/ URL.
+  const staticOut = resolve(OUT, 'static');
+  await mkdir(staticOut, { recursive: true });
+  await copyFile(resolve(PUBLIC_DIR, 'style.css'), resolve(staticOut, 'style.css'));
+  await copyFile(resolve(PUBLIC_DIR, 'favicon.svg'), resolve(staticOut, 'favicon.svg'));
+
+  // SEO files at the root of the bucket.
+  await writeFile(resolve(OUT, 'sitemap.xml'), renderSitemap({
+    categories: CATEGORIES.map((c) => c.slug),
+    listings: listings.map((r) => ({ publisher: r.publisher, slug: r.slug, lastmod: r.created_at })),
+  }), 'utf8');
+  await writeFile(resolve(OUT, 'robots.txt'), renderRobots(), 'utf8');
+
+  log(`wrote ${listings.length + CATEGORIES.length + 6} HTML files + static assets`);
+
+  if (args.upload) {
+    await uploadToSia();
+  } else {
+    log(`skipping upload — rerun with --upload (or --dry-run to just print the aws command)`);
+  }
+  log('done.');
+}
+
+/**
+ * Resolve the page's URL to the filesystem path the bucket serves at.
+ *
+ *   /                     → index.html
+ *   /browse               → browse/index.html
+ *   /c/privacy            → c/privacy/index.html
+ *   /app/foo.x/bar        → app/foo.x/bar/index.html
+ *
+ * Directory-with-index-html lets `/browse` and `/browse/` both work on an
+ * S3-style static host — the bucket's "serve index when the path is a prefix"
+ * rule picks index.html for us.
+ */
+async function writePage(urlPath, page) {
+  const html = layout(page);
+  const rel = urlPath === '/' ? 'index.html' : urlPath.replace(/^\/+/, '').replace(/\/+$/, '') + '/index.html';
+  const full = resolve(OUT, rel);
+  await mkdir(dirname(full), { recursive: true });
+  await writeFile(full, html, 'utf8');
+}
+
+async function uploadToSia() {
+  const cmd = 'aws';
+  const argv = [
+    '--profile', 'sia-storage',
+    ...(ENDPOINT ? ['--endpoint-url', ENDPOINT] : []),
+    's3', 'sync', OUT + '/', `s3://${BUCKET}/`,
+    '--delete',   // objects dropped from the snapshot vanish from the bucket too
+  ];
+  if (args['dry-run']) {
+    log(`DRY RUN: ${cmd} ${argv.join(' ')}`);
+    return;
+  }
+  log(`$ ${cmd} ${argv.join(' ')}`);
+  await spawnOk(cmd, argv);
+}
+
+function spawnOk(cmd, argv) {
+  return new Promise((resolveP, rejectP) => {
+    const c = spawn(cmd, argv, { stdio: 'inherit' });
+    c.on('error', rejectP);
+    c.on('close', (code) => code === 0 ? resolveP() : rejectP(new Error(`${cmd} exited ${code}`)));
+  });
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith('--')) continue;
+    const k = a.slice(2);
+    const next = argv[i + 1];
+    if (next && !next.startsWith('--')) {
+      out[k] = next; i++;
+    } else {
+      out[k] = true;
+    }
+  }
+  return out;
+}
+
+main().catch((err) => {
+  console.error('[snapshot] FAIL:', err?.stack || err);
+  process.exit(1);
+});

service/seo.js

@@ -0,0 +1,99 @@
+/**
+ * SEO artefacts — sitemap and robots.
+ *
+ * sitemap.xml enumerates every crawlable URL. The snapshot writes it to the
+ * bucket root; the live service serves the same file on request so a crawler
+ * hitting prometheus.x (via the apps.silentmode.st mirror in production) sees
+ * the same contents the snapshot does.
+ *
+ * robots.txt is the usual shape: allow everything except the admin and the
+ * raw API routes, which have no public content.
+ */
+
+const BASE = process.env.PUBLIC_BASE || 'https://prometheus.x';
+
+/**
+ * @param {{categories: string[], listings: Array<{publisher: string, slug: string, lastmod?: string}>}} data
+ */
+export function renderSitemap({ categories, listings }) {
+  const now = new Date().toISOString();
+  const urls = [
+    { loc: '/',           lastmod: now, priority: '1.0' },
+    { loc: '/browse',     lastmod: now, priority: '0.9' },
+    { loc: '/publish',    lastmod: now, priority: '0.6' },
+    { loc: '/boundaries', lastmod: now, priority: '0.5' },
+    { loc: '/submit',     lastmod: now, priority: '0.4' },
+    ...categories.map((slug) => ({
+      loc: `/c/${slug}`, lastmod: now, priority: '0.7',
+    })),
+    ...listings.map((l) => ({
+      loc: `/app/${l.publisher}/${l.slug}`,
+      lastmod: l.lastmod || now,
+      priority: '0.8',
+    })),
+  ];
+
+  const body = urls.map((u) =>
+    `  <url><loc>${BASE}${u.loc}</loc><lastmod>${u.lastmod}</lastmod><priority>${u.priority}</priority></url>`
+  ).join('\n');
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${body}
+</urlset>
+`;
+}
+
+export function renderRobots() {
+  return `User-agent: *
+Allow: /
+Disallow: /admin
+Disallow: /admin/
+Disallow: /api/
+Disallow: /.well-known/
+
+Sitemap: ${BASE}/sitemap.xml
+`;
+}
+
+/**
+ * schema.org JSON-LD for a listing — Google renders a rich card for apps.
+ * Returns the raw JSON string; the template wraps it in a <script> tag.
+ */
+export function softwareApplicationLd({ listing, scanVerdict }) {
+  const payload = {
+    '@context': 'https://schema.org',
+    '@type': 'SoftwareApplication',
+    name: listing.name,
+    description: listing.description,
+    softwareVersion: listing.version,
+    applicationCategory: humanCategory(listing.category),
+    url: `${BASE}/app/${listing.publisher}/${listing.slug}`,
+    downloadUrl: listing.download_url,
+    fileSize: `${listing.download_size} B`,
+    offers: { '@type': 'Offer', price: '0', priceCurrency: 'USD' },
+    author: {
+      '@type': 'Organization',
+      name: listing.publisher,
+    },
+  };
+  if (listing.icon) payload.image = listing.icon;
+  if (scanVerdict === 'clean') {
+    payload.contentRating = 'ClamAV clean';
+  }
+  return JSON.stringify(payload);
+}
+
+function humanCategory(slug) {
+  const map = {
+    communication: 'CommunicationApplication',
+    privacy: 'SecurityApplication',
+    wallets: 'FinanceApplication',
+    games: 'GameApplication',
+    devtools: 'DeveloperApplication',
+    media: 'MultimediaApplication',
+    utilities: 'UtilitiesApplication',
+    other: 'ApplicationSuite',
+  };
+  return map[slug] || 'ApplicationSuite';
+}

service/server.js

@@ -26,8 +26,10 @@ import {
   handleListingIndex,
   handleListingGet,
 } from './listings.js';
-import { categoryPage, appPage } from './pages-browse.js';
+import { categoryPage, appPage, CATEGORIES } from './pages-browse.js';
 import { handleAdmin } from './admin.js';
+import { renderSitemap, renderRobots } from './seo.js';
+import { allActiveListings } from './db.js';
 
 const PORT = Number(process.env.PORT) || 3700;
 const BIND = process.env.BIND || '127.0.0.1';
@@ -90,6 +92,10 @@ const server = createServer(async (req, res) => {
       await serveHealth(res);
     } else if (url === '/.well-known/prometheus-service-key.json') {
       serveServiceKey(res);
+    } else if (url === '/sitemap.xml') {
+      serveSitemap(res);
+    } else if (url === '/robots.txt') {
+      serveRobots(res);
     } else if (url === '/admin' || url.startsWith('/admin/')) {
       await handleAdmin(req, res, url);
     } else if (url.startsWith('/c/')) {
@@ -197,6 +203,29 @@ async function serveHealth(res) {
  * verify scan reports. Lives at a well-known path so clients can cache it by
  * origin.
  */
+function serveSitemap(res) {
+  const listings = allActiveListings().map((r) => ({
+    publisher: r.publisher, slug: r.slug, lastmod: r.created_at,
+  }));
+  const xml = renderSitemap({ categories: CATEGORIES.map((c) => c.slug), listings });
+  res.writeHead(200, {
+    'content-type': 'application/xml; charset=utf-8',
+    'cache-control': 'public, max-age=600',
+    'content-length': Buffer.byteLength(xml),
+  });
+  res.end(xml);
+}
+
+function serveRobots(res) {
+  const body = renderRobots();
+  res.writeHead(200, {
+    'content-type': 'text/plain; charset=utf-8',
+    'cache-control': 'public, max-age=600',
+    'content-length': Buffer.byteLength(body),
+  });
+  res.end(body);
+}
+
 function serveServiceKey(res) {
   const body = JSON.stringify({
     service: 'prometheus',