prometheus: s3d sidecar + /dl/<sha256> for Sia-backed hosting

Publishers no longer need external hosting at all. The service runs its
own s3d (SiaFoundation gateway) as a Docker sidecar on the internal
compose network; scan-clean uploads land there via PUT; readers fetch
through /dl/<sha256> on prometheus.x which proxies back out from s3d.

- deploy/docker-compose.yml: s3d service added. Image is
  ghcr.io/siafoundation/s3d:latest, data folder bind-mounted at
  ./s3d-data/ so the operator can seed it from a Pithos connection
  export. Only prometheus-internal network — never public.
- deploy/setup-s3d.sh: one-time import of the Pithos connection
  tarball, writes a fresh s3d.yml with a random adminPassword, mints
  an S3 access key for the Prometheus service, appends
  PROMETHEUS_S3_* lines to .env. Idempotent on re-run.
- service/storage.js: endpoint now defaults to the internal s3d
  (http://s3d:8000); publicBase defaults to https://prometheus.x so
  download URLs flow through the service's own /dl handler. Bucket is
  auto-created on first use so a fresh data folder bootstraps.
  Added getObjectStream() for the read side.
- service/downloads.js: GET/HEAD /dl/<sha256> — forwards Range,
  content-type and content-length from s3d, drops s3d-specific
  headers (x-amz-*), sets immutable long-cache. Hash is validated
  against ^[0-9a-f]{64}$ before touching upstream.
- service/server.js: route table wires /dl/* to the handler above.

deploy/docker-compose.yml

@@ -22,6 +22,8 @@ services:
     depends_on:
       clamav:
         condition: service_healthy
+      s3d:
+        condition: service_started
     environment:
       BNS_NETWORK: chipnet
       CLAMD_HOST: clamav
@@ -33,15 +35,16 @@ services:
       PROMETHEUS_LISTING_FEE_USD: ${PROMETHEUS_LISTING_FEE_USD:-1}
       BNS_GATEWAY: ${BNS_GATEWAY:-}
       # Sia-backed storage: when set, scan-clean uploads are pushed to the
-      # hosted s3d and the submit UI pre-fills the download URL. All four
+      # sidecar s3d and the submit UI pre-fills the download URL. The
-      # must be set for storage to turn on; otherwise publishers host
+      # endpoint points at the internal s3d container, not the public
-      # their own binaries. See service/storage.js for defaults.
+      # internet; the publicBase is Prometheus's own hostname so download
-      PROMETHEUS_S3_ENDPOINT: ${PROMETHEUS_S3_ENDPOINT:-}
+      # URLs route through /dl/<sha256> here.
+      PROMETHEUS_S3_ENDPOINT: ${PROMETHEUS_S3_ENDPOINT:-http://s3d:8000}
       PROMETHEUS_S3_ACCESS_KEY: ${PROMETHEUS_S3_ACCESS_KEY:-}
       PROMETHEUS_S3_SECRET: ${PROMETHEUS_S3_SECRET:-}
-      PROMETHEUS_S3_BUCKET: ${PROMETHEUS_S3_BUCKET:-bns}
+      PROMETHEUS_S3_BUCKET: ${PROMETHEUS_S3_BUCKET:-prometheus}
-      PROMETHEUS_S3_PREFIX: ${PROMETHEUS_S3_PREFIX:-prometheus-storage/}
+      PROMETHEUS_S3_PREFIX: ${PROMETHEUS_S3_PREFIX:-apps/}
-      PROMETHEUS_PUBLIC_BASE: ${PROMETHEUS_PUBLIC_BASE:-}
+      PROMETHEUS_PUBLIC_BASE: ${PROMETHEUS_PUBLIC_BASE:-https://prometheus.x}
     volumes:
       - data:/data
     networks:
@@ -83,6 +86,35 @@ services:
       # Signature download on first start can take a few minutes.
       start_period: 5m
 
+  # s3d sidecar — Sia S3 gateway. Serves the write side (Prometheus PUTs
+  # scan-clean uploads here) and the read side (Prometheus /dl/<sha256>
+  # proxies back out). Not exposed on the public network: the only clients
+  # are Prometheus and the operator via docker exec.
+  #
+  # First-time setup is `deploy/setup-s3d.sh` which copies the Pithos
+  # connection export's data folder in and writes a fresh s3d.yml.
+  s3d:
+    image: ghcr.io/siafoundation/s3d:latest
+    restart: unless-stopped
+    command: ["-api.s3", "0.0.0.0:8000"]
+    environment:
+      S3D_CONFIG_FILE: /data/s3d.yml
+      S3D_DATA_DIR: /data
+    volumes:
+      # Bind mount — operator pre-seeds s3d.db + s3d.yml here before first
+      # start. A named volume would hide the files from the host, making
+      # the Pithos-export import awkward.
+      - ./s3d-data:/data
+    networks:
+      - prometheus-internal
+    expose:
+      - "8000"
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8000/ >/dev/null 2>&1 || true; exit 0"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+
 volumes:
   data:
   clamav-sigs:

deploy/setup-s3d.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# One-time setup for the s3d sidecar.
+#
+# Pre-conditions
+# --------------
+# - The operator holds a Pithos connection export tarball
+#   (`pithos-connection.tar.gz`) created from the Pithos account that
+#   should own Prometheus's bucket. The export contains the connected
+#   s3d data folder — this script copies it into place.
+# - This script runs from `~/prometheus/deploy/` on the VPS.
+#
+# What this script does
+# ---------------------
+# 1. Extracts the Pithos data folder into ./s3d-data/
+# 2. Writes ./s3d-data/s3d.yml with a fresh random adminPassword
+# 3. Starts the s3d container
+# 4. Mints an S3 access key for the Prometheus service
+# 5. Appends PROMETHEUS_S3_* lines to .env
+# 6. Reports the keys that landed (operator reads them off the terminal)
+#
+# Safe to re-run: steps 2-5 are gated behind "file already there?" checks.
+# If you need a clean import, `docker compose down s3d && rm -rf s3d-data`
+# first.
+
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+TARBALL="${1:-$HOME/pithos-connection.tar.gz}"
+DATA_DIR="./s3d-data"
+ENV_FILE="./.env"
+
+if [ ! -d "$DATA_DIR" ] || [ ! -f "$DATA_DIR/s3d.db" ]; then
+  if [ ! -f "$TARBALL" ]; then
+    echo "error: no s3d data folder yet, and $TARBALL does not exist." >&2
+    echo "       scp the pithos-connection.tar.gz up first, or pass its path as \$1." >&2
+    exit 1
+  fi
+  echo "→ importing Pithos data folder from $TARBALL …"
+  mkdir -p "$DATA_DIR"
+  tar -xzf "$TARBALL" -C /tmp
+  cp -a /tmp/data/. "$DATA_DIR/"
+  rm -rf /tmp/data /tmp/MANIFEST.json /tmp/RESTORE.md
+  chown -R 1000:1000 "$DATA_DIR"   # s3d image runs as uid 1000
+fi
+
+if [ ! -f "$DATA_DIR/s3d.yml" ]; then
+  echo "→ writing fresh s3d.yml with a random adminPassword …"
+  # Random 32-byte base64 — plenty for an admin password that stays local.
+  ADMIN_PW="$(head -c 32 /dev/urandom | base64 | tr -d '/+=' | head -c 40)"
+  cat > "$DATA_DIR/s3d.yml" <<EOF
+directory: /data
+apiAddress: 0.0.0.0:8000
+adminAddress: 127.0.0.1:8001
+adminPassword: $ADMIN_PW
+EOF
+  chmod 600 "$DATA_DIR/s3d.yml"
+  chown 1000:1000 "$DATA_DIR/s3d.yml"
+fi
+
+echo "→ bringing up s3d …"
+docker compose up -d s3d
+
+# Wait for the daemon to answer.
+echo -n "→ waiting for s3d to accept connections "
+for _ in $(seq 1 30); do
+  if docker compose exec -T s3d /usr/bin/s3d status >/dev/null 2>&1; then
+    echo " ok"; break
+  fi
+  echo -n "."; sleep 1
+done
+
+# Pick a user to mint the key under. The Pithos export lands with one user
+# (the account the export was taken from); default to that.
+USER="$(docker compose exec -T s3d /usr/bin/s3d users list 2>/dev/null | awk 'NR==1{print $1}' || true)"
+if [ -z "$USER" ]; then
+  USER="prometheus"
+  echo "→ no user found, creating '$USER' …"
+  docker compose exec -T s3d /usr/bin/s3d users create "$USER"
+fi
+echo "→ using user: $USER"
+
+# If PROMETHEUS_S3_ACCESS_KEY is already set in .env, assume this already
+# ran and skip the key mint — avoids piling up unused keys on re-run.
+if grep -q '^PROMETHEUS_S3_ACCESS_KEY=' "$ENV_FILE" 2>/dev/null; then
+  echo "→ .env already has PROMETHEUS_S3_ACCESS_KEY — leaving it alone."
+else
+  echo "→ minting an access key for '$USER' …"
+  OUT="$(docker compose exec -T s3d /usr/bin/s3d keys create "$USER")"
+  AK="$(echo "$OUT" | sed -nE 's/.*Access Key:[[:space:]]+(.+)/\1/p' | tr -d '\r')"
+  SK="$(echo "$OUT" | sed -nE 's/.*Secret Key:[[:space:]]+(.+)/\1/p' | tr -d '\r')"
+  if [ -z "$AK" ] || [ -z "$SK" ]; then
+    echo "error: could not parse keys from s3d output:" >&2
+    echo "$OUT" >&2
+    exit 2
+  fi
+  touch "$ENV_FILE"
+  {
+    echo ""
+    echo "# s3d sidecar — created $(date -u +%Y-%m-%dT%H:%M:%SZ) by setup-s3d.sh"
+    echo "PROMETHEUS_S3_ENDPOINT=http://s3d:8000"
+    echo "PROMETHEUS_S3_ACCESS_KEY=$AK"
+    echo "PROMETHEUS_S3_SECRET=$SK"
+    echo "PROMETHEUS_S3_BUCKET=prometheus"
+    echo "PROMETHEUS_S3_PREFIX=apps/"
+    echo "PROMETHEUS_PUBLIC_BASE=https://prometheus.x"
+  } >> "$ENV_FILE"
+  echo "→ wrote PROMETHEUS_S3_* to $ENV_FILE"
+fi
+
+echo ""
+echo "✓ s3d sidecar is up. Restart prometheus to pick up the new .env:"
+echo "    docker compose up -d --force-recreate prometheus"

service/downloads.js

@@ -0,0 +1,90 @@
+/**
+ * GET/HEAD /dl/<sha256> — stream a stored upload back to the browser.
+ *
+ * Does not re-scan on the way out; the file lands here only after passing
+ * through the clamd + VirusTotal gate at upload time. A publisher's
+ * manifest signature covers the sha256, so the hash in the URL is also the
+ * integrity check — a reader whose client verifies the hash sees any
+ * substitution immediately.
+ *
+ * Range requests are forwarded verbatim to s3d so Theseus / Chrome / the
+ * Android download manager can resume on a flaky connection. The handler
+ * never touches the bytes — it is a thin proxy that forwards status,
+ * content-type, content-length, content-range, accept-ranges and etag.
+ *
+ * The sha256 is the only path segment we accept; anything else 404s. We
+ * never let the URL carry the original filename because the uploader
+ * chose it and a browser-saved name should be stable across republishes
+ * of the same bytes.
+ */
+
+import { getObjectStream, isConfigured } from './storage.js';
+
+const SHA256_RE = /^[0-9a-f]{64}$/;
+
+export async function handleDownload(req, res, rest) {
+  if (!isConfigured()) {
+    return respond(res, 503, 'storage not configured');
+  }
+  if (req.method !== 'GET' && req.method !== 'HEAD') {
+    return respond(res, 405, 'method not allowed');
+  }
+
+  // Strip anything after the hash (e.g., `/dl/<sha>/MyApp.exe` for a
+  // nicer saved filename) — we tolerate the suffix but don't use it.
+  const hash = String(rest || '').split('/')[0].split('?')[0].toLowerCase();
+  if (!SHA256_RE.test(hash)) return respond(res, 404, 'not found');
+
+  const range = req.headers['range'];
+
+  let upstream;
+  try {
+    upstream = await getObjectStream(hash, range ? { range } : {});
+  } catch (err) {
+    return respond(res, 502, 'upstream unreachable: ' + String(err?.message || err));
+  }
+
+  if (upstream.statusCode === 404) {
+    // Drain so we don't leak the socket, then 404.
+    upstream.resume();
+    return respond(res, 404, 'not found');
+  }
+  if (upstream.statusCode >= 400) {
+    upstream.resume();
+    return respond(res, 502, `upstream HTTP ${upstream.statusCode}`);
+  }
+
+  // Forward the storage headers the browser cares about. Deliberately
+  // omit anything s3d-specific (x-amz-*, server) to keep the response
+  // looking like a Prometheus one.
+  const headers = {};
+  const forward = ['content-type', 'content-length', 'content-range',
+    'accept-ranges', 'etag', 'last-modified'];
+  for (const h of forward) {
+    if (upstream.headers[h]) headers[h] = upstream.headers[h];
+  }
+  // Objects are content-addressed by sha256 — safe to cache forever.
+  headers['cache-control'] = 'public, max-age=31536000, immutable';
+  // Minimal defaults when upstream didn't say.
+  if (!headers['content-type']) headers['content-type'] = 'application/octet-stream';
+
+  res.writeHead(upstream.statusCode, headers);
+
+  if (req.method === 'HEAD') {
+    upstream.resume();
+    return res.end();
+  }
+
+  upstream.pipe(res);
+  // If the client disconnects, don't let the upstream read linger.
+  req.on('close', () => upstream.destroy?.());
+}
+
+function respond(res, status, text) {
+  res.writeHead(status, {
+    'content-type': 'text/plain; charset=utf-8',
+    'content-length': Buffer.byteLength(text),
+    'cache-control': 'no-store',
+  });
+  res.end(text);
+}

service/server.js

@@ -19,6 +19,7 @@ import { dirname, normalize, resolve } from 'node:path';
 import { layout } from './layout.js';
 import * as pages from './pages.js';
 import { handleUpload } from './uploads.js';
+import { handleDownload } from './downloads.js';
 import { clamdPing } from './clamd.js';
 import { publicKeyBase64, loadServiceKey } from './signing.js';
 import {
@@ -83,6 +84,11 @@ const server = createServer(async (req, res) => {
       await serveStatic(url.slice('/static/'.length), res);
     } else if (url === '/api/upload') {
       await handleUpload(req, res);
+    } else if (url.startsWith('/dl/')) {
+      // Public download endpoint — pulls the object from the sidecar s3d
+      // and streams it back. Keeps the s3d itself off the public network
+      // and lets Caddy do TLS termination + bandwidth controls on top.
+      await handleDownload(req, res, url.slice('/dl/'.length));
     } else if (url === '/api/listing') {
       await handleListingCreate(req, res);
     } else if (url === '/api/listings') {

service/storage.js

@@ -1,51 +1,49 @@
 /**
  * Sia-backed storage for scan-clean uploads.
  *
- * When the service is configured with S3 credentials pointing at the Silent
+ * When the service is configured with S3 credentials pointing at the
- * Mode hosted s3d (s3.silentmode.st), every clean upload also goes to Sia and
+ * sidecar s3d, every clean upload is pushed to Sia and the public download
- * the public download URL is handed back to the submit UI. The publisher no
+ * URL — hosted by Prometheus itself on `/dl/<sha256>` — is handed back to
- * longer has to self-host — their binary gets a stable https URL right out
+ * the submit UI. The publisher no longer has to self-host.
- * of step 1.
  *
  * Design notes
  * ------------
  * - Writes go to `s3://${bucket}/${prefix}<sha256>.bin` by content hash.
- *   Collisions on hash are a no-op: the same bytes always land at the same
+ *   The bucket is created on first use if missing, so a fresh s3d data
- *   key, so re-uploads do not duplicate storage.
+ *   folder bootstraps itself.
- * - Downloads are served through the s3 gateway's public URL
+ * - The returned URL is a Prometheus-hosted URL (`${publicBase}/dl/<sha256>`),
- *   (`${publicBase}/${bucket}/${prefix}<sha256>.bin`). The bns bucket on
+ *   NOT the raw s3d endpoint. The service's `/dl` handler streams from s3d
- *   s3.silentmode.st is already public-read so site mirrors and Pithos
+ *   on the back end; the s3d itself stays off the public network.
- *   share-links work from any client — nothing extra to configure here.
  * - If storage is not configured, isConfigured() returns false and the
  *   service falls back to its old behavior (publisher provides their own
  *   download_url). That keeps dev loops working without creds.
- * - Content-Type is passed through so the gateway serves apk/exe with the
- *   right MIME, which matters for mobile download prompts.
  *
  * Env
  * ---
- *   PROMETHEUS_S3_ENDPOINT     e.g. https://s3.silentmode.st
+ *   PROMETHEUS_S3_ENDPOINT     write-side endpoint, e.g. http://s3d:8000
- *   PROMETHEUS_S3_ACCESS_KEY   access key for a sia-storage user
+ *   PROMETHEUS_S3_ACCESS_KEY   access key minted by `s3d keys create <user>`
  *   PROMETHEUS_S3_SECRET       matching secret
- *   PROMETHEUS_S3_BUCKET       default: bns
+ *   PROMETHEUS_S3_BUCKET       default: prometheus
- *   PROMETHEUS_S3_PREFIX       default: prometheus-storage/
+ *   PROMETHEUS_S3_PREFIX       default: apps/
- *   PROMETHEUS_PUBLIC_BASE     default: same as PROMETHEUS_S3_ENDPOINT
+ *   PROMETHEUS_PUBLIC_BASE     default: https://prometheus.x  — origin for
+ *                              the public /dl URL readers follow
  */
 
 import { createReadStream } from 'node:fs';
 import { stat } from 'node:fs/promises';
-import { S3Client } from './s3.js';
+import { S3Client, S3Error } from './s3.js';
 
 const CFG = {
   endpoint: process.env.PROMETHEUS_S3_ENDPOINT || '',
   accessKey: process.env.PROMETHEUS_S3_ACCESS_KEY || '',
   secret: process.env.PROMETHEUS_S3_SECRET || '',
-  bucket: process.env.PROMETHEUS_S3_BUCKET || 'bns',
+  bucket: process.env.PROMETHEUS_S3_BUCKET || 'prometheus',
-  prefix: normalizePrefix(process.env.PROMETHEUS_S3_PREFIX || 'prometheus-storage/'),
+  prefix: normalizePrefix(process.env.PROMETHEUS_S3_PREFIX || 'apps/'),
-  publicBase: (process.env.PROMETHEUS_PUBLIC_BASE || process.env.PROMETHEUS_S3_ENDPOINT || '').replace(/\/+$/, ''),
+  publicBase: (process.env.PROMETHEUS_PUBLIC_BASE || 'https://prometheus.x').replace(/\/+$/, ''),
 };
 
 let client = null;
+let bucketReady = false;
 
 export function isConfigured() {
   return Boolean(CFG.endpoint && CFG.accessKey && CFG.secret);
@@ -74,18 +72,36 @@ function getClient() {
   return client;
 }
 
+/** Make sure the bucket exists; called once lazily, idempotent on error. */
+async function ensureBucket(c) {
+  if (bucketReady) return;
+  try {
+    await c.createBucket(CFG.bucket);
+  } catch (err) {
+    // BucketAlreadyOwnedByYou / BucketAlreadyExists → done. Anything else
+    // is a real failure the caller should see.
+    if (err instanceof S3Error && /already/i.test(err.code || '')) {
+      // noop
+    } else if (err?.status === 409) {
+      // noop
+    } else {
+      throw err;
+    }
+  }
+  bucketReady = true;
+}
+
 /**
  * Upload a scan-clean file to Sia and return the public URL readers can fetch.
  *
  * Idempotent: if an object with the same key already exists (same sha256 →
  * same bytes), the PUT overwrites with identical content and the URL is the
- * same. The returned URL is stable across reruns, so a listing that already
+ * same. The returned URL is stable across reruns.
- * references it stays valid.
  *
  * @param {object} args
  * @param {string} args.localPath — absolute path to the file on disk
  * @param {string} args.sha256    — content hash, used as the object key
- * @param {string} args.name      — original filename (for the display suffix)
+ * @param {string} args.name      — original filename (for display suffix, unused server-side)
  * @param {string} args.mime      — content-type for the stored object
  * @returns {Promise<{ downloadUrl: string, key: string, bytes: number }>}
  */
@@ -93,32 +109,38 @@ export async function uploadScannedFile({ localPath, sha256, name, mime }) {
   const c = getClient();
   if (!c) throw new Error('storage not configured — set PROMETHEUS_S3_*');
 
+  await ensureBucket(c);
+
   const { size } = await stat(localPath);
   const key = `${CFG.prefix}${sha256}.bin`;
   const body = createReadStream(localPath);
   const contentType = mime || 'application/octet-stream';
 
-  // The s3d upstream wants Content-Length for streaming PUTs; without it
+  // s3d wants a Content-Length on streaming PUTs; chunked encoding does not
-  // s3d falls back to transfer-encoding: chunked which Sia's gateway does
+  // round-trip cleanly for large bodies in its current gateway.
-  // not pass through cleanly on large bodies. Pithos has the same gotcha.
   await c.putObject(CFG.bucket, key, body, {
     contentType,
     contentLength: size,
   });
 
-  // Content-Disposition is a client decision, not something we persist on
+  return { downloadUrl: publicUrlFor(sha256), key, bytes: size };
-  // the object; the UI can add it to the download link if needed. We just
-  // hand back the raw public URL — stable, cacheable, hash-addressed.
-  const downloadUrl = publicUrlFor(key);
-  return { downloadUrl, key, bytes: size };
 }
 
-export function publicUrlFor(key) {
+/**
-  if (!CFG.publicBase) return null;
+ * Open a read stream from s3d for a given sha256. Returns the raw HTTP
-  // Path-style URL: the bucket is a path segment, matching how s3d exposes
+ * response so callers can stream it straight back to the browser and
-  // the hosted gateway. (Virtual-hosted URLs would need per-bucket DNS,
+ * forward content-type / content-length / etag / accept-ranges. Supports
-  // which the public gateway does not set up.)
+ * Range requests via the optional `range` header.
-  return `${CFG.publicBase}/${encodeURIComponent(CFG.bucket)}/${encodeURI(key)}`;
+ */
+export async function getObjectStream(sha256, { range } = {}) {
+  const c = getClient();
+  if (!c) throw new Error('storage not configured');
+  const key = `${CFG.prefix}${sha256}.bin`;
+  return c.getObject(CFG.bucket, key, range ? { range } : {});
+}
+
+export function publicUrlFor(sha256) {
+  return `${CFG.publicBase}/dl/${sha256}`;
 }
 
 function normalizePrefix(p) {