From d25ed36e146a6a6d273e20650b18a35f15cd2057 Mon Sep 17 00:00:00 2001 From: Local Dev Date: Sat, 24 Oct 2026 01:08:24 +0300 Subject: [PATCH] prometheus: s3d sidecar + /dl/ for Sia-backed hosting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishers no longer need external hosting at all. The service runs its own s3d (SiaFoundation gateway) as a Docker sidecar on the internal compose network; scan-clean uploads land there via PUT; readers fetch through /dl/ on prometheus.x which proxies back out from s3d. - deploy/docker-compose.yml: s3d service added. Image is ghcr.io/siafoundation/s3d:latest, data folder bind-mounted at ./s3d-data/ so the operator can seed it from a Pithos connection export. Only prometheus-internal network — never public. - deploy/setup-s3d.sh: one-time import of the Pithos connection tarball, writes a fresh s3d.yml with a random adminPassword, mints an S3 access key for the Prometheus service, appends PROMETHEUS_S3_* lines to .env. Idempotent on re-run. - service/storage.js: endpoint now defaults to the internal s3d (http://s3d:8000); publicBase defaults to https://prometheus.x so download URLs flow through the service's own /dl handler. Bucket is auto-created on first use so a fresh data folder bootstraps. Added getObjectStream() for the read side. - service/downloads.js: GET/HEAD /dl/ — forwards Range, content-type and content-length from s3d, drops s3d-specific headers (x-amz-*), sets immutable long-cache. Hash is validated against ^[0-9a-f]{64}$ before touching upstream. - service/server.js: route table wires /dl/* to the handler above. --- deploy/docker-compose.yml | 46 +++++++++++++--- deploy/setup-s3d.sh | 113 ++++++++++++++++++++++++++++++++++++++ service/downloads.js | 90 ++++++++++++++++++++++++++++++ service/server.js | 6 ++ service/storage.js | 100 ++++++++++++++++++++------------- 5 files changed, 309 insertions(+), 46 deletions(-) create mode 100644 deploy/setup-s3d.sh create mode 100644 service/downloads.js diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 903119d..da7b8f4 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -22,6 +22,8 @@ services: depends_on: clamav: condition: service_healthy + s3d: + condition: service_started environment: BNS_NETWORK: chipnet CLAMD_HOST: clamav @@ -33,15 +35,16 @@ services: PROMETHEUS_LISTING_FEE_USD: ${PROMETHEUS_LISTING_FEE_USD:-1} BNS_GATEWAY: ${BNS_GATEWAY:-} # Sia-backed storage: when set, scan-clean uploads are pushed to the - # hosted s3d and the submit UI pre-fills the download URL. All four - # must be set for storage to turn on; otherwise publishers host - # their own binaries. See service/storage.js for defaults. - PROMETHEUS_S3_ENDPOINT: ${PROMETHEUS_S3_ENDPOINT:-} + # sidecar s3d and the submit UI pre-fills the download URL. The + # endpoint points at the internal s3d container, not the public + # internet; the publicBase is Prometheus's own hostname so download + # URLs route through /dl/ here. + PROMETHEUS_S3_ENDPOINT: ${PROMETHEUS_S3_ENDPOINT:-http://s3d:8000} PROMETHEUS_S3_ACCESS_KEY: ${PROMETHEUS_S3_ACCESS_KEY:-} PROMETHEUS_S3_SECRET: ${PROMETHEUS_S3_SECRET:-} - PROMETHEUS_S3_BUCKET: ${PROMETHEUS_S3_BUCKET:-bns} - PROMETHEUS_S3_PREFIX: ${PROMETHEUS_S3_PREFIX:-prometheus-storage/} - PROMETHEUS_PUBLIC_BASE: ${PROMETHEUS_PUBLIC_BASE:-} + PROMETHEUS_S3_BUCKET: ${PROMETHEUS_S3_BUCKET:-prometheus} + PROMETHEUS_S3_PREFIX: ${PROMETHEUS_S3_PREFIX:-apps/} + PROMETHEUS_PUBLIC_BASE: ${PROMETHEUS_PUBLIC_BASE:-https://prometheus.x} volumes: - data:/data networks: @@ -83,6 +86,35 @@ services: # Signature download on first start can take a few minutes. start_period: 5m + # s3d sidecar — Sia S3 gateway. Serves the write side (Prometheus PUTs + # scan-clean uploads here) and the read side (Prometheus /dl/ + # proxies back out). Not exposed on the public network: the only clients + # are Prometheus and the operator via docker exec. + # + # First-time setup is `deploy/setup-s3d.sh` which copies the Pithos + # connection export's data folder in and writes a fresh s3d.yml. + s3d: + image: ghcr.io/siafoundation/s3d:latest + restart: unless-stopped + command: ["-api.s3", "0.0.0.0:8000"] + environment: + S3D_CONFIG_FILE: /data/s3d.yml + S3D_DATA_DIR: /data + volumes: + # Bind mount — operator pre-seeds s3d.db + s3d.yml here before first + # start. A named volume would hide the files from the host, making + # the Pithos-export import awkward. + - ./s3d-data:/data + networks: + - prometheus-internal + expose: + - "8000" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8000/ >/dev/null 2>&1 || true; exit 0"] + interval: 60s + timeout: 5s + retries: 3 + volumes: data: clamav-sigs: diff --git a/deploy/setup-s3d.sh b/deploy/setup-s3d.sh new file mode 100644 index 0000000..98d4104 --- /dev/null +++ b/deploy/setup-s3d.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# One-time setup for the s3d sidecar. +# +# Pre-conditions +# -------------- +# - The operator holds a Pithos connection export tarball +# (`pithos-connection.tar.gz`) created from the Pithos account that +# should own Prometheus's bucket. The export contains the connected +# s3d data folder — this script copies it into place. +# - This script runs from `~/prometheus/deploy/` on the VPS. +# +# What this script does +# --------------------- +# 1. Extracts the Pithos data folder into ./s3d-data/ +# 2. Writes ./s3d-data/s3d.yml with a fresh random adminPassword +# 3. Starts the s3d container +# 4. Mints an S3 access key for the Prometheus service +# 5. Appends PROMETHEUS_S3_* lines to .env +# 6. Reports the keys that landed (operator reads them off the terminal) +# +# Safe to re-run: steps 2-5 are gated behind "file already there?" checks. +# If you need a clean import, `docker compose down s3d && rm -rf s3d-data` +# first. + +set -euo pipefail + +cd "$(dirname "$0")" + +TARBALL="${1:-$HOME/pithos-connection.tar.gz}" +DATA_DIR="./s3d-data" +ENV_FILE="./.env" + +if [ ! -d "$DATA_DIR" ] || [ ! -f "$DATA_DIR/s3d.db" ]; then + if [ ! -f "$TARBALL" ]; then + echo "error: no s3d data folder yet, and $TARBALL does not exist." >&2 + echo " scp the pithos-connection.tar.gz up first, or pass its path as \$1." >&2 + exit 1 + fi + echo "→ importing Pithos data folder from $TARBALL …" + mkdir -p "$DATA_DIR" + tar -xzf "$TARBALL" -C /tmp + cp -a /tmp/data/. "$DATA_DIR/" + rm -rf /tmp/data /tmp/MANIFEST.json /tmp/RESTORE.md + chown -R 1000:1000 "$DATA_DIR" # s3d image runs as uid 1000 +fi + +if [ ! -f "$DATA_DIR/s3d.yml" ]; then + echo "→ writing fresh s3d.yml with a random adminPassword …" + # Random 32-byte base64 — plenty for an admin password that stays local. + ADMIN_PW="$(head -c 32 /dev/urandom | base64 | tr -d '/+=' | head -c 40)" + cat > "$DATA_DIR/s3d.yml" </dev/null 2>&1; then + echo " ok"; break + fi + echo -n "."; sleep 1 +done + +# Pick a user to mint the key under. The Pithos export lands with one user +# (the account the export was taken from); default to that. +USER="$(docker compose exec -T s3d /usr/bin/s3d users list 2>/dev/null | awk 'NR==1{print $1}' || true)" +if [ -z "$USER" ]; then + USER="prometheus" + echo "→ no user found, creating '$USER' …" + docker compose exec -T s3d /usr/bin/s3d users create "$USER" +fi +echo "→ using user: $USER" + +# If PROMETHEUS_S3_ACCESS_KEY is already set in .env, assume this already +# ran and skip the key mint — avoids piling up unused keys on re-run. +if grep -q '^PROMETHEUS_S3_ACCESS_KEY=' "$ENV_FILE" 2>/dev/null; then + echo "→ .env already has PROMETHEUS_S3_ACCESS_KEY — leaving it alone." +else + echo "→ minting an access key for '$USER' …" + OUT="$(docker compose exec -T s3d /usr/bin/s3d keys create "$USER")" + AK="$(echo "$OUT" | sed -nE 's/.*Access Key:[[:space:]]+(.+)/\1/p' | tr -d '\r')" + SK="$(echo "$OUT" | sed -nE 's/.*Secret Key:[[:space:]]+(.+)/\1/p' | tr -d '\r')" + if [ -z "$AK" ] || [ -z "$SK" ]; then + echo "error: could not parse keys from s3d output:" >&2 + echo "$OUT" >&2 + exit 2 + fi + touch "$ENV_FILE" + { + echo "" + echo "# s3d sidecar — created $(date -u +%Y-%m-%dT%H:%M:%SZ) by setup-s3d.sh" + echo "PROMETHEUS_S3_ENDPOINT=http://s3d:8000" + echo "PROMETHEUS_S3_ACCESS_KEY=$AK" + echo "PROMETHEUS_S3_SECRET=$SK" + echo "PROMETHEUS_S3_BUCKET=prometheus" + echo "PROMETHEUS_S3_PREFIX=apps/" + echo "PROMETHEUS_PUBLIC_BASE=https://prometheus.x" + } >> "$ENV_FILE" + echo "→ wrote PROMETHEUS_S3_* to $ENV_FILE" +fi + +echo "" +echo "✓ s3d sidecar is up. Restart prometheus to pick up the new .env:" +echo " docker compose up -d --force-recreate prometheus" diff --git a/service/downloads.js b/service/downloads.js new file mode 100644 index 0000000..cd935f2 --- /dev/null +++ b/service/downloads.js @@ -0,0 +1,90 @@ +/** + * GET/HEAD /dl/ — stream a stored upload back to the browser. + * + * Does not re-scan on the way out; the file lands here only after passing + * through the clamd + VirusTotal gate at upload time. A publisher's + * manifest signature covers the sha256, so the hash in the URL is also the + * integrity check — a reader whose client verifies the hash sees any + * substitution immediately. + * + * Range requests are forwarded verbatim to s3d so Theseus / Chrome / the + * Android download manager can resume on a flaky connection. The handler + * never touches the bytes — it is a thin proxy that forwards status, + * content-type, content-length, content-range, accept-ranges and etag. + * + * The sha256 is the only path segment we accept; anything else 404s. We + * never let the URL carry the original filename because the uploader + * chose it and a browser-saved name should be stable across republishes + * of the same bytes. + */ + +import { getObjectStream, isConfigured } from './storage.js'; + +const SHA256_RE = /^[0-9a-f]{64}$/; + +export async function handleDownload(req, res, rest) { + if (!isConfigured()) { + return respond(res, 503, 'storage not configured'); + } + if (req.method !== 'GET' && req.method !== 'HEAD') { + return respond(res, 405, 'method not allowed'); + } + + // Strip anything after the hash (e.g., `/dl//MyApp.exe` for a + // nicer saved filename) — we tolerate the suffix but don't use it. + const hash = String(rest || '').split('/')[0].split('?')[0].toLowerCase(); + if (!SHA256_RE.test(hash)) return respond(res, 404, 'not found'); + + const range = req.headers['range']; + + let upstream; + try { + upstream = await getObjectStream(hash, range ? { range } : {}); + } catch (err) { + return respond(res, 502, 'upstream unreachable: ' + String(err?.message || err)); + } + + if (upstream.statusCode === 404) { + // Drain so we don't leak the socket, then 404. + upstream.resume(); + return respond(res, 404, 'not found'); + } + if (upstream.statusCode >= 400) { + upstream.resume(); + return respond(res, 502, `upstream HTTP ${upstream.statusCode}`); + } + + // Forward the storage headers the browser cares about. Deliberately + // omit anything s3d-specific (x-amz-*, server) to keep the response + // looking like a Prometheus one. + const headers = {}; + const forward = ['content-type', 'content-length', 'content-range', + 'accept-ranges', 'etag', 'last-modified']; + for (const h of forward) { + if (upstream.headers[h]) headers[h] = upstream.headers[h]; + } + // Objects are content-addressed by sha256 — safe to cache forever. + headers['cache-control'] = 'public, max-age=31536000, immutable'; + // Minimal defaults when upstream didn't say. + if (!headers['content-type']) headers['content-type'] = 'application/octet-stream'; + + res.writeHead(upstream.statusCode, headers); + + if (req.method === 'HEAD') { + upstream.resume(); + return res.end(); + } + + upstream.pipe(res); + // If the client disconnects, don't let the upstream read linger. + req.on('close', () => upstream.destroy?.()); +} + +function respond(res, status, text) { + res.writeHead(status, { + 'content-type': 'text/plain; charset=utf-8', + 'content-length': Buffer.byteLength(text), + 'cache-control': 'no-store', + }); + res.end(text); +} diff --git a/service/server.js b/service/server.js index 2d0039d..9b61546 100644 --- a/service/server.js +++ b/service/server.js @@ -19,6 +19,7 @@ import { dirname, normalize, resolve } from 'node:path'; import { layout } from './layout.js'; import * as pages from './pages.js'; import { handleUpload } from './uploads.js'; +import { handleDownload } from './downloads.js'; import { clamdPing } from './clamd.js'; import { publicKeyBase64, loadServiceKey } from './signing.js'; import { @@ -83,6 +84,11 @@ const server = createServer(async (req, res) => { await serveStatic(url.slice('/static/'.length), res); } else if (url === '/api/upload') { await handleUpload(req, res); + } else if (url.startsWith('/dl/')) { + // Public download endpoint — pulls the object from the sidecar s3d + // and streams it back. Keeps the s3d itself off the public network + // and lets Caddy do TLS termination + bandwidth controls on top. + await handleDownload(req, res, url.slice('/dl/'.length)); } else if (url === '/api/listing') { await handleListingCreate(req, res); } else if (url === '/api/listings') { diff --git a/service/storage.js b/service/storage.js index 4677ca8..7d16bf5 100644 --- a/service/storage.js +++ b/service/storage.js @@ -1,51 +1,49 @@ /** * Sia-backed storage for scan-clean uploads. * - * When the service is configured with S3 credentials pointing at the Silent - * Mode hosted s3d (s3.silentmode.st), every clean upload also goes to Sia and - * the public download URL is handed back to the submit UI. The publisher no - * longer has to self-host — their binary gets a stable https URL right out - * of step 1. + * When the service is configured with S3 credentials pointing at the + * sidecar s3d, every clean upload is pushed to Sia and the public download + * URL — hosted by Prometheus itself on `/dl/` — is handed back to + * the submit UI. The publisher no longer has to self-host. * * Design notes * ------------ * - Writes go to `s3://${bucket}/${prefix}.bin` by content hash. - * Collisions on hash are a no-op: the same bytes always land at the same - * key, so re-uploads do not duplicate storage. - * - Downloads are served through the s3 gateway's public URL - * (`${publicBase}/${bucket}/${prefix}.bin`). The bns bucket on - * s3.silentmode.st is already public-read so site mirrors and Pithos - * share-links work from any client — nothing extra to configure here. + * The bucket is created on first use if missing, so a fresh s3d data + * folder bootstraps itself. + * - The returned URL is a Prometheus-hosted URL (`${publicBase}/dl/`), + * NOT the raw s3d endpoint. The service's `/dl` handler streams from s3d + * on the back end; the s3d itself stays off the public network. * - If storage is not configured, isConfigured() returns false and the * service falls back to its old behavior (publisher provides their own * download_url). That keeps dev loops working without creds. - * - Content-Type is passed through so the gateway serves apk/exe with the - * right MIME, which matters for mobile download prompts. * * Env * --- - * PROMETHEUS_S3_ENDPOINT e.g. https://s3.silentmode.st - * PROMETHEUS_S3_ACCESS_KEY access key for a sia-storage user + * PROMETHEUS_S3_ENDPOINT write-side endpoint, e.g. http://s3d:8000 + * PROMETHEUS_S3_ACCESS_KEY access key minted by `s3d keys create ` * PROMETHEUS_S3_SECRET matching secret - * PROMETHEUS_S3_BUCKET default: bns - * PROMETHEUS_S3_PREFIX default: prometheus-storage/ - * PROMETHEUS_PUBLIC_BASE default: same as PROMETHEUS_S3_ENDPOINT + * PROMETHEUS_S3_BUCKET default: prometheus + * PROMETHEUS_S3_PREFIX default: apps/ + * PROMETHEUS_PUBLIC_BASE default: https://prometheus.x — origin for + * the public /dl URL readers follow */ import { createReadStream } from 'node:fs'; import { stat } from 'node:fs/promises'; -import { S3Client } from './s3.js'; +import { S3Client, S3Error } from './s3.js'; const CFG = { endpoint: process.env.PROMETHEUS_S3_ENDPOINT || '', accessKey: process.env.PROMETHEUS_S3_ACCESS_KEY || '', secret: process.env.PROMETHEUS_S3_SECRET || '', - bucket: process.env.PROMETHEUS_S3_BUCKET || 'bns', - prefix: normalizePrefix(process.env.PROMETHEUS_S3_PREFIX || 'prometheus-storage/'), - publicBase: (process.env.PROMETHEUS_PUBLIC_BASE || process.env.PROMETHEUS_S3_ENDPOINT || '').replace(/\/+$/, ''), + bucket: process.env.PROMETHEUS_S3_BUCKET || 'prometheus', + prefix: normalizePrefix(process.env.PROMETHEUS_S3_PREFIX || 'apps/'), + publicBase: (process.env.PROMETHEUS_PUBLIC_BASE || 'https://prometheus.x').replace(/\/+$/, ''), }; let client = null; +let bucketReady = false; export function isConfigured() { return Boolean(CFG.endpoint && CFG.accessKey && CFG.secret); @@ -74,18 +72,36 @@ function getClient() { return client; } +/** Make sure the bucket exists; called once lazily, idempotent on error. */ +async function ensureBucket(c) { + if (bucketReady) return; + try { + await c.createBucket(CFG.bucket); + } catch (err) { + // BucketAlreadyOwnedByYou / BucketAlreadyExists → done. Anything else + // is a real failure the caller should see. + if (err instanceof S3Error && /already/i.test(err.code || '')) { + // noop + } else if (err?.status === 409) { + // noop + } else { + throw err; + } + } + bucketReady = true; +} + /** * Upload a scan-clean file to Sia and return the public URL readers can fetch. * * Idempotent: if an object with the same key already exists (same sha256 → * same bytes), the PUT overwrites with identical content and the URL is the - * same. The returned URL is stable across reruns, so a listing that already - * references it stays valid. + * same. The returned URL is stable across reruns. * * @param {object} args * @param {string} args.localPath — absolute path to the file on disk * @param {string} args.sha256 — content hash, used as the object key - * @param {string} args.name — original filename (for the display suffix) + * @param {string} args.name — original filename (for display suffix, unused server-side) * @param {string} args.mime — content-type for the stored object * @returns {Promise<{ downloadUrl: string, key: string, bytes: number }>} */ @@ -93,32 +109,38 @@ export async function uploadScannedFile({ localPath, sha256, name, mime }) { const c = getClient(); if (!c) throw new Error('storage not configured — set PROMETHEUS_S3_*'); + await ensureBucket(c); + const { size } = await stat(localPath); const key = `${CFG.prefix}${sha256}.bin`; const body = createReadStream(localPath); const contentType = mime || 'application/octet-stream'; - // The s3d upstream wants Content-Length for streaming PUTs; without it - // s3d falls back to transfer-encoding: chunked which Sia's gateway does - // not pass through cleanly on large bodies. Pithos has the same gotcha. + // s3d wants a Content-Length on streaming PUTs; chunked encoding does not + // round-trip cleanly for large bodies in its current gateway. await c.putObject(CFG.bucket, key, body, { contentType, contentLength: size, }); - // Content-Disposition is a client decision, not something we persist on - // the object; the UI can add it to the download link if needed. We just - // hand back the raw public URL — stable, cacheable, hash-addressed. - const downloadUrl = publicUrlFor(key); - return { downloadUrl, key, bytes: size }; + return { downloadUrl: publicUrlFor(sha256), key, bytes: size }; } -export function publicUrlFor(key) { - if (!CFG.publicBase) return null; - // Path-style URL: the bucket is a path segment, matching how s3d exposes - // the hosted gateway. (Virtual-hosted URLs would need per-bucket DNS, - // which the public gateway does not set up.) - return `${CFG.publicBase}/${encodeURIComponent(CFG.bucket)}/${encodeURI(key)}`; +/** + * Open a read stream from s3d for a given sha256. Returns the raw HTTP + * response so callers can stream it straight back to the browser and + * forward content-type / content-length / etag / accept-ranges. Supports + * Range requests via the optional `range` header. + */ +export async function getObjectStream(sha256, { range } = {}) { + const c = getClient(); + if (!c) throw new Error('storage not configured'); + const key = `${CFG.prefix}${sha256}.bin`; + return c.getObject(CFG.bucket, key, range ? { range } : {}); +} + +export function publicUrlFor(sha256) { + return `${CFG.publicBase}/dl/${sha256}`; } function normalizePrefix(p) {