BNS: publish the raw name snapshot publicly, in two copies

The only published raw snapshot sat on the Sia S3 endpoint, which needs
SigV4 for every read: every Theseus launch got a 403, and the copy had not
been refreshed since 2026-08-31 (it was written by hand).

cdn-name-snapshot.mjs runs on the VPS every 10 min: one electrum history
call plus only the new transactions, then writes the same raw format as two
public, read-only copies on two storage backends:
  https://dl.silentmode.st/bns-name-snapshot.json
  https://navigate.st/bns/silentmode.bch/bns-name-snapshot.json  (Sia)
No wallet or credentials are needed to read either; clients still rebuild
the index from the transactions. A failed upload of the Sia copy retries on
the next run. nameListUrl (+ nameListMirrorUrl) point at them.
This commit is contained in:
Local Dev 2026-10-03 13:08:27 +02:00
parent 0a32023a79
commit a462441e66
2 changed files with 112 additions and 1 deletions

105
src/cdn-name-snapshot.mjs Normal file
View file

@ -0,0 +1,105 @@
// Keep a public, read-only RAW name-beacon snapshot on the CDN.
//
// Theseus warm-starts its name index from a raw snapshot — the beacon's
// history plus the verbose transactions — and rebuilds the index from that
// evidence itself (buildIndexFromSnapshot), so the operator is not trusted
// for the interpretation. The only published copy lived on the Sia S3
// endpoint, which needs SigV4 credentials for every read: every Theseus
// launch got a 403 and the copy was never refreshed anyway (it was written
// by publish-name-mirror.mjs, by hand, last on 2026-08-31).
//
// This writes the same snapshot format to a plain static file served by the
// dl vhost — public, view-only, no credentials, no wallet. Each run fetches
// the beacon history over electrum and only the transactions the previous
// file doesn't already hold, so a run is one history call plus whatever is
// new. Runs on the VPS from a systemd timer (VPS/SilentMode/cdn-name-snapshot.*),
// beside cdn-name-index.mjs, which publishes the interpreted index.
//
// Two public, read-only copies, on two storage backends:
// 1. --out the dl vhost's disk (https://dl.silentmode.st/bns-name-snapshot.json)
// 2. --sia <s3 uri> the Sia bucket, served by the gateway
// (https://navigate.st/bns/silentmode.bch/bns-name-snapshot.json)
// The Sia copy is uploaded whenever its last uploaded root (kept beside --out
// as <out>.sia-root) differs from the current one, so a failed upload heals
// on the next run. Credentials come from the environment (AK / SK, as in
// /etc/s3d/sync.env) and an S3 endpoint (--s3-endpoint).
//
// node cdn-name-snapshot.mjs [--out /opt/silent-mode/dl/bns-name-snapshot.json]
// [--sia s3://bns/silentmode/bns-name-snapshot.json --s3-endpoint http://127.0.0.1:8500]
import { readFileSync, writeFileSync, renameSync, existsSync } from "node:fs";
import { spawnSync } from "node:child_process";
import WebSocket from "ws";
// The gateway runtime on the VPS is flat (/opt/bns-gateway/*.js); the repo
// keeps the libraries under lib/.
async function load(file) {
for (const p of [`./${file}`, `./lib/${file}`]) {
try { return await import(new URL(p, import.meta.url)); }
catch (e) { if (e?.code !== "ERR_MODULE_NOT_FOUND") throw e; }
}
throw new Error(`${file} not found next to this script or in lib/`);
}
const R = await load("resolver-web.js");
const { buildNameSnapshot } = await load("snapshot-name.js");
const args = process.argv.slice(2);
const opt = (k, d) => { const i = args.indexOf(k); return i >= 0 && args[i + 1] ? args[i + 1] : d; };
const OUT = opt("--out", "/opt/silent-mode/dl/bns-name-snapshot.json");
const SIA = opt("--sia", "");
const S3_ENDPOINT = opt("--s3-endpoint", "http://127.0.0.1:8500");
const MIN_NAMES = 10; // fewer is a broken answer, not the registry
// Second copy: push the file at OUT to Sia unless that root is already there.
function mirrorToSia(root) {
if (!SIA) return;
const marker = OUT + ".sia-root";
let done = ""; try { done = readFileSync(marker, "utf8").trim(); } catch {}
if (done === root) return;
const env = { ...process.env };
if (env.AK && !env.AWS_ACCESS_KEY_ID) env.AWS_ACCESS_KEY_ID = env.AK;
if (env.SK && !env.AWS_SECRET_ACCESS_KEY) env.AWS_SECRET_ACCESS_KEY = env.SK;
const r = spawnSync("aws", ["--endpoint-url", S3_ENDPOINT, "s3", "cp", OUT, SIA, "--content-type", "application/json", "--only-show-errors"], { env, encoding: "utf8" });
if (r.status !== 0) { console.error(`sia mirror failed (${r.status}): ${(r.stderr || r.error?.message || "").trim()}`); process.exitCode = 1; return; }
writeFileSync(marker, root);
console.log(`mirrored to ${SIA} (root ${root.slice(0, 12)})`);
}
let prev = null;
try { if (existsSync(OUT)) prev = JSON.parse(readFileSync(OUT, "utf8")); } catch { prev = null; }
const prevTxs = (prev && prev.beacon === R.BEACON_SCRIPTHASH && prev.txs) || {};
const el = await R.connectElectrum({ WebSocket, directIP: true });
let history, txs = {}, fetched = 0;
try {
history = await el.call("blockchain.scripthash.get_history", [R.BEACON_SCRIPTHASH]);
if (!Array.isArray(history)) throw new Error("electrum returned no history");
for (const h of history) {
if (prevTxs[h.tx_hash]) { txs[h.tx_hash] = prevTxs[h.tx_hash]; continue; }
txs[h.tx_hash] = await el.call("blockchain.transaction.get", [h.tx_hash, true]);
fetched++;
}
} finally { try { el.close(); } catch {} }
// A server answering with a shorter history than we already published is
// lagging (or lying); keep the published copy rather than go backwards.
if (prev && Array.isArray(prev.history) && history.length < prev.history.length) {
console.log(`electrum history shorter than published (${history.length} < ${prev.history.length}) — not writing`);
process.exit(0);
}
const heights = history.map((h) => Number(h.height) || 0).filter((h) => h > 0);
const snapshot = buildNameSnapshot({
beacon: R.BEACON_SCRIPTHASH,
history: history.map((h) => ({ tx_hash: h.tx_hash, height: h.height })),
txs,
asOfHeight: heights.length ? Math.max(...heights) : null,
});
const names = R.buildIndexFromSnapshot({ snapshot }).size;
if (names < MIN_NAMES) throw new Error(`snapshot resolves to only ${names} names — not writing`);
if (prev && prev.root === snapshot.root) {
console.log(`unchanged (${names} names, root ${snapshot.root.slice(0, 12)})`);
} else {
writeFileSync(OUT + ".tmp", JSON.stringify(snapshot));
renameSync(OUT + ".tmp", OUT);
console.log(`wrote ${OUT}: ${history.length} beacon txs (${fetched} fetched), ${names} names, height ${snapshot.asOfHeight ?? "?"}, root ${snapshot.root.slice(0, 12)}`);
}
mirrorToSia(snapshot.root);

View file

@ -65,7 +65,13 @@ export const NETWORKS = {
// Sia bucket for operator-provided hosting (`<bucket>/<name>/`) and the
// published snapshots; CDN copy and Nostr d-tags of the same lists.
hostingBucket: "bns",
nameListUrl: "https://s3.silentmode.st:8600/bns/name-list.json",
// Raw beacon snapshot (history + txs; clients rebuild the index from it),
// public and read-only — refreshed every 10 min by cdn-name-snapshot.mjs,
// which keeps two copies on two storage backends: the dl vhost's disk and
// the Sia bucket behind the gateway. (The old bns/name-list.json on the
// Sia S3 endpoint needs SigV4 for every read.)
nameListUrl: "https://dl.silentmode.st/bns-name-snapshot.json",
nameListMirrorUrl: "https://navigate.st/bns/silentmode.bch/bns-name-snapshot.json",
cdnIndexUrl: "https://dl.silentmode.st/bns-name-index.json",
nostrTags: { names: "bns-name-list", tlds: "bns-tld-list" },
explorerTx: "https://chipnet.imaginary.cash/tx/",