dgb-wallet/packages/p2p/bin/dgb-daemon.ts

881 lines
43 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// Local HTTP daemon that exposes @dgb-wallet/p2p features to non-Node
// consumers (the web UI, which can't do raw TCP).
//
// Endpoints:
// GET /health — pool + chain status
// POST /broadcast — {hex} → {txid, requestedBy, totalPeers, relayed, relayedBy, via}
// replies within EARLY_REPLY_MS; if the relay
// echo is still pending → {txid, pending:true}
// GET /broadcast/status?txid=… → same shape, state:'pending'|'done'|'failed'
// POST /balance — {xpub, purpose, account,
// gapLimit?, startHeight?} → {totalSats, utxos, ...}
//
// Bound to 127.0.0.1 only. No auth (localhost trust boundary).
//
// Usage: npm run daemon -w @dgb-wallet/p2p [--port 5174]
import * as http from 'node:http';
import { existsSync, readFileSync, writeFileSync, renameSync } from 'node:fs';
import { join } from 'node:path';
import { createHash } from 'node:crypto';
import { BIP32Factory } from 'bip32';
import * as ecc from '@bitcoinerlab/secp256k1';
import { payments, Transaction } from 'bitcoinjs-lib';
import { digibyte, DGB_COIN_TYPE } from '@dgb-wallet/core';
import type { Block } from 'bitcoinjs-lib';
import {
PeerPool,
HeaderChain,
FilterCache,
defaultCacheDir,
encodeGetHeaders,
encodeGetBlock,
broadcastTx,
broadcastDandelion,
scanWithPeers,
crawlFilters,
checkProofOfWork,
workFromBits,
loadPowWasm,
NODE_COMPACT_FILTERS,
NODE_NETWORK,
NODE_NETWORK_LIMITED,
type BlockHeader,
} from '../src/index.js';
const args = process.argv.slice(2);
const portFlagIdx = args.indexOf('--port');
const PORT = portFlagIdx >= 0 ? Number(args[portFlagIdx + 1]) : 5174;
// A tiny fraction of the DGB network advertises NODE_COMPACT_FILTERS
// (only a few nodes have the BIP157/158 patchset), so a small pool
// starves the filter path. 32 gets us a couple of CF peers in practice.
const POOL_SIZE = 32;
const DGB_GENESIS_HASH = '7497ea1b465eb39f1c8f507bc877078fe016d6fcb6dfad3a64c98dcc6e1e8496';
const bip32 = BIP32Factory(ecc);
console.log(`[daemon] starting P2P pool (target ${POOL_SIZE}, needs at least 1 CF-serving peer for /balance)`);
const pool = new PeerPool({ poolSize: POOL_SIZE, peerOpts: { handshakeTimeoutMs: 10_000 } });
pool.on('connect', ({ host, services }: { host: string; services: string }) =>
console.log(`[pool] +${host} services=${services}`),
);
pool.on('disconnect', ({ key }: { key: string }) => console.log(`[pool] -${key}`));
// ---- Disk-backed header chain (constant RAM) + background sync ----
const filterCache = new FilterCache({ dir: defaultCacheDir() });
const chain = new HeaderChain({ dir: defaultCacheDir() });
chain.appendGenesis({
hash: DGB_GENESIS_HASH, prevBlock: '00'.repeat(32), merkleRoot: '',
version: 0, timestamp: 0, bits: 0, nonce: 0, raw: Buffer.alloc(80),
}, 0);
console.log(`[chain] disk-backed, tip ${chain.tipHeight.toLocaleString()} (adopted from ${defaultCacheDir()})`);
let syncing = false;
let syncRound = 0;
// Peers that served an invalid (bad-PoW / future-dated) header. Skipped by
// the sync rotation for a while; their connection is otherwise unaffected.
const badPeers = new Map<string, number>();
const BAD_PEER_MS = 6 * 60 * 60 * 1000;
loadPowWasm();
console.log('[chain] proof-of-work validation on (sha256d, scrypt, skein, qubit, odo, groestl)');
// Header sync. Rotates through ready peers so no single peer decides
// what our chain looks like, and handles forks: when a peer's batch does
// not attach to our tip, find where it does attach, roll back to there
// and re-grow along the peer's branch. Reorgs are logged loudly.
async function backgroundSync(): Promise<void> {
if (syncing) return;
syncing = true;
let before = chain.tipHeight;
let stalled = false;
try {
// Round-robin over every ready peer. (Sorting by advertised height
// was tried and picked exactly the nodes that lie about it and never
// answer getheaders.) The advertised height is still used below as
// a guard: a reorg is only accepted from a peer at least as high as
// our tip.
const now = Date.now();
const ready = pool.pickReady().filter((p) => (badPeers.get(p.host) ?? 0) < now);
const readyPeer = ready[syncRound++ % Math.max(1, ready.length)];
if (!readyPeer) {
syncing = false;
setTimeout(backgroundSync, 2000);
return;
}
before = chain.tipHeight;
stalled = false;
let lastBatchAt = Date.now();
// Walk the peer's chain. Peers answer from the highest locator entry
// they recognise. If we sit on a stale branch, that entry is deep
// below our tip and the first batches are headers we already have;
// we must keep walking UP the peer's chain (locator = last header of
// the batch) until it either diverges from ours (→ reorg candidate)
// or climbs past our tip (→ plain append). Restarting from our own
// tip locator each time would loop forever on the same known batch —
// which is exactly how the daemon sat on a garbage branch for hours.
let cursor: string | null = null; // last header hash of a fully-known batch, when walking up
const ask = (locator: string[]) => readyPeer.send('getheaders', encodeGetHeaders(70019, locator));
// Proof-of-work gate. Every header a peer sends is hashed with the
// algorithm its version names and checked against its own target
// (Core's CheckProofOfWork) before it can touch the chain. Invalid →
// the batch is cut at the first bad header and the peer is benched.
// Timestamps more than 2h in the future are rejected too.
// Work is summed per batch so fork choice below is by accumulated
// proof-of-work, not by header count.
const validate = (hs: BlockHeader[]): { good: BlockHeader[]; bad?: { h: BlockHeader; reason: string } } => {
const limit = Math.floor(Date.now() / 1000) + 2 * 60 * 60;
for (let i = 0; i < hs.length; i++) {
const h = hs[i];
if (h.timestamp > limit) return { good: hs.slice(0, i), bad: { h, reason: 'timestamp too far in the future' } };
const r = checkProofOfWork(h);
if (!r.ok) return { good: hs.slice(0, i), bad: { h, reason: r.reason ?? 'bad pow' } };
}
return { good: hs };
};
const workOf = (hs: BlockHeader[]): bigint => hs.reduce((a, h) => a + workFromBits(h.bits), 0n);
const workOfRange = (from: number, to: number): bigint => {
let w = 0n;
for (let h = from; h <= to; h++) { const x = chain.getByHeight(h); if (x) w += workFromBits(x.bits); }
return w;
};
// A candidate competing branch being accumulated across batches until
// its work beats what it would replace (or it ends and loses).
let pendingBranch: { divergeH: number; headers: BlockHeader[]; droppedWork: bigint } | null = null;
const onHeaders = (headersRaw: BlockHeader[]) => {
lastBatchAt = Date.now();
const t0 = Date.now();
if (headersRaw.length === 0) { stalled = true; return; }
const v = validate(headersRaw);
if (v.bad) {
console.log(`[chain] ${readyPeer.host} sent an INVALID header at ${v.bad.h.hash.slice(0, 16)}… (${v.bad.reason}) — benching peer, keeping ${v.good.length} valid header(s) before it`);
badPeers.set(readyPeer.host, Date.now() + BAD_PEER_MS);
if (v.good.length === 0) { stalled = true; return; }
}
const headers = v.good;
const vMs = Date.now() - t0;
// Continuing a candidate branch from a previous batch?
if (pendingBranch && headers[0].prevBlock === pendingBranch.headers[pendingBranch.headers.length - 1].hash) {
pendingBranch.headers.push(...headers);
} else if (pendingBranch) {
pendingBranch = null; // peer changed subject; drop the candidate
}
let r = pendingBranch ? { appended: 0, rejectedAt: headers[0], reason: 'candidate' } : chain.append(headers);
if (r.rejectedAt && r.appended === 0) {
let divergeH: number;
let branch: BlockHeader[];
if (pendingBranch) {
divergeH = pendingBranch.divergeH; branch = pendingBranch.headers;
} else {
const forkH = chain.findHeight(headers[0].prevBlock);
if (forkH < 0) {
console.log(`[chain] batch from ${readyPeer.host} rejected: ${r.reason?.slice(0, 100)} (fork point not in our chain — next peer)`);
stalled = true; return;
}
// How much of the batch do we already have on our branch?
let known = 0;
while (known < headers.length && chain.getByHeight(forkH + 1 + known)?.hash === headers[known].hash) known++;
if (known === headers.length) {
// Entirely known: keep walking up the peer's chain from here.
cursor = headers[headers.length - 1].hash;
const nextH = forkH + known;
if (nextH >= chain.tipHeight) { stalled = true; return; } // peer is behind us
ask([cursor, DGB_GENESIS_HASH]);
return;
}
divergeH = forkH + known;
branch = headers.slice(known);
pendingBranch = { divergeH, headers: branch, droppedWork: workOfRange(divergeH + 1, chain.tipHeight) };
}
const depth = chain.tipHeight - divergeH;
const branchWork = workOf(branch);
const dropped = pendingBranch!.droppedWork;
if (branchWork <= dropped) {
if (headersRaw.length >= 2000 && !v.bad) {
// Branch may continue — ask for more before judging.
cursor = branch[branch.length - 1].hash;
ask([cursor, DGB_GENESIS_HASH]);
return;
}
console.log(`[chain] ${readyPeer.host} offered a ${branch.length}-header branch at ${divergeH.toLocaleString()} with less work than our ${depth.toLocaleString()}-deep tip (${(Number(branchWork * 1000n / (dropped || 1n)) / 10).toFixed(1)}%) — ignored`);
pendingBranch = null;
stalled = true; return;
}
console.log(`[chain] REORG via ${readyPeer.host}: a branch from ${divergeH.toLocaleString()} carries more work than our tip ${chain.tipHeight.toLocaleString()} (+${((Number(branchWork * 1000n / (dropped || 1n)) / 10) - 100).toFixed(1)}%); rolling back ${depth.toLocaleString()} headers, applying ${branch.length} (validated, ${vMs}ms)`);
chain.rollbackTo(divergeH);
r = chain.append(branch);
pendingBranch = null;
if (chain.tipHeight < before) {
let forgot = 0;
for (const [k, rs] of lastScan) if (rs.endHeight > chain.tipHeight) { lastScan.delete(k); forgot++; }
if (forgot) { saveLastScansSoon(); console.log(`[scan] forgot ${forgot} remembered scan(s) that reached past the fork point`); }
}
}
if (r.rejectedAt) {
console.log(`[chain] batch from ${readyPeer.host} partially applied (+${r.appended}): ${r.reason?.slice(0, 120)}`);
}
cursor = null;
if (v.bad) { stalled = true; return; } // don't ask a benched peer for more
// DGB peers answer with up to 2000 headers (some send more); a short
// batch means we are at their tip.
if (headersRaw.length >= 2000) ask(chain.buildLocator());
else stalled = true;
};
readyPeer.on('headers', onHeaders as never);
readyPeer.send('getheaders', encodeGetHeaders(70019, chain.buildLocator()));
// Poll until caught up, or the peer goes quiet for 20s.
for (let i = 0; i < 600; i++) {
await sleep(1000);
if (stalled) break;
if (Date.now() - lastBatchAt > 30_000) { console.log(`[chain] ${readyPeer.host} (adv. height ${(readyPeer.info?.startHeight ?? 0).toLocaleString()}) stopped answering getheaders — next round uses another peer`); break; }
}
readyPeer.off('headers', onHeaders as never);
if (chain.tipHeight !== before) {
// append() wrote each batch straight to disk — no separate save.
const delta = chain.tipHeight - before;
console.log(`[chain] ${delta >= 0 ? '+' : ''}${delta.toLocaleString()} headers, tip ${chain.tipHeight.toLocaleString()} (${new Date((chain.tip?.timestamp ?? 0) * 1000).toISOString()})`);
}
} finally {
syncing = false;
}
// Re-sync every minute — DGB blocks are 15s, and a fresh tip is what
// makes the P2P balance scan see recent coins. When the last round got
// nothing at all, try the next peer right away instead of idling.
setTimeout(backgroundSync, chain.tipHeight === before && stalled === false ? 5 * 1000 : 60 * 1000);
}
// Kick off after peers likely connected.
setTimeout(backgroundSync, 5000);
// ---- Background filter crawl ------------------------------------------
// Keep the cfilter cache populated for the last CRAWL_BLOCKS blocks (top
// down, newest first) and follow the tip. With the cache warm, a wallet
// balance scan over the default window never waits on a CF peer — it is
// CPU-bound matching, which scan-driver yields around. Override with
// DGB_CRAWL_BLOCKS (env); 0 disables.
const CRAWL_BLOCKS = Number(process.env.DGB_CRAWL_BLOCKS ?? 365 * 5760);
let crawling = false;
const crawlStatus = { from: 0, to: 0, fetched: 0, failedSpans: 0, lastRun: 0, running: false };
function shuffled<T>(arr: T[]): T[] {
const a = [...arr];
for (let i = a.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); [a[i], a[j]] = [a[j], a[i]]; }
return a;
}
async function backgroundCrawl(): Promise<void> {
if (crawling || CRAWL_BLOCKS <= 0) return;
crawling = true;
crawlStatus.running = true;
try {
const tip = chain.tipHeight;
if (tip < 1000 || syncing) return;
const from = Math.max(1, tip - CRAWL_BLOCKS);
const cfPeers = () => shuffled(pool.pickWithServices(NODE_COMPACT_FILTERS));
if (cfPeers().length === 0) return;
crawlStatus.from = from; crawlStatus.to = tip;
const t0 = Date.now();
const r = await crawlFilters({
cfPeers, chain, filterCache, fromHeight: from, toHeight: tip,
filterTimeoutMs: 20_000, pauseMs: 200,
// Wallet scans get the CF peers to themselves; the crawl resumes
// as soon as no scan job is pending.
waitWhile: () => [...scanJobs.values()].some((j) => j.state === 'pending'),
onSpan: (i) => {
if (i.failed) console.log(`[crawl] span ${i.start.toLocaleString()}–${i.end.toLocaleString()}: no CF peer answered`);
else if (i.start % 100_000 < 1000) console.log(`[crawl] at ${i.start.toLocaleString()} (+${i.fetched} filters this span, ${Math.round((Date.now() - t0) / 1000)}s)`);
},
});
crawlStatus.fetched += r.fetched; crawlStatus.failedSpans = r.failedSpans; crawlStatus.lastRun = Date.now();
if (r.fetched > 0 || r.failedSpans > 0) {
console.log(`[crawl] pass done: +${r.fetched.toLocaleString()} filters, ${r.failedSpans} spans failed, window ${from.toLocaleString()}–${tip.toLocaleString()}, ${Math.round((Date.now() - t0) / 1000)}s`);
}
} catch (e) {
console.log(`[crawl] error: ${(e as Error).message}`);
} finally {
crawling = false;
crawlStatus.running = false;
// Follow the tip: a fresh pass every 2 min only fetches what is new.
setTimeout(backgroundCrawl, 2 * 60 * 1000);
}
}
setTimeout(backgroundCrawl, 45_000);
const server = http.createServer(async (req, res) => {
const setCors = () => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'POST, GET, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
};
if (req.method === 'OPTIONS') { setCors(); res.writeHead(204); res.end(); return; }
setCors();
res.setHeader('Content-Type', 'application/json');
try {
if (req.method === 'GET' && req.url?.startsWith('/balance/status')) {
const id = new URL(req.url, 'http://localhost').searchParams.get('job') ?? '';
const job = scanJobs.get(id);
if (!job) { res.writeHead(404); res.end(JSON.stringify({ error: 'unknown job', job: id })); return; }
res.writeHead(200); res.end(scanJobView(job));
return;
}
if (req.method === 'GET' && req.url?.startsWith('/broadcast/status')) {
const txid = new URL(req.url, 'http://localhost').searchParams.get('txid') ?? '';
const job = broadcasts.get(txid);
if (!job) { res.writeHead(404); res.end(JSON.stringify({ error: 'unknown txid', txid })); return; }
res.writeHead(200); res.end(JSON.stringify(jobView(job)));
return;
}
if (req.method === 'GET' && req.url === '/health') {
const peers = pool.pickReady().map((p) => ({
host: p.host, port: p.port, services: p.services.toString(),
hasCF: (p.services & NODE_COMPACT_FILTERS) === NODE_COMPACT_FILTERS,
}));
res.writeHead(200);
res.end(JSON.stringify({
readyPeers: peers.length,
cfPeers: peers.filter((p) => p.hasCF).length,
candidates: pool.candidateCount,
chainHeight: chain.tipHeight,
crawl: crawlStatus,
filterCache: filterCache.summary(),
syncing,
peers,
}));
return;
}
if (req.method === 'GET' && req.url?.startsWith('/height-at')) {
// Resolve a wall-clock time to a block height by binary search
// over the header chain's on-disk timestamps. Used by the wallet
// UI's "since 2019" recovery presets so users pick the year the
// seed was created rather than a raw block number.
//
// GET /height-at?year=2019 → block on/after Jan 1 2019 UTC
// GET /height-at?ts=1546300800 → same, raw unix seconds
const q = new URL(req.url, 'http://localhost').searchParams;
let targetTs: number | null = null;
const y = q.get('year');
const t = q.get('ts');
if (y) {
const year = Number(y);
if (!Number.isInteger(year) || year < 2014 || year > 2100) {
res.writeHead(400); res.end(JSON.stringify({ error: 'year out of range (2014–2100)' })); return;
}
targetTs = Math.floor(Date.UTC(year, 0, 1) / 1000);
} else if (t) {
targetTs = Number(t);
if (!Number.isFinite(targetTs)) { res.writeHead(400); res.end(JSON.stringify({ error: 'ts not a number' })); return; }
} else {
res.writeHead(400); res.end(JSON.stringify({ error: 'expected ?year=YYYY or ?ts=<unix>' })); return;
}
if (chain.tipHeight <= 0) {
res.writeHead(503); res.end(JSON.stringify({ error: 'chain not synced' })); return;
}
// Binary search — headers are ordered by height AND by timestamp
// (block timestamps are monotonic enough for this purpose; small
// occasional out-of-order stamps don't matter since we're picking
// the first block on/after the target).
let lo = 1;
let hi = chain.tipHeight;
while (lo < hi) {
const mid = (lo + hi) >>> 1;
const h = chain.getByHeight(mid);
if (!h) { hi = mid - 1; continue; }
if (h.timestamp < targetTs) lo = mid + 1;
else hi = mid;
}
const found = chain.getByHeight(lo);
res.writeHead(200); res.end(JSON.stringify({
targetTs,
height: lo,
blockHash: found?.hash ?? null,
blockTimestamp: found?.timestamp ?? null,
chainTip: chain.tipHeight,
}));
return;
}
if (req.method === 'GET' && req.url?.startsWith('/tx/')) {
// Fetch a full transaction hex by txid, requiring the caller to
// supply the containing block hash (they get it from /balance's
// UTXO records). We fetch the whole block from a peer and pluck
// the tx out. Without the block hash we'd need a full mempool
// index — which the daemon doesn't run and doesn't want to run.
//
// Used by the Send flow to build BIP44 legacy P2PKH inputs which
// require `nonWitnessUtxo` (the full prev-tx) for signing.
//
// GET /tx/<txid>?blockHash=<hex> → {hex, blockHash, txid}
const parts = req.url.split('?');
const txid = parts[0].slice(4);
const q = new URL(req.url, 'http://localhost').searchParams;
const blockHash = q.get('blockHash');
if (!/^[0-9a-f]{64}$/i.test(txid)) {
res.writeHead(400); res.end(JSON.stringify({ error: 'txid must be 64 hex chars' })); return;
}
if (!blockHash || !/^[0-9a-f]{64}$/i.test(blockHash)) {
res.writeHead(400); res.end(JSON.stringify({ error: 'expected ?blockHash=<64 hex chars>' })); return;
}
const ready = pool.pickReady();
if (ready.length === 0) { res.writeHead(503); res.end(JSON.stringify({ error: 'no ready peer' })); return; }
const peer = ready[0];
try {
const block = await fetchBlockOnce(peer, blockHash, 30_000);
const found = block.transactions?.find((t) => t.getId() === txid);
if (!found) {
res.writeHead(404); res.end(JSON.stringify({ error: `tx ${txid} not found in block ${blockHash}` })); return;
}
res.writeHead(200); res.end(JSON.stringify({
txid,
blockHash,
hex: found.toHex(),
}));
} catch (e) {
res.writeHead(504); res.end(JSON.stringify({ error: (e as Error).message }));
}
return;
}
if ((req.method === 'POST' || req.method === 'GET') && req.url?.match(/^\/broadcast(\?|$)/)) {
// Same POST-with-body / GET-with-?body= duality as /balance.
let bodyRaw: string;
const q = new URL(req.url!, 'http://localhost').searchParams;
if (req.method === 'POST') {
bodyRaw = await readBody(req);
} else {
const b = q.get('body');
if (!b) { res.writeHead(400); res.end(JSON.stringify({ error: 'GET /broadcast needs ?body=<base64url-encoded {hex:...}>' })); return; }
try { bodyRaw = Buffer.from(b, 'base64url').toString('utf8'); }
catch (e) { res.writeHead(400); res.end(JSON.stringify({ error: `body= not valid base64url: ${(e as Error).message}` })); return; }
}
const { hex } = JSON.parse(bodyRaw);
if (typeof hex !== 'string' || !/^[0-9a-fA-F]+$/.test(hex)) {
res.writeHead(400); res.end(JSON.stringify({ error: 'expected {hex: "..."} hex-only' })); return;
}
const ready = pool.pickReady();
if (ready.length < 3) { res.writeHead(503); res.end(JSON.stringify({ error: `only ${ready.length} peers ready, need 3` })); return; }
// Dandelion++ stem-phase relay is on by default (privacy-first).
// Pass ?dandelion=0 to force the direct fan-out path.
//
// Announce to at most 8 peers; watch the WHOLE ready pool for the
// tx to be announced back by a peer we did not hand it to. That
// echo is the only proof of mempool acceptance available over
// P2P (Core drops policy failures silently). `relayed: false` in
// the response means "not known to be accepted" — the wallet
// treats it as a failed send.
const useDandelion = q.get('dandelion') !== '0';
const announceTo = ready.slice(0, 8);
// The full dance (getdata wait + relay-echo wait) can run 20-45s.
// Reverse proxies in front of us — notably the navigate.st BNS
// gateway — give up around 15s and hand the wallet a 502 for a
// send that then SUCCEEDS in the background. So: run the job
// detached, keyed by txid, and reply as soon as it finishes OR
// after EARLY_REPLY_MS with {pending:true}; the wallet polls
// /broadcast/status. A repeat POST for the same txid (wallet retry
// after a proxy error) attaches to the running job instead of
// announcing the tx a second time.
let txid: string;
try { txid = Transaction.fromHex(hex).getId(); }
catch (e) { res.writeHead(400); res.end(JSON.stringify({ error: `bad tx hex: ${(e as Error).message}` })); return; }
pruneBroadcasts();
let job = broadcasts.get(txid);
if (!job) {
const t0 = Date.now();
const promise = useDandelion
? broadcastDandelion(announceTo, hex, { listeners: ready, relayWaitMs: 30_000 })
: broadcastTx(announceTo, hex, { listeners: ready, relayWaitMs: 20_000 }).then((r) => ({ ...r, via: 'direct' as const }));
job = { txid, startedAt: t0, state: 'pending', via: useDandelion ? 'dandelion' : 'direct' };
broadcasts.set(txid, job);
promise.then((result) => {
job!.state = 'done'; job!.result = result; job!.finishedAt = Date.now();
console.log(
`[broadcast] ${result.txid} via=${result.via}${'stemPeer' in result && result.stemPeer ? `(${result.stemPeer})` : ''} ` +
`requestedBy=${result.requestedBy}/${result.totalPeers} relayed=${result.relayed}${result.relayedBy ? ` by ${result.relayedBy}` : ''} ` +
`listeners=${result.listeners} rejects=${result.rejects.length} ${Date.now() - t0}ms`,
);
}).catch((e: Error) => {
job!.state = 'failed'; job!.error = e.message; job!.finishedAt = Date.now();
console.log(`[broadcast] ${txid} FAILED after ${Date.now() - t0}ms: ${e.message}`);
});
job.promise = promise.then(() => undefined, () => undefined);
}
await Promise.race([job.promise, sleep(EARLY_REPLY_MS)]);
res.writeHead(200); res.end(JSON.stringify(jobView(job)));
return;
}
if ((req.method === 'POST' || req.method === 'GET') && req.url?.match(/^\/balance(\?|$)/)) {
// Accepts POST with JSON body OR GET with ?body=<base64url JSON>.
// The GET path is a compat fallback for browsers/clients that
// downgrade POST-with-body to GET (Theseus BNS-resolved fetch
// does this empirically) and thereby lose the body.
let raw: string;
if (req.method === 'POST') {
raw = await readBody(req);
} else {
const q = new URL(req.url!, 'http://localhost').searchParams;
const b = q.get('body');
if (!b) {
res.writeHead(400);
res.end(JSON.stringify({ error: 'GET /balance needs ?body=<base64url-encoded JSON> — the same object POST would carry' }));
return;
}
try {
raw = Buffer.from(b, 'base64url').toString('utf8');
} catch (e) {
res.writeHead(400); res.end(JSON.stringify({ error: `body= not valid base64url: ${(e as Error).message}` })); return;
}
}
const body = JSON.parse(raw) as {
xpub?: string;
purpose?: 44 | 49 | 84 | 86;
account?: number;
descriptors?: Array<{ xpub: string; purpose: 44 | 49 | 84 | 86; account?: number }>;
// Raw scriptPubKey hex list. Used for WIF-import scans where
// the client already knows the exact scripts to watch and there
// are no xpub-derived children to enumerate.
scripts?: string[];
gapLimit?: number;
gapInternal?: number;
startHeight?: number;
endHeight?: number;
};
const descriptors = body.descriptors
?? (typeof body.xpub === 'string'
? [{ xpub: body.xpub, purpose: body.purpose ?? 84, account: body.account ?? 0 }]
: null);
const rawScripts = Array.isArray(body.scripts)
? body.scripts.filter((s): s is string => typeof s === 'string' && /^[0-9a-f]+$/i.test(s))
: [];
if ((!descriptors || descriptors.length === 0) && rawScripts.length === 0) {
res.writeHead(400);
res.end(JSON.stringify({ error: 'expected {xpub}, {descriptors:[...]}, or {scripts:[hex,…]}' }));
return;
}
// Receive-chain gap (default 20, cap 200) and change-chain gap
// (defaults to the receive gap). 200/100 matches what other DGB
// recovery wallets probe, so a sparse old wallet is not missed.
const gapLimit = Math.min(body.gapLimit ?? 20, 200);
const gapInternal = Math.min(body.gapInternal ?? gapLimit, 200);
const DEFAULT_WINDOW = 1000;
const rawStart = body.startHeight ?? Math.max(1, chain.tipHeight - DEFAULT_WINDOW);
const startHeight = Math.max(1, rawStart);
const endHeight = Math.min(chain.tipHeight, body.endHeight ?? chain.tipHeight);
if (endHeight < startHeight) {
res.writeHead(503); res.end(JSON.stringify({ error: `chain not synced enough (tip ${endHeight})` })); return;
}
const cfPeers = pool.pickWithServices(NODE_COMPACT_FILTERS);
if (cfPeers.length === 0) {
res.writeHead(503); res.end(JSON.stringify({ error: 'no CF-serving peer ready' })); return;
}
// Shuffle so a hot peer doesn't get pummeled and we spread load.
for (let i = cfPeers.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[cfPeers[i], cfPeers[j]] = [cfPeers[j], cfPeers[i]];
}
// Merge all descriptors' scriptPubKeys into one watch set. Same
// scan work regardless of descriptor count — filter probing is
// O(watched-scripts × filter-bits), block extract is O(tx-outs).
// The daemon does the whole recovery sweep in one filter pass.
const scriptBuffers: Buffer[] = [];
const watchScripts = new Set<string>();
const perScript = new Map<string, { purpose: number; account: number; change: 0 | 1; index: number } | { kind: 'wif' }>();
if (descriptors) {
for (const d of descriptors) {
const account = bip32.fromBase58(d.xpub, digibyte);
for (const change of [0, 1] as const) {
const chainNode = account.derive(change);
for (let i = 0; i < (change === 0 ? gapLimit : gapInternal); i++) {
const node = chainNode.derive(i);
const pubkey = Buffer.from(node.publicKey);
const spk = spkForPurpose(d.purpose, pubkey);
const hex = spk.toString('hex');
if (watchScripts.has(hex)) continue; // dedupe across descriptors
scriptBuffers.push(spk);
watchScripts.add(hex);
perScript.set(hex, {
purpose: d.purpose, account: d.account ?? 0, change, index: i,
});
}
}
}
}
for (const hex of rawScripts) {
const lower = hex.toLowerCase();
if (watchScripts.has(lower)) continue;
scriptBuffers.push(Buffer.from(lower, 'hex'));
watchScripts.add(lower);
perScript.set(lower, { kind: 'wif' });
}
// Filters from CF peers with per-span failover; blocks from FULL
// nodes (pruned peers ignore getdata for anything older than ~288
// blocks — the old "blocks=0/3 … timed out" failure). See
// scan-driver.ts.
//
// Scans run as JOBS, like broadcasts: the request returns within
// EARLY_REPLY_MS with either the finished result or {pending:true,
// job} for the wallet to poll at /balance/status. Two reasons:
// proxies in front of us time out at ~15s, and a one-year window
// is minutes of filter matching even from cache.
//
// Results are remembered per script-set. A later request for the
// same scripts only scans the blocks above the previous result
// and merges (spends of previously-found coins are detected via
// the owned-seed), so a periodic refresh is a few new blocks.
const key = createHash('sha256').update([...watchScripts].sort().join(',')).digest('hex').slice(0, 24);
const jobId = `${key}:${startHeight}:${endHeight}`;
pruneScanJobs();
let job = scanJobs.get(jobId);
// A page reload re-requests the same scripts a few blocks later
// (new endHeight → new job id). Attach to the scan already running
// for this script set instead of starting a competing copy: the
// wallet gets that result, and its next refresh is incremental.
if (!job) {
for (const j of scanJobs.values()) {
if (j.state === 'pending' && j.key === key && j.startHeight <= startHeight) { job = j; break; }
}
}
if (!job) {
const prev = lastScan.get(key);
// Incremental when the previous result covers our start and is
// not ahead of us (a narrower re-scan after a wider one is fine:
// the previous coins are all inside the requested window's past).
const incremental = !!prev && prev.startHeight <= startHeight && prev.endHeight < endHeight && prev.endHeight >= startHeight - 1;
const scanFrom = incremental ? prev!.endHeight + 1 : startHeight;
const blockPeers = shuffled(pool.pickReady().filter((p) =>
(p.services & NODE_NETWORK) === NODE_NETWORK && (p.services & NODE_NETWORK_LIMITED) === 0n));
const blockPool = blockPeers.length > 0 ? blockPeers : pool.pickReady();
const t0 = Date.now();
job = { id: jobId, key, startedAt: t0, state: 'pending', startHeight, endHeight, scanFrom, progress: scanFrom - 1, incremental };
scanJobs.set(jobId, job);
const perScriptSnapshot = perScript;
const run = (async () => {
const r = await scanWithPeers({
cfPeers, blockPeers: blockPool, chain, startHeight: scanFrom, endHeight,
watchScripts, scriptBuffers, filterCache,
filterTimeoutMs: 12_000, blockTimeoutMs: 15_000,
ownedSeed: incremental ? prev!.utxos.map((u) => `${u.txid}:${u.vout}`) : undefined,
onProgress: (p) => {
job!.progress = p.height;
if ((p.height - scanFrom) % 100_000 < 1000) {
console.log(`[scan] ${key.slice(0, 8)} … ${p.height.toLocaleString()}/${endHeight.toLocaleString()} cache=${p.fromCache} net=${p.fromNetwork} hits=${p.hits} ${Date.now() - t0}ms`);
}
},
});
// Merge with the previous result in incremental mode.
const spentSet = new Set(r.spent);
const base = incremental ? prev!.utxos.filter((u) => !spentSet.has(`${u.txid}:${u.vout}`)) : [];
const utxos = [...base, ...r.utxos.map((u) => ({
txid: u.txid, vout: u.vout, value: Number(u.value),
scriptPubKey: u.scriptPubKey.toString('hex'), blockHash: u.blockHash,
}))];
const totalSats = utxos.reduce((a, u) => a + u.value, 0);
const result = {
totalSats,
utxos: utxos.map((u) => ({ ...u, path: perScriptSnapshot.get(u.scriptPubKey) ?? null })),
scannedRange: { startHeight, endHeight },
incremental, scannedFrom: scanFrom,
blocksFetched: r.blocksFetched, filterHits: r.filterHits,
filtersFromCache: r.filtersFromCache, filtersFromNetwork: r.filtersFromNetwork,
elapsedMs: Date.now() - t0,
descriptorsScanned: descriptors?.length ?? 0,
watchedScripts: watchScripts.size,
};
lastScan.set(key, { startHeight, endHeight, utxos, at: Date.now() });
saveLastScansSoon();
console.log(`[scan] ${key.slice(0, 8)} ok${incremental ? ' (incremental from ' + scanFrom.toLocaleString() + ')' : ''}: ${utxos.length} utxos, ${totalSats} sat, ${r.filtersFromCache}c/${r.filtersFromNetwork}n filters, ${r.blocksFetched} blocks, ${Date.now() - t0}ms`);
return result;
})();
run.then((result) => { job!.state = 'done'; job!.result = result; job!.finishedAt = Date.now(); })
.catch((e: Error) => {
job!.state = 'failed'; job!.error = e.message; job!.finishedAt = Date.now();
console.log(`[scan] ${key.slice(0, 8)} ${scanFrom}-${endHeight} failed after ${Date.now() - t0}ms: ${e.message.slice(0, 160)}`);
});
job.promise = run.then(() => undefined, () => undefined);
}
await Promise.race([job.promise, sleep(EARLY_REPLY_MS)]);
res.writeHead(200);
res.end(scanJobView(job));
return;
}
res.writeHead(404);
res.end(JSON.stringify({ error: 'not found', endpoints: ['/health', 'POST /broadcast', 'POST /balance'] }));
} catch (e) {
// Never let a handler error take the whole daemon down.
console.log(`[http] ${req.method} ${req.url?.split('?')[0]} failed: ${(e as Error).stack ?? (e as Error).message}`);
if (res.headersSent) { try { res.end(); } catch { /* socket gone */ } return; }
res.writeHead(500);
res.end(JSON.stringify({ error: (e as Error).message }));
}
});
// Last-resort guards: log and keep serving rather than exit on a stray
// rejection from a peer handler.
process.on('unhandledRejection', (e) => console.log(`[daemon] unhandled rejection: ${(e as Error)?.stack ?? String(e)}`));
process.on('uncaughtException', (e) => console.log(`[daemon] uncaught exception: ${e.stack ?? e.message}`));
// Request a single block from `peer` by hash, resolve when it arrives
// (or reject on timeout). Detaches its listener afterwards so it does
// not leak across concurrent /tx/ requests.
function fetchBlockOnce(peer: import('../src/peer.js').Peer, blockHash: string, timeoutMs: number): Promise<Block> {
return new Promise((resolve, reject) => {
const onBlock = (b: Block) => {
if (b.getId() !== blockHash) return; // not ours; keep waiting
cleanup();
resolve(b);
};
const cleanup = () => {
clearTimeout(timer);
peer.off('block', onBlock as never);
};
const timer = setTimeout(() => {
cleanup();
reject(new Error(`fetchBlockOnce: no block ${blockHash} in ${timeoutMs}ms`));
}, timeoutMs);
peer.on('block', onBlock as never);
peer.send('getdata', encodeGetBlock(blockHash));
});
}
function spkForPurpose(purpose: 44 | 49 | 84 | 86, pubkey: Buffer): Buffer {
switch (purpose) {
case 44:
return Buffer.from(payments.p2pkh({ pubkey, network: digibyte }).output!);
case 49: {
const redeem = payments.p2wpkh({ pubkey, network: digibyte });
return Buffer.from(payments.p2sh({ redeem, network: digibyte }).output!);
}
case 84:
return Buffer.from(payments.p2wpkh({ pubkey, network: digibyte }).output!);
case 86:
return Buffer.from(payments.p2tr({ internalPubkey: pubkey.subarray(1, 33), network: digibyte }).output!);
}
}
server.listen(PORT, '127.0.0.1', () => {
console.log(`[daemon] http://127.0.0.1:${PORT}`);
console.log(`[daemon] GET /health`);
console.log(`[daemon] POST /broadcast {"hex": "..."}`);
console.log(`[daemon] POST /balance {"xpub":"...","purpose":84,"account":0,"gapLimit":20,"startHeight":...}`);
});
async function readBody(req: http.IncomingMessage): Promise<string> {
const chunks: Buffer[] = [];
for await (const c of req) chunks.push(c as Buffer);
return Buffer.concat(chunks).toString('utf8');
}
function sleep(ms: number): Promise<void> { return new Promise((r) => setTimeout(r, ms)); }
// ---- scan job registry + per-script-set memory (see /balance handler) ----
interface ScanJob {
id: string; key: string; startedAt: number; finishedAt?: number;
state: 'pending' | 'done' | 'failed';
startHeight: number; endHeight: number; scanFrom: number; progress: number; incremental: boolean;
result?: unknown; error?: string; promise?: Promise<void>;
}
interface RememberedScan {
startHeight: number; endHeight: number; at: number;
utxos: Array<{ txid: string; vout: number; value: number; scriptPubKey: string; blockHash: string }>;
}
const scanJobs = new Map<string, ScanJob>();
const lastScan = new Map<string, RememberedScan>();
const SCAN_JOB_TTL_MS = 30 * 60_000;
// Remembered results are what make refreshes incremental. Keep them for
// a month (a wallet that has not been opened for a month re-scans its
// window once) and PERSIST them, so a daemon restart does not send every
// wallet back to a from-scratch scan. Nothing sensitive is stored: only
// a hash of the watched scripts, the scanned range, and the found coins
// (txid/vout/value/script) — the same data any block explorer shows.
const LAST_SCAN_TTL_MS = 30 * 24 * 60 * 60_000;
const LAST_SCAN_PATH = join(defaultCacheDir(), 'scans.json');
function loadLastScans(): void {
try {
if (!existsSync(LAST_SCAN_PATH)) return;
const raw = JSON.parse(readFileSync(LAST_SCAN_PATH, 'utf8')) as Record<string, RememberedScan>;
const now = Date.now();
for (const [k, r] of Object.entries(raw)) {
if (r && typeof r.endHeight === 'number' && Array.isArray(r.utxos) && now - r.at <= LAST_SCAN_TTL_MS) lastScan.set(k, r);
}
console.log(`[scan] restored ${lastScan.size} remembered wallet scan(s) from ${LAST_SCAN_PATH}`);
} catch (e) {
console.log(`[scan] could not restore remembered scans: ${(e as Error).message}`);
}
}
let saveTimer: ReturnType<typeof setTimeout> | null = null;
function saveLastScansSoon(): void {
if (saveTimer) return;
saveTimer = setTimeout(() => {
saveTimer = null;
try {
const tmp = LAST_SCAN_PATH + '.tmp';
writeFileSync(tmp, JSON.stringify(Object.fromEntries(lastScan)));
renameSync(tmp, LAST_SCAN_PATH);
} catch (e) {
console.log(`[scan] could not persist remembered scans: ${(e as Error).message}`);
}
}, 1000);
}
function pruneScanJobs(): void {
const now = Date.now();
for (const [k, j] of scanJobs) if (j.state !== 'pending' && now - j.startedAt > SCAN_JOB_TTL_MS) scanJobs.delete(k);
let dropped = false;
for (const [k, r] of lastScan) if (now - r.at > LAST_SCAN_TTL_MS) { lastScan.delete(k); dropped = true; }
if (dropped) saveLastScansSoon();
}
loadLastScans();
function scanJobView(j: ScanJob): string {
const elapsedMs = (j.finishedAt ?? Date.now()) - j.startedAt;
if (j.state === 'done') return JSON.stringify({ ...(j.result as object), state: 'done', pending: false, job: j.id, elapsedMs });
if (j.state === 'failed') return JSON.stringify({ state: 'failed', pending: false, job: j.id, error: j.error, elapsedMs });
const total = Math.max(1, j.endHeight - j.scanFrom + 1);
const done = Math.max(0, j.progress - j.scanFrom + 1);
return JSON.stringify({
state: 'pending', pending: true, job: j.id, elapsedMs,
progress: { height: j.progress, from: j.scanFrom, to: j.endHeight, percent: Math.min(100, Math.round(100 * done / total)) },
incremental: j.incremental,
});
}
// ---- broadcast job registry (see /broadcast handler) ----
const EARLY_REPLY_MS = 6_000; // well under any proxy's ~15s timeout
const BROADCAST_TTL_MS = 15 * 60_000;
interface BroadcastJob {
txid: string;
startedAt: number;
finishedAt?: number;
state: 'pending' | 'done' | 'failed';
via: 'dandelion' | 'direct';
result?: Awaited<ReturnType<typeof broadcastDandelion>>;
error?: string;
promise?: Promise<void>;
}
const broadcasts = new Map<string, BroadcastJob>();
function pruneBroadcasts(): void {
const now = Date.now();
for (const [k, j] of broadcasts) if (now - j.startedAt > BROADCAST_TTL_MS) broadcasts.delete(k);
}
function jobView(j: BroadcastJob): Record<string, unknown> {
if (j.state === 'done' && j.result) return { ...j.result, state: 'done', pending: false, elapsedMs: (j.finishedAt ?? Date.now()) - j.startedAt };
if (j.state === 'failed') return { txid: j.txid, state: 'failed', pending: false, relayed: false, error: j.error, elapsedMs: (j.finishedAt ?? Date.now()) - j.startedAt };
return { txid: j.txid, state: 'pending', pending: true, via: j.via, elapsedMs: Date.now() - j.startedAt };
}
for (const sig of ['SIGINT', 'SIGTERM'] as const) {
process.on(sig, () => {
console.log(`\n[daemon] ${sig} — shutting down`);
server.close();
pool.stop();
chain.close();
process.exit(0);
});
}
// Reference kept so it's not stripped by dead-code elimination
void DGB_COIN_TYPE;