p2p: BIP158 matching in WebAssembly (gcs.c), Number-based JS fallback; Blockbook upstream → Atomic

The daemon's wallet scan hashed every watched script against every block
filter with a BigInt SipHash: ~3 filters/s on the VPS once the gap limits
went to 200/100 (~3,300 scripts), i.e. days for a one-year window. The
matcher now lives in dgbpow.wasm next to the PoW hashes — scripts are
uploaded once per scan, each filter is one call, the set is decoded once
and each target binary-searched — ~5,500 filters/s locally, ~2,000/s on
the VPS with block fetches. The JS path was rewritten with uint32 lanes
and exact 32×32 partial products as a fallback (module missing or filter
beyond its buffers). scripts/gcs-check.ts cross-checks both against a
BigInt reference on random filters; `npm run gcs:test -w @dgb-wallet/p2p`.

NOWNodes stopped accepting the proxy's API key ("Unknown API_key" on every
endpoint), which is why the wallet showed "Blockbook HTTP 401" for every
path. The vhost mirror now proxies /blockbook/ to Atomic Wallet's public
Blockbook; ops/blockbook-upstream.sh switches the live vhost either way.
This commit is contained in:
Digibyte.X 2026-10-11 23:37:28 +02:00
parent 93cd8b4bf4
commit 690959a983
11 changed files with 604 additions and 145 deletions

View file

@ -7,7 +7,7 @@ web wallet needs:
| Piece | Where | Notes |
|---|---|---|
| nginx vhost | `/etc/nginx/sites-enabled/digibyte-x` — mirrored in [`nginx-digibyte-x.conf`](nginx-digibyte-x.conf) | SNI-selected on `digibyte.x`, self-signed cert pinned on-chain via the BNS `tls` record. Serves `/web/` (wallet), proxies `/p2p/*` → daemon and `/blockbook/*` → NOWNodes (API key injected server-side from `/etc/dgb-core/nownodes.env`), strict Content-Security-Policy on `/web/`. |
| nginx vhost | `/etc/nginx/sites-enabled/digibyte-x` — mirrored in [`nginx-digibyte-x.conf`](nginx-digibyte-x.conf) | SNI-selected on `digibyte.x`, self-signed cert pinned on-chain via the BNS `tls` record. Serves `/web/` (wallet), proxies `/p2p/*` → daemon and `/blockbook/*` → a public DGB Blockbook (Atomic Wallet's since 2026-10-11; NOWNodes before that, API key injected server-side from `/etc/dgb-core/nownodes.env` — switch with [`blockbook-upstream.sh`](blockbook-upstream.sh)), strict Content-Security-Policy on `/web/`. |
| `dgb-daemon.service` | `/opt/dgb-wallet`, runs `packages/p2p/bin/dgb-daemon.ts` via tsx, binds `127.0.0.1:5174` | The wallet's only server-side component: header sync with reorg handling, BIP157/158 filter cache (packed 1000/file under `/var/lib/dgb-wallet/.dgb-wallet/p2p/packs/`), incremental balance scans remembered in `scans.json`, Dandelion++ broadcast with relay-echo verification. |
| **No DigiByte Core, no Fulcrum** | — | Removed 2026-09-06. A full node with indexes needs ≥40 GB disk and ≥4 GB RAM; the VPS has neither, and the daemon does not use it. |
@ -125,6 +125,7 @@ If you'd rather understand each piece, see [MANUAL.md](MANUAL.md) — same steps
|---|---|
| `push.sh` | **Routine deploy from the dev box** — build, ship web bundle and/or daemon sources, restart, verify (see Production layout) |
| `nginx-digibyte-x.conf` | Mirror of the live nginx vhost for digibyte.x (`@NOWNODES_API_KEY@` placeholder; render with `sed` as the file header shows) |
| `blockbook-upstream.sh` | `atomic` or `nownodes` — repoint the live `/blockbook/` proxy, test, reload, verify |
| `setup.sh` | RPC-VPS bootstrap (Core + Caddy in front of it) — optional Core backend, not deployed |
| `deploy-wallet.sh` | Wallet-site VPS bootstrap (web bundle at `/` + P2P daemon reverse-proxy) |
| `deploy-wallet-web.sh` | Wallet-subpath bootstrap — plain production build (relative `./` base), ships bundle to `<SITE_ROOT>/web/`, renders Caddy import snippet |

50
ops/blockbook-upstream.sh Normal file
View file

@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Point the live digibyte.x vhost's /blockbook/ proxy at a Blockbook
# upstream, test the config, reload nginx and prove the route works.
#
# ops/blockbook-upstream.sh atomic # Atomic Wallet's public Blockbook (no key)
# ops/blockbook-upstream.sh nownodes # NOWNodes, key from /etc/dgb-core/nownodes.env on the VPS
#
# Runs over `ssh silentmode`; the previous vhost is kept under
# /etc/nginx/sites-enabled-stale-baks/. The repo mirror
# ops/nginx-digibyte-x.conf documents the same two blocks.
set -euo pipefail
target="${1:-}"
case "$target" in
atomic) host=digibyte.atomicwallet.io; keyhdr=0 ;;
nownodes) host=dgbbook.nownodes.io; keyhdr=1 ;;
*) echo "usage: $0 atomic|nownodes" >&2; exit 2 ;;
esac
ssh silentmode "HOST=$host KEYHDR=$keyhdr bash -s" <<'REMOTE'
set -euo pipefail
F=/etc/nginx/sites-enabled/digibyte-x
mkdir -p /etc/nginx/sites-enabled-stale-baks
cp "$F" "/etc/nginx/sites-enabled-stale-baks/digibyte-x.bak.$(date +%s)"
KEY=""
if [ "$KEYHDR" = 1 ]; then
KEY="$(. /etc/dgb-core/nownodes.env; printf %s "$NOWNODES_API_KEY" | tr -d '\r\n ')"
[ -n "$KEY" ] || { echo "no NOWNODES_API_KEY in /etc/dgb-core/nownodes.env" >&2; exit 1; }
fi
HOST="$HOST" KEY="$KEY" python3 - "$F" <<'PY'
import os, re, sys
p = sys.argv[1]; s = open(p).read()
host, key = os.environ['HOST'], os.environ['KEY']
a = s.index(' location /blockbook/ {'); b = s.index('\n }\n', a) + 7
blk = s[a:b]
blk = re.sub(r'proxy_pass https://[^/]+/api/v2/;', f'proxy_pass https://{host}/api/v2/;', blk)
blk = re.sub(r'proxy_ssl_name [^;]+;', f'proxy_ssl_name {host};', blk)
blk = re.sub(r'proxy_set_header Host [^;]+;', f'proxy_set_header Host {host};', blk)
blk = re.sub(r'\n proxy_set_header api-key "[^"]*";', '', blk)
if key:
blk = blk.replace(' proxy_set_header X-Real-IP "";\n', f' proxy_set_header X-Real-IP "";\n proxy_set_header api-key "{key}";\n')
open(p, 'w').write(s[:a] + blk + s[b:])
print('vhost now proxies /blockbook/ ->', host, '(api-key header)' if key else '(no key)')
PY
nginx -t
systemctl reload nginx
sleep 1
code=$(curl -sk -o /dev/null -w '%{http_code}' --resolve digibyte.x:443:127.0.0.1 'https://digibyte.x/blockbook/')
echo "GET /blockbook/ via vhost -> HTTP $code"
[ "$code" = 200 ] || { echo "upstream not answering 200 — check the key / host" >&2; exit 1; }
REMOTE

View file

@ -97,19 +97,25 @@ server {
# individual browsers. (This VPS operator still can — the
# wallet's Blockbook backend toggle carries an inline note.)
location /blockbook/ {
# Upstream: NOWNodes DGB Blockbook (fully synced; api-key
# injected server-side so the browser never sees the credential).
# Same posture as Atomic — proxy hides individual user IPs from
# the upstream operator. Both Atomic and NOWNodes' Blockbook
# fork rejects raw bech32 address queries; `/xpub` works for
# both (server-side derivation covers all script kinds).
proxy_pass https://dgbbook.nownodes.io/api/v2/;
# Upstream: Atomic Wallet's public DGB Blockbook. The proxy hides
# individual user IPs from the upstream operator. Every public DGB
# Blockbook rejects raw bech32 address queries; `/xpub` works
# (server-side derivation covers all script kinds).
#
# NOWNodes (dgbbook.nownodes.io, api-key injected here from
# /etc/dgb-core/nownodes.env) was the upstream 2026-09-02..10-11
# until the key stopped being accepted ("Unknown API_key" on every
# endpoint). To switch back with a fresh key:
# proxy_pass https://dgbbook.nownodes.io/api/v2/;
# proxy_ssl_name dgbbook.nownodes.io;
# proxy_set_header Host dgbbook.nownodes.io;
# proxy_set_header api-key "@NOWNODES_API_KEY@";
proxy_pass https://digibyte.atomicwallet.io/api/v2/;
proxy_ssl_server_name on;
proxy_ssl_name dgbbook.nownodes.io;
proxy_set_header Host dgbbook.nownodes.io;
proxy_ssl_name digibyte.atomicwallet.io;
proxy_set_header Host digibyte.atomicwallet.io;
proxy_set_header X-Forwarded-For "";
proxy_set_header X-Real-IP "";
proxy_set_header api-key "@NOWNODES_API_KEY@";
proxy_http_version 1.1;
proxy_read_timeout 60s;
proxy_hide_header Access-Control-Allow-Origin;

View file

@ -14,10 +14,10 @@ MSYS_NO_PATHCONV=1 docker run --rm -v "$here":/work -w /work "$IMG" bash -c '
set -e
emcc -O3 -DNDEBUG -fno-exceptions -fno-rtti \
-s STANDALONE_WASM=1 -s PURE_WASI=0 --no-entry \
-s EXPORTED_FUNCTIONS=_in_ptr,_out_ptr,_hash_qubit,_hash_skein512,_hash_groestl512,_hash_odo \
-s INITIAL_MEMORY=1048576 -s ALLOW_MEMORY_GROWTH=0 -s STACK_SIZE=262144 \
-s EXPORTED_FUNCTIONS=_in_ptr,_out_ptr,_hash_qubit,_hash_skein512,_hash_groestl512,_hash_odo,_gcs_scripts_ptr,_gcs_offsets_ptr,_gcs_key_ptr,_gcs_filter_ptr,_gcs_matches_ptr,_gcs_limits_ptr,_gcs_set_count,_gcs_match \
-s INITIAL_MEMORY=33554432 -s ALLOW_MEMORY_GROWTH=0 -s STACK_SIZE=262144 \
-Isrc \
dgbpow.c odo_glue.cpp \
dgbpow.c gcs.c odo_glue.cpp \
src/luffa.cpp src/cubehash.cpp src/shavite.cpp src/simd.cpp src/echo.cpp \
src/skein.cpp src/groestl.cpp src/odocrypt.cpp src/KeccakP-800-reference.cpp \
-o dgbpow.wasm

Binary file not shown.

144
packages/p2p/native/gcs.c Normal file
View file

@ -0,0 +1,144 @@
// BIP158 compact-filter matching, compiled to WebAssembly alongside the
// proof-of-work hashes (same module, dgbpow.wasm).
//
// Why native: a wallet scan tests every watched script (a few thousand
// with gap limits 200/100 over ~10 derivation profiles) against every
// block filter, and the SipHash key is per block, so nothing can be
// precomputed across blocks. In JavaScript that is ~3 µs per script
// even with int32 lanes; here it is well under 100 ns, which turns a
// one-year scan (2.1M DigiByte blocks) from hours into minutes.
//
// Interface (pointer-free for the caller, fixed static buffers):
// gcs_scripts_ptr() → byte area the JS side packs the scripts into
// gcs_offsets_ptr() → uint32[n+1] start offsets of each script
// gcs_set_count(n) → declares how many scripts are uploaded
// gcs_key_ptr() → 16-byte SipHash key (block hash LE, first 16 bytes)
// gcs_filter_ptr() → the GCS body (after the varint N), ≤ GCS_FILTER_BYTES
// gcs_match(n_set, body_len, want_all)
// → number of matching scripts (0 = none); with want_all=0 stops
// at the first match. Matching script indices are written to
// gcs_matches_ptr() (uint32[]), in no particular order.
// gcs_limits_ptr() → uint32[3] { max scripts, script bytes, filter bytes }
//
// Build: see build.sh (emscripten in docker).
#include <stdint.h>
#include <string.h>
#define EXPORT __attribute__((used, visibility("default")))
#define GCS_MAX_SCRIPTS 65536
#define GCS_SCRIPT_BYTES (4u * 1024 * 1024)
#define GCS_FILTER_BYTES (2u * 1024 * 1024)
#define GCS_SET_CAP (1u << 20) /* decoded set values kept for binary search */
#define GCS_P 19
#define GCS_M 784931ull
static uint8_t g_scripts[GCS_SCRIPT_BYTES];
static uint32_t g_offsets[GCS_MAX_SCRIPTS + 1];
static uint32_t g_count = 0;
static uint8_t g_key[16];
static uint8_t g_filter[GCS_FILTER_BYTES];
static uint32_t g_matches[GCS_MAX_SCRIPTS];
static uint64_t g_set[GCS_SET_CAP];
static uint32_t g_limits[3] = { GCS_MAX_SCRIPTS, GCS_SCRIPT_BYTES, GCS_FILTER_BYTES };
EXPORT uint8_t* gcs_scripts_ptr(void) { return g_scripts; }
EXPORT uint32_t* gcs_offsets_ptr(void) { return g_offsets; }
EXPORT uint8_t* gcs_key_ptr(void) { return g_key; }
EXPORT uint8_t* gcs_filter_ptr(void) { return g_filter; }
EXPORT uint32_t* gcs_matches_ptr(void) { return g_matches; }
EXPORT uint32_t* gcs_limits_ptr(void) { return g_limits; }
EXPORT void gcs_set_count(uint32_t n) { g_count = n > GCS_MAX_SCRIPTS ? GCS_MAX_SCRIPTS : n; }
// ---- SipHash-2-4 (Bitcoin Core src/crypto/siphash.cpp semantics) ----
#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
#define SIPROUND do { \
v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; v0 = ROTL(v0, 32); \
v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \
v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \
v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; v2 = ROTL(v2, 32); \
} while (0)
static inline uint64_t rd64(const uint8_t* p) {
uint64_t v; memcpy(&v, p, 8); return v; // wasm is little-endian
}
static inline uint64_t siphash24(uint64_t k0, uint64_t k1, const uint8_t* m, uint32_t len) {
uint64_t v0 = k0 ^ 0x736f6d6570736575ull;
uint64_t v1 = k1 ^ 0x646f72616e646f6dull;
uint64_t v2 = k0 ^ 0x6c7967656e657261ull;
uint64_t v3 = k1 ^ 0x7465646279746573ull;
uint32_t full = len & ~7u, i;
for (i = 0; i < full; i += 8) {
uint64_t w = rd64(m + i);
v3 ^= w; SIPROUND; SIPROUND; v0 ^= w;
}
uint64_t t = ((uint64_t)(len & 0xff)) << 56;
switch (len & 7) {
case 7: t |= ((uint64_t)m[i + 6]) << 48; /* fallthrough */
case 6: t |= ((uint64_t)m[i + 5]) << 40; /* fallthrough */
case 5: t |= ((uint64_t)m[i + 4]) << 32; /* fallthrough */
case 4: t |= ((uint64_t)m[i + 3]) << 24; /* fallthrough */
case 3: t |= ((uint64_t)m[i + 2]) << 16; /* fallthrough */
case 2: t |= ((uint64_t)m[i + 1]) << 8; /* fallthrough */
case 1: t |= ((uint64_t)m[i]);
}
v3 ^= t; SIPROUND; SIPROUND; v0 ^= t;
v2 ^= 0xff;
SIPROUND; SIPROUND; SIPROUND; SIPROUND;
return v0 ^ v1 ^ v2 ^ v3;
}
// Decode the Golomb-Rice set (unary quotient, P-bit remainder,
// cumulative) into g_set. Returns the number of values decoded (short
// on a truncated body).
static uint32_t decode_set(uint32_t n_set, uint32_t body_len) {
const uint8_t* buf = g_filter;
uint32_t bits = body_len * 8, pos = 0, s;
uint64_t last = 0;
for (s = 0; s < n_set; s++) {
uint64_t q = 0;
while (pos < bits && ((buf[pos >> 3] >> (7 - (pos & 7))) & 1)) { q++; pos++; }
if (pos + 1 + GCS_P > bits) return s;
pos++;
uint64_t r = 0; int k;
for (k = 0; k < GCS_P; k++, pos++) r = (r << 1) | ((buf[pos >> 3] >> (7 - (pos & 7))) & 1);
last += (q << GCS_P) | r;
g_set[s] = last;
}
return s;
}
static inline int in_set(uint64_t v, uint32_t n) {
uint32_t lo = 0, hi = n;
while (lo < hi) {
uint32_t mid = lo + ((hi - lo) >> 1);
if (g_set[mid] < v) lo = mid + 1; else hi = mid;
}
return lo < n && g_set[lo] == v;
}
// Decode the set once (it is already sorted), then hash each script and
// binary-search its target. No sort of the script targets is needed,
// and a typical DigiByte block (tens of elements) is a 5-6 step search.
// Returns -1 when the filter is beyond the static buffers (the caller
// falls back to the JS matcher), never a silently partial answer.
EXPORT int32_t gcs_match(uint32_t n_set, uint32_t body_len, int32_t want_all) {
if (g_count == 0 || n_set == 0) return 0;
if (n_set > GCS_SET_CAP || body_len > GCS_FILTER_BYTES) return -1;
uint32_t n = decode_set(n_set, body_len);
if (n == 0) return 0;
uint64_t k0 = rd64(g_key), k1 = rd64(g_key + 8);
uint64_t F = (uint64_t)n_set * GCS_M;
uint32_t i, found = 0;
for (i = 0; i < g_count; i++) {
uint64_t h = siphash24(k0, k1, g_scripts + g_offsets[i], g_offsets[i + 1] - g_offsets[i]);
uint64_t target = (uint64_t)(((unsigned __int128)h * F) >> 64);
if (in_set(target, n)) {
g_matches[found++] = i;
if (!want_all) return (int32_t)found;
}
}
return (int32_t)found;
}

View file

@ -24,7 +24,8 @@
"blocks": "tsx scripts/block-fetch-demo.ts",
"cache": "tsx scripts/cache-demo.ts",
"daemon": "tsx bin/dgb-daemon.ts",
"pow:test": "tsx scripts/pow-vectors.ts"
"pow:test": "tsx scripts/pow-vectors.ts",
"gcs:test": "tsx scripts/gcs-check.ts"
},
"dependencies": {
"@bitcoinerlab/secp256k1": "^1.1.1",

View file

@ -0,0 +1,111 @@
// Cross-check the Number-based SipHash / GCS matcher against a straight
// BigInt reference (the previous implementation), on random filters and
// elements, and report throughput for a realistic watched-script set.
//
// npx tsx packages/p2p/scripts/gcs-check.ts
import { randomBytes } from 'node:crypto';
import { filterMatchesAny, filterMatches, keyFromBlockHash, gcsBackend, gcsUseJs, P, M } from '../src/gcs.js';
import { siphash24 } from '../src/siphash.js';
// ---- reference (BigInt) implementation ----
const MASK64 = 0xffffffffffffffffn;
const rot = (v: bigint, n: number): bigint => ((v << BigInt(n)) & MASK64) | (v >> BigInt(64 - n));
function sipRound(s: bigint[]): void {
let [v0, v1, v2, v3] = s;
v0 = (v0 + v1) & MASK64; v1 = rot(v1, 13); v1 ^= v0; v0 = rot(v0, 32);
v2 = (v2 + v3) & MASK64; v3 = rot(v3, 16); v3 ^= v2;
v0 = (v0 + v3) & MASK64; v3 = rot(v3, 21); v3 ^= v0;
v2 = (v2 + v1) & MASK64; v1 = rot(v1, 17); v1 ^= v2; v2 = rot(v2, 32);
s[0] = v0; s[1] = v1; s[2] = v2; s[3] = v3;
}
function refSiphash(key: Buffer, msg: Buffer): bigint {
const k0 = key.readBigUInt64LE(0), k1 = key.readBigUInt64LE(8);
const s = [k0 ^ 0x736f6d6570736575n, k1 ^ 0x646f72616e646f6dn, k0 ^ 0x6c7967656e657261n, k1 ^ 0x7465646279746573n];
const full = Math.floor(msg.length / 8);
for (let i = 0; i < full; i++) { const m = msg.readBigUInt64LE(i * 8); s[3] ^= m; sipRound(s); sipRound(s); s[0] ^= m; }
const tail = Buffer.alloc(8); msg.subarray(full * 8).copy(tail); tail[7] = msg.length & 0xff;
const m = tail.readBigUInt64LE(0); s[3] ^= m; sipRound(s); sipRound(s); s[0] ^= m; s[2] ^= 0xffn;
sipRound(s); sipRound(s); sipRound(s); sipRound(s);
return (s[0] ^ s[1] ^ s[2] ^ s[3]) & MASK64;
}
const refRange = (key: Buffer, el: Buffer, F: bigint): bigint => (refSiphash(key, el) * F) >> 64n;
// ---- BIP158 encoder (for building test filters) ----
function encodeVarint(n: number): Buffer {
if (n < 0xfd) return Buffer.from([n]);
if (n <= 0xffff) { const b = Buffer.alloc(3); b[0] = 0xfd; b.writeUInt16LE(n, 1); return b; }
const b = Buffer.alloc(5); b[0] = 0xfe; b.writeUInt32LE(n, 1); return b;
}
function buildFilter(blockHash: string, elements: Buffer[]): Buffer {
const key = keyFromBlockHash(blockHash);
const F = BigInt(elements.length) * M;
const vals = [...new Set(elements.map((e) => refRange(key, e, F)))].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
const bits: number[] = [];
let last = 0n;
for (const v of vals) {
const d = v - last; last = v;
const q = d >> BigInt(P), r = d & ((1n << BigInt(P)) - 1n);
for (let i = 0n; i < q; i++) bits.push(1);
bits.push(0);
for (let i = P - 1; i >= 0; i--) bits.push(Number((r >> BigInt(i)) & 1n));
}
const body = Buffer.alloc(Math.ceil(bits.length / 8));
bits.forEach((b, i) => { if (b) body[i >> 3] |= 0x80 >> (i & 7); });
// N is the element count the encoder hashed with (duplicates collapse
// in the set but N·M stays the range), exactly as Core writes it.
return Buffer.concat([encodeVarint(elements.length), body]);
}
// 1. SipHash agreement on random keys / lengths 0..40 (incl. the 7-byte tail cases).
let fails = 0;
for (let i = 0; i < 2000; i++) {
const key = randomBytes(16), msg = randomBytes(i % 41);
if (siphash24(key, msg) !== refSiphash(key, msg)) { fails++; if (fails < 5) console.log('siphash mismatch len', msg.length); }
}
console.log(`siphash: ${fails === 0 ? 'OK' : fails + ' MISMATCHES'} (2000 random messages)`);
// 2. Matcher agreement: random filters (1..3000 elements), random
// probes, plus probes that deliberately include members so the
// positive path is exercised. Run for each backend available.
let mfails = 0;
const backends: Array<'wasm' | 'js'> = gcsBackend() === 'wasm' ? ['wasm', 'js'] : ['js'];
if (backends[0] === 'js') console.log('note: dgbpow.wasm without gcs_match — only the JS path is checked');
for (const backend of backends) {
gcsUseJs(backend === 'js');
let bf = 0, pos = 0, neg = 0;
for (let round = 0; round < 60; round++) {
const n = 1 + Math.floor(Math.random() * (round < 50 ? 60 : 3000));
const inSet = Array.from({ length: n }, () => randomBytes(20 + Math.floor(Math.random() * 15)));
const hash = randomBytes(32).toString('hex');
const filter = buildFilter(hash, inSet);
for (const el of inSet.slice(0, 30)) if (!filterMatchesAny(filter, hash, [el])) { bf++; console.log('member missed'); }
const key = keyFromBlockHash(hash); const F = BigInt(n) * M;
const setVals = new Set(inSet.map((e) => refRange(key, e, F)));
const probes = Array.from({ length: 500 }, () => randomBytes(25));
if (round % 2) probes[Math.floor(Math.random() * probes.length)] = inSet[Math.floor(Math.random() * n)];
const expectAny = probes.some((p) => setVals.has(refRange(key, p, F)));
const got = filterMatchesAny(filter, hash, probes);
if (got !== expectAny) { bf++; console.log('verdict mismatch'); }
got ? pos++ : neg++;
const expectWhich = probes.filter((p) => setVals.has(refRange(key, p, F))).map((p) => p.toString('hex')).sort();
const gotWhich = filterMatches(filter, hash, probes).map((p) => p.toString('hex')).sort();
if (expectWhich.join() !== gotWhich.join()) { bf++; console.log('filterMatches mismatch'); }
}
mfails += bf;
console.log(`matcher[${backend}]: ${bf === 0 ? 'OK' : bf + ' MISMATCHES'} (60 filters; ${pos} any-hit, ${neg} no-hit rounds)`);
}
// 3. Throughput: 3,300 watched scripts vs 200 filters of ~40 elements (a typical DGB block).
const scripts = Array.from({ length: 3300 }, () => randomBytes(25));
const filters = Array.from({ length: 200 }, () => { const h = randomBytes(32).toString('hex'); return { h, f: buildFilter(h, Array.from({ length: 40 }, () => randomBytes(25))) }; });
for (const backend of backends) {
gcsUseJs(backend === 'js');
const t0 = performance.now();
for (const { h, f } of filters) filterMatchesAny(f, h, scripts);
const ms = (performance.now() - t0) / filters.length;
console.log(`throughput[${backend}]: ${(1000 / ms).toFixed(0)} filters/s with 3,300 scripts (${ms.toFixed(2)} ms/filter)`);
}
let t0 = performance.now();
for (const { h } of filters.slice(0, 10)) { const key = keyFromBlockHash(h); const F = BigInt(40) * M; for (const s of scripts) refRange(key, s, F); }
console.log(`throughput[bigint reference, hashing only]: ${((performance.now() - t0) / 10).toFixed(1)} ms/filter`);
process.exit(fails + mfails === 0 ? 0 : 1);

View file

@ -1,10 +1,26 @@
import { siphash24 } from './siphash.js';
import { sipKey, siphash24Pair, type SipKey } from './siphash.js';
import { decodeVarint } from './varint.js';
import { loadPowWasm } from './pow.js';
// BIP158 Golomb-Coded Set filter operations.
// Parameters for the "basic" filter type (0x00):
export const P = 19;
export const M = 784931n;
const M_NUM = 784931;
// Two implementations of the same thing:
//
// • native — gcs.c inside ../native/dgbpow.wasm. The watched scripts
// are uploaded into the module once per script set (the scan driver
// passes the same array for a whole scan) and each filter is then
// one call. ~30 ns per script; a few thousand scripts × a year of
// DigiByte blocks is minutes, not hours.
// • JS — Number arithmetic throughout (a filter's range N·M is far
// below 2^53, and the one overflowing step, (h·F) >> 64, is done
// with 32×32-bit partial products). ~3 µs per script. Used when the
// module is missing or the script set exceeds its static buffers.
//
// Both are checked against a BigInt reference by scripts/gcs-check.ts.
// Derive the SipHash key from a block hash: first 16 bytes of the
// block hash's LITTLE-endian byte order.
@ -16,55 +32,145 @@ export function keyFromBlockHash(blockHashHex: string): Buffer {
return Buffer.from(le.subarray(0, 16));
}
// Map an element to the filter's [0, N*M) range: fastReduce of the
// SipHash output over N*M. Matches BIP158's `hashToRange`.
function hashToRange(key: Buffer, element: Buffer, F: bigint): bigint {
const h = siphash24(key, element);
// (h * F) >> 64 — do it in bigint to avoid precision loss.
return (h * F) >> 64n;
// ---------------------------------------------------------------- native
interface Native {
mem: Uint8Array;
scriptsPtr: number; offsetsPtr: number; keyPtr: number; filterPtr: number; matchesPtr: number;
maxScripts: number; scriptBytes: number; filterBytes: number;
setCount: (n: number) => void;
match: (nSet: number, bodyLen: number, wantAll: number) => number;
}
let native: Native | null | undefined; // undefined = not tried yet
let forceJs = false;
/** Test hook: `gcsUseJs(true)` disables the native path for this process. */
export function gcsUseJs(on: boolean): void { forceJs = on; uploaded = null; }
export function gcsBackend(): 'wasm' | 'js' { return !forceJs && getNative() ? 'wasm' : 'js'; }
function getNative(): Native | null {
if (native !== undefined) return native;
try {
const w = loadPowWasm();
const x = w.exports;
if (typeof x.gcs_match !== 'function') { native = null; return null; }
const limitsPtr = (x.gcs_limits_ptr as () => number)();
const limits = new Uint32Array(w.memory.buffer, limitsPtr, 3);
native = {
mem: w.mem,
scriptsPtr: (x.gcs_scripts_ptr as () => number)(),
offsetsPtr: (x.gcs_offsets_ptr as () => number)(),
keyPtr: (x.gcs_key_ptr as () => number)(),
filterPtr: (x.gcs_filter_ptr as () => number)(),
matchesPtr: (x.gcs_matches_ptr as () => number)(),
maxScripts: limits[0], scriptBytes: limits[1], filterBytes: limits[2],
setCount: x.gcs_set_count as (n: number) => void,
match: x.gcs_match as (nSet: number, bodyLen: number, wantAll: number) => number,
};
} catch {
native = null;
}
return native;
}
// Read one Golomb-Rice-coded value: unary prefix (`q` ones, then 0)
// followed by P bits of remainder. Value = q*2^P + r.
class BitReader {
private byte = 0;
private bit = 0;
constructor(private buf: Buffer) {}
readBit(): number {
if (this.byte >= this.buf.length) throw new Error('BitReader: past end of buffer');
const b = (this.buf[this.byte] >> (7 - this.bit)) & 1;
if (++this.bit === 8) {
this.bit = 0;
this.byte++;
}
return b;
}
readBits(n: number): bigint {
let out = 0n;
for (let i = 0; i < n; i++) out = (out << 1n) | BigInt(this.readBit());
return out;
// The script set currently uploaded into the module, by array identity
// and length (the scan driver never mutates its array mid-scan, but a
// grown array is re-uploaded anyway).
let uploaded: { arr: Buffer[]; len: number } | null = null;
function ensureUploaded(n: Native, elements: Buffer[]): boolean {
if (uploaded && uploaded.arr === elements && uploaded.len === elements.length) return true;
if (elements.length > n.maxScripts) return false;
let total = 0;
for (const e of elements) total += e.length;
if (total > n.scriptBytes) return false;
const offsets = new Uint32Array(n.mem.buffer, n.offsetsPtr, elements.length + 1);
let off = 0;
for (let i = 0; i < elements.length; i++) {
offsets[i] = off;
n.mem.set(elements[i], n.scriptsPtr + off);
off += elements[i].length;
}
offsets[elements.length] = off;
n.setCount(elements.length);
uploaded = { arr: elements, len: elements.length };
return true;
}
// Decode all values in a Golomb-coded set into a sorted array.
// The set body is what `cfilter.filter` returns AFTER the varint N
// prefix has been consumed.
function decodeGcsValues(body: Buffer, n: number): bigint[] {
const reader = new BitReader(body);
const values: bigint[] = new Array(n);
let last = 0n;
function nativeMatch(n: Native, filterBody: Buffer, blockHash: string, elements: Buffer[], wantAll: boolean): number | null {
if (!ensureUploaded(n, elements)) return null;
const { value: nBig, bytesRead } = decodeVarint(filterBody, 0);
const nSet = Number(nBig);
if (nSet === 0) return 0;
const body = filterBody.subarray(bytesRead);
if (body.length > n.filterBytes) return null;
n.mem.set(keyFromBlockHash(blockHash), n.keyPtr);
n.mem.set(body, n.filterPtr);
const r = n.match(nSet, body.length, wantAll ? 1 : 0);
return r < 0 ? null : r; // -1: filter too big for the module's buffers → JS
}
// -------------------------------------------------------------------- JS
// Exact 32×32 → 64-bit product as [hi, lo] uint32 (via 16-bit halves).
const mulOut = new Float64Array(2);
function mul32(a: number, b: number): void {
const al = a & 0xffff, ah = a >>> 16, bl = b & 0xffff, bh = b >>> 16;
const ll = al * bl, mid = al * bh + ah * bl, hh = ah * bh;
const lo = ll + (mid % 65536) * 65536;
const carry = Math.floor(lo / 4294967296);
mulOut[1] = lo - carry * 4294967296;
mulOut[0] = hh + Math.floor(mid / 65536) + carry;
}
// Map an element to the filter's [0, N·M) range: fastReduce of the
// SipHash output over N·M, i.e. (h · F) >> 64. Matches BIP158's
// `hashToRange`. `F` is split once per filter into fh·2^32 + fl.
const hashOut = new Uint32Array(2);
function hashToRange(key: SipKey, element: Buffer, fh: number, fl: number): number {
siphash24Pair(key, element, hashOut);
const hi = hashOut[0], lo = hashOut[1];
// h·F = hi·fh·2^64 + (hi·fl + lo·fh)·2^32 + lo·fl
mul32(lo, fl); const zHi = mulOut[0];
mul32(hi, fl); const xHi = mulOut[0], xLo = mulOut[1];
mul32(lo, fh); const yHi = mulOut[0], yLo = mulOut[1];
const l = xLo + yLo + zHi;
const c = Math.floor(l / 4294967296);
return hi * fh + xHi + yHi + c;
}
function splitF(n: number): { fh: number; fl: number } {
const F = n * M_NUM;
const fh = Math.floor(F / 4294967296);
return { fh, fl: F - fh * 4294967296 };
}
// Decode all values of a Golomb-Rice-coded set (unary quotient, P-bit
// remainder, cumulative) into a sorted Float64Array. The body is what
// `cfilter.filter` returns AFTER the varint N prefix has been consumed.
function decodeGcsValues(body: Buffer, n: number): Float64Array {
const values = new Float64Array(n);
const bitLen = body.length * 8;
let pos = 0;
let last = 0;
const bit = (p: number): number => (body[p >>> 3] >> (7 - (p & 7))) & 1;
for (let i = 0; i < n; i++) {
// Unary quotient: count 1-bits until we see a 0.
let q = 0n;
while (reader.readBit() === 1) q++;
const r = reader.readBits(P);
const delta = (q << BigInt(P)) | r;
last += delta;
let q = 0;
while (pos < bitLen && bit(pos) === 1) { q++; pos++; }
if (pos + 1 + P > bitLen) throw new Error('GCS: past end of buffer');
pos++; // the terminating 0 of the unary prefix
let r = 0;
for (let k = 0; k < P; k++) r = r * 2 + bit(pos++);
last += q * 524288 + r; // q·2^P + r
values[i] = last;
}
return values;
}
function targetsFor(key: SipKey, elements: Buffer[], n: number): Float64Array {
const { fh, fl } = splitF(n);
const t = new Float64Array(elements.length);
for (let i = 0; i < elements.length; i++) t[i] = hashToRange(key, elements[i], fh, fl);
return t.sort();
}
// ------------------------------------------------------------------ API
// Test whether ANY of `elements` are in the filter. Returns true on
// the first match; false if none matched. This is the primary
// wallet-side operation: pass every scriptPubKey (and any input
@ -75,22 +181,22 @@ export function filterMatchesAny(
elements: Buffer[],
): boolean {
if (elements.length === 0) return false;
// Parse N (element count) as a varint at the start of the body.
if (!forceJs) {
const n = getNative();
if (n) {
const r = nativeMatch(n, filterBody, blockHash, elements, false);
if (r !== null) return r > 0;
}
}
const { value: nBig, bytesRead } = decodeVarint(filterBody, 0);
const n = Number(nBig);
if (n === 0) return false;
const setBody = filterBody.subarray(bytesRead);
const F = BigInt(n) * M;
const key = keyFromBlockHash(blockHash);
const targets = elements
.map((el) => hashToRange(key, el, F))
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
// Two-pointer sweep over sorted values + sorted targets.
const setValues = decodeGcsValues(setBody, n);
const key = sipKey(keyFromBlockHash(blockHash));
const targets = targetsFor(key, elements, n);
const setValues = decodeGcsValues(filterBody.subarray(bytesRead), n);
let ti = 0;
for (const v of setValues) {
for (let i = 0; i < n; i++) {
const v = setValues[i];
while (ti < targets.length && targets[ti] < v) ti++;
if (ti === targets.length) return false;
if (targets[ti] === v) return true;
@ -107,26 +213,31 @@ export function filterMatches(
elements: Buffer[],
): Buffer[] {
if (elements.length === 0) return [];
if (!forceJs) {
const nat = getNative();
if (nat) {
const r = nativeMatch(nat, filterBody, blockHash, elements, true);
if (r !== null) {
const idx = new Uint32Array(nat.mem.buffer, nat.matchesPtr, r);
return Array.from(idx, (i) => elements[i]);
}
}
}
const { value: nBig, bytesRead } = decodeVarint(filterBody, 0);
const n = Number(nBig);
if (n === 0) return [];
const setBody = filterBody.subarray(bytesRead);
const F = BigInt(n) * M;
const key = keyFromBlockHash(blockHash);
const withHash = elements.map((el) => ({ el, target: hashToRange(key, el, F) }));
withHash.sort((a, b) => (a.target < b.target ? -1 : a.target > b.target ? 1 : 0));
const setValues = decodeGcsValues(setBody, n);
const key = sipKey(keyFromBlockHash(blockHash));
const { fh, fl } = splitF(n);
const withHash = elements.map((el) => ({ el, target: hashToRange(key, el, fh, fl) }));
withHash.sort((a, b) => a.target - b.target);
const setValues = decodeGcsValues(filterBody.subarray(bytesRead), n);
const matched: Buffer[] = [];
let ti = 0;
for (const v of setValues) {
for (let i = 0; i < n; i++) {
const v = setValues[i];
while (ti < withHash.length && withHash[ti].target < v) ti++;
if (ti === withHash.length) break;
if (withHash[ti].target === v) {
matched.push(withHash[ti].el);
ti++;
}
while (ti < withHash.length && withHash[ti].target === v) matched.push(withHash[ti++].el);
}
return matched;
}

View file

@ -62,8 +62,11 @@ export function workFromBits(bits: number): bigint {
}
// ---- WebAssembly module (lazy, loaded once) ----
interface Wasm {
export interface Wasm {
mem: Uint8Array;
memory: WebAssembly.Memory;
/** Every export of the module — the GCS matcher (gcs.ts) picks its own. */
exports: Record<string, unknown>;
inPtr: number; outPtr: number;
hash_qubit: () => void; hash_skein512: () => void; hash_groestl512: () => void; hash_odo: (key: number) => void;
}
@ -77,6 +80,8 @@ export function loadPowWasm(path = new URL('../native/dgbpow.wasm', import.meta.
const memory = x.memory as WebAssembly.Memory;
wasm = {
mem: new Uint8Array(memory.buffer),
memory,
exports: x,
inPtr: (x.in_ptr as () => number)(),
outPtr: (x.out_ptr as () => number)(),
hash_qubit: x.hash_qubit as () => void,

View file

@ -1,71 +1,101 @@
// SipHash-2-4 for BIP158 element hashing. Pure JS, no deps.
// Implementation follows Jean-Philippe Aumasson's reference and
// matches Bitcoin Core's src/crypto/siphash.cpp bit-for-bit.
// Matches Bitcoin Core's src/crypto/siphash.cpp bit-for-bit.
//
// Public API: `siphash24(key: Buffer, msg: Buffer): bigint` — returns
// the 64-bit output as a bigint. BIP158 feeds this into a
// map-to-range step before checking against the Golomb-coded set.
// The 64-bit lanes are kept as pairs of uint32 Numbers rather than
// BigInts: a wallet watching a few thousand scripts hashes every one of
// them against every block filter (the key is per-block), and BigInt
// arithmetic made that ~100× slower than it needs to be — the 1-year
// scan dropped to ~3 filters/s once the gap limits went to 200/100.
//
// Public API:
// siphash24(key, msg): bigint — 64-bit output (kept for callers/tests)
// siphash24Pair(key, msg, out): void — writes [hi, lo] uint32 into `out`
// SipKey / sipKey(key) — pre-split key for the hot loop
const MASK64 = 0xffffffffffffffffn;
export interface SipKey { k0h: number; k0l: number; k1h: number; k1l: number }
function rot(v: bigint, n: number): bigint {
return ((v << BigInt(n)) & MASK64) | (v >> BigInt(64 - n));
}
// One SipRound: mixes four 64-bit lanes.
function sipRound(state: bigint[]): void {
let [v0, v1, v2, v3] = state;
v0 = (v0 + v1) & MASK64;
v1 = rot(v1, 13);
v1 ^= v0;
v0 = rot(v0, 32);
v2 = (v2 + v3) & MASK64;
v3 = rot(v3, 16);
v3 ^= v2;
v0 = (v0 + v3) & MASK64;
v3 = rot(v3, 21);
v3 ^= v0;
v2 = (v2 + v1) & MASK64;
v1 = rot(v1, 17);
v1 ^= v2;
v2 = rot(v2, 32);
state[0] = v0;
state[1] = v1;
state[2] = v2;
state[3] = v3;
}
export function siphash24(key: Buffer, msg: Buffer): bigint {
export function sipKey(key: Buffer): SipKey {
if (key.length !== 16) throw new Error('SipHash key must be 16 bytes');
const k0 = key.readBigUInt64LE(0);
const k1 = key.readBigUInt64LE(8);
const state = [
k0 ^ 0x736f6d6570736575n,
k1 ^ 0x646f72616e646f6dn,
k0 ^ 0x6c7967656e657261n,
k1 ^ 0x7465646279746573n,
];
const fullBlocks = Math.floor(msg.length / 8);
for (let i = 0; i < fullBlocks; i++) {
const m = msg.readBigUInt64LE(i * 8);
state[3] ^= m;
sipRound(state);
sipRound(state);
state[0] ^= m;
}
// Final block: 0..7 remaining bytes, LE, plus msg length in top byte.
const tail = Buffer.alloc(8);
msg.subarray(fullBlocks * 8).copy(tail);
tail[7] = msg.length & 0xff;
const m = tail.readBigUInt64LE(0);
state[3] ^= m;
sipRound(state);
sipRound(state);
state[0] ^= m;
state[2] ^= 0xffn;
sipRound(state);
sipRound(state);
sipRound(state);
sipRound(state);
return (state[0] ^ state[1] ^ state[2] ^ state[3]) & MASK64;
return {
k0l: key.readUInt32LE(0), k0h: key.readUInt32LE(4),
k1l: key.readUInt32LE(8), k1h: key.readUInt32LE(12),
};
}
// Lanes: v0 = (a0,b0) … v3 = (a3,b3) with a = high word, b = low word.
let a0 = 0, b0 = 0, a1 = 0, b1 = 0, a2 = 0, b2 = 0, a3 = 0, b3 = 0;
// One SipRound on the module-level lanes (avoids allocation per call).
function sipRound(): void {
let t: number;
// v0 += v1
t = b0 + b1; b0 = t >>> 0; a0 = (a0 + a1 + (t >= 0x100000000 ? 1 : 0)) >>> 0;
// v1 = rotl(v1, 13)
t = (a1 << 13) | (b1 >>> 19); b1 = ((b1 << 13) | (a1 >>> 19)) >>> 0; a1 = t >>> 0;
// v1 ^= v0
a1 = (a1 ^ a0) >>> 0; b1 = (b1 ^ b0) >>> 0;
// v0 = rotl(v0, 32)
t = a0; a0 = b0; b0 = t;
// v2 += v3
t = b2 + b3; b2 = t >>> 0; a2 = (a2 + a3 + (t >= 0x100000000 ? 1 : 0)) >>> 0;
// v3 = rotl(v3, 16)
t = (a3 << 16) | (b3 >>> 16); b3 = ((b3 << 16) | (a3 >>> 16)) >>> 0; a3 = t >>> 0;
// v3 ^= v2
a3 = (a3 ^ a2) >>> 0; b3 = (b3 ^ b2) >>> 0;
// v0 += v3
t = b0 + b3; b0 = t >>> 0; a0 = (a0 + a3 + (t >= 0x100000000 ? 1 : 0)) >>> 0;
// v3 = rotl(v3, 21)
t = (a3 << 21) | (b3 >>> 11); b3 = ((b3 << 21) | (a3 >>> 11)) >>> 0; a3 = t >>> 0;
// v3 ^= v0
a3 = (a3 ^ a0) >>> 0; b3 = (b3 ^ b0) >>> 0;
// v2 += v1
t = b2 + b1; b2 = t >>> 0; a2 = (a2 + a1 + (t >= 0x100000000 ? 1 : 0)) >>> 0;
// v1 = rotl(v1, 17)
t = (a1 << 17) | (b1 >>> 15); b1 = ((b1 << 17) | (a1 >>> 15)) >>> 0; a1 = t >>> 0;
// v1 ^= v2
a1 = (a1 ^ a2) >>> 0; b1 = (b1 ^ b2) >>> 0;
// v2 = rotl(v2, 32)
t = a2; a2 = b2; b2 = t;
}
/** SipHash-2-4; writes the 64-bit result as out[0] = high word, out[1] = low word. */
export function siphash24Pair(key: SipKey, msg: Buffer, out: Uint32Array | number[]): void {
a0 = (key.k0h ^ 0x736f6d65) >>> 0; b0 = (key.k0l ^ 0x70736575) >>> 0;
a1 = (key.k1h ^ 0x646f7261) >>> 0; b1 = (key.k1l ^ 0x6e646f6d) >>> 0;
a2 = (key.k0h ^ 0x6c796765) >>> 0; b2 = (key.k0l ^ 0x6e657261) >>> 0;
a3 = (key.k1h ^ 0x74656462) >>> 0; b3 = (key.k1l ^ 0x79746573) >>> 0;
const len = msg.length;
const full = len - (len & 7);
let i = 0;
for (; i < full; i += 8) {
const ml = msg.readUInt32LE(i), mh = msg.readUInt32LE(i + 4);
a3 = (a3 ^ mh) >>> 0; b3 = (b3 ^ ml) >>> 0;
sipRound(); sipRound();
a0 = (a0 ^ mh) >>> 0; b0 = (b0 ^ ml) >>> 0;
}
// Final block: remaining 0..7 bytes LE, message length in the top byte.
let ml = 0, mh = 0;
switch (len & 7) {
case 7: mh |= msg[i + 6] << 16; // falls through
case 6: mh |= msg[i + 5] << 8; // falls through
case 5: mh |= msg[i + 4]; // falls through
case 4: ml |= msg[i + 3] << 24; // falls through
case 3: ml |= msg[i + 2] << 16; // falls through
case 2: ml |= msg[i + 1] << 8; // falls through
case 1: ml |= msg[i];
}
mh = (mh | ((len & 0xff) << 24)) >>> 0; ml >>>= 0;
a3 = (a3 ^ mh) >>> 0; b3 = (b3 ^ ml) >>> 0;
sipRound(); sipRound();
a0 = (a0 ^ mh) >>> 0; b0 = (b0 ^ ml) >>> 0;
b2 = (b2 ^ 0xff) >>> 0;
sipRound(); sipRound(); sipRound(); sipRound();
out[0] = (a0 ^ a1 ^ a2 ^ a3) >>> 0;
out[1] = (b0 ^ b1 ^ b2 ^ b3) >>> 0;
}
const scratch = new Uint32Array(2);
export function siphash24(key: Buffer, msg: Buffer): bigint {
siphash24Pair(sipKey(key), msg, scratch);
return (BigInt(scratch[0]) << 32n) | BigInt(scratch[1]);
}