690959a983
The daemon's wallet scan hashed every watched script against every block
filter with a BigInt SipHash: ~3 filters/s on the VPS once the gap limits
went to 200/100 (~3,300 scripts), i.e. days for a one-year window. The
matcher now lives in dgbpow.wasm next to the PoW hashes — scripts are
uploaded once per scan, each filter is one call, the set is decoded once
and each target binary-searched — ~5,500 filters/s locally, ~2,000/s on
the VPS with block fetches. The JS path was rewritten with uint32 lanes
and exact 32×32 partial products as a fallback (module missing or filter
beyond its buffers). scripts/gcs-check.ts cross-checks both against a
BigInt reference on random filters; `npm run gcs:test -w @dgb-wallet/p2p`.
NOWNodes stopped accepting the proxy's API key ("Unknown API_key" on every
endpoint), which is why the wallet showed "Blockbook HTTP 401" for every
path. The vhost mirror now proxies /blockbook/ to Atomic Wallet's public
Blockbook; ops/blockbook-upstream.sh switches the live vhost either way.
111 lines
6.3 KiB
TypeScript
111 lines
6.3 KiB
TypeScript
// Cross-check the Number-based SipHash / GCS matcher against a straight
|
|
// BigInt reference (the previous implementation), on random filters and
|
|
// elements, and report throughput for a realistic watched-script set.
|
|
//
|
|
// npx tsx packages/p2p/scripts/gcs-check.ts
|
|
import { randomBytes } from 'node:crypto';
|
|
import { filterMatchesAny, filterMatches, keyFromBlockHash, gcsBackend, gcsUseJs, P, M } from '../src/gcs.js';
|
|
import { siphash24 } from '../src/siphash.js';
|
|
|
|
// ---- reference (BigInt) implementation ----
|
|
const MASK64 = 0xffffffffffffffffn;
|
|
const rot = (v: bigint, n: number): bigint => ((v << BigInt(n)) & MASK64) | (v >> BigInt(64 - n));
|
|
function sipRound(s: bigint[]): void {
|
|
let [v0, v1, v2, v3] = s;
|
|
v0 = (v0 + v1) & MASK64; v1 = rot(v1, 13); v1 ^= v0; v0 = rot(v0, 32);
|
|
v2 = (v2 + v3) & MASK64; v3 = rot(v3, 16); v3 ^= v2;
|
|
v0 = (v0 + v3) & MASK64; v3 = rot(v3, 21); v3 ^= v0;
|
|
v2 = (v2 + v1) & MASK64; v1 = rot(v1, 17); v1 ^= v2; v2 = rot(v2, 32);
|
|
s[0] = v0; s[1] = v1; s[2] = v2; s[3] = v3;
|
|
}
|
|
function refSiphash(key: Buffer, msg: Buffer): bigint {
|
|
const k0 = key.readBigUInt64LE(0), k1 = key.readBigUInt64LE(8);
|
|
const s = [k0 ^ 0x736f6d6570736575n, k1 ^ 0x646f72616e646f6dn, k0 ^ 0x6c7967656e657261n, k1 ^ 0x7465646279746573n];
|
|
const full = Math.floor(msg.length / 8);
|
|
for (let i = 0; i < full; i++) { const m = msg.readBigUInt64LE(i * 8); s[3] ^= m; sipRound(s); sipRound(s); s[0] ^= m; }
|
|
const tail = Buffer.alloc(8); msg.subarray(full * 8).copy(tail); tail[7] = msg.length & 0xff;
|
|
const m = tail.readBigUInt64LE(0); s[3] ^= m; sipRound(s); sipRound(s); s[0] ^= m; s[2] ^= 0xffn;
|
|
sipRound(s); sipRound(s); sipRound(s); sipRound(s);
|
|
return (s[0] ^ s[1] ^ s[2] ^ s[3]) & MASK64;
|
|
}
|
|
const refRange = (key: Buffer, el: Buffer, F: bigint): bigint => (refSiphash(key, el) * F) >> 64n;
|
|
|
|
// ---- BIP158 encoder (for building test filters) ----
|
|
function encodeVarint(n: number): Buffer {
|
|
if (n < 0xfd) return Buffer.from([n]);
|
|
if (n <= 0xffff) { const b = Buffer.alloc(3); b[0] = 0xfd; b.writeUInt16LE(n, 1); return b; }
|
|
const b = Buffer.alloc(5); b[0] = 0xfe; b.writeUInt32LE(n, 1); return b;
|
|
}
|
|
function buildFilter(blockHash: string, elements: Buffer[]): Buffer {
|
|
const key = keyFromBlockHash(blockHash);
|
|
const F = BigInt(elements.length) * M;
|
|
const vals = [...new Set(elements.map((e) => refRange(key, e, F)))].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
|
|
const bits: number[] = [];
|
|
let last = 0n;
|
|
for (const v of vals) {
|
|
const d = v - last; last = v;
|
|
const q = d >> BigInt(P), r = d & ((1n << BigInt(P)) - 1n);
|
|
for (let i = 0n; i < q; i++) bits.push(1);
|
|
bits.push(0);
|
|
for (let i = P - 1; i >= 0; i--) bits.push(Number((r >> BigInt(i)) & 1n));
|
|
}
|
|
const body = Buffer.alloc(Math.ceil(bits.length / 8));
|
|
bits.forEach((b, i) => { if (b) body[i >> 3] |= 0x80 >> (i & 7); });
|
|
// N is the element count the encoder hashed with (duplicates collapse
|
|
// in the set but N·M stays the range), exactly as Core writes it.
|
|
return Buffer.concat([encodeVarint(elements.length), body]);
|
|
}
|
|
|
|
// 1. SipHash agreement on random keys / lengths 0..40 (incl. the 7-byte tail cases).
|
|
let fails = 0;
|
|
for (let i = 0; i < 2000; i++) {
|
|
const key = randomBytes(16), msg = randomBytes(i % 41);
|
|
if (siphash24(key, msg) !== refSiphash(key, msg)) { fails++; if (fails < 5) console.log('siphash mismatch len', msg.length); }
|
|
}
|
|
console.log(`siphash: ${fails === 0 ? 'OK' : fails + ' MISMATCHES'} (2000 random messages)`);
|
|
|
|
// 2. Matcher agreement: random filters (1..3000 elements), random
|
|
// probes, plus probes that deliberately include members so the
|
|
// positive path is exercised. Run for each backend available.
|
|
let mfails = 0;
|
|
const backends: Array<'wasm' | 'js'> = gcsBackend() === 'wasm' ? ['wasm', 'js'] : ['js'];
|
|
if (backends[0] === 'js') console.log('note: dgbpow.wasm without gcs_match — only the JS path is checked');
|
|
for (const backend of backends) {
|
|
gcsUseJs(backend === 'js');
|
|
let bf = 0, pos = 0, neg = 0;
|
|
for (let round = 0; round < 60; round++) {
|
|
const n = 1 + Math.floor(Math.random() * (round < 50 ? 60 : 3000));
|
|
const inSet = Array.from({ length: n }, () => randomBytes(20 + Math.floor(Math.random() * 15)));
|
|
const hash = randomBytes(32).toString('hex');
|
|
const filter = buildFilter(hash, inSet);
|
|
for (const el of inSet.slice(0, 30)) if (!filterMatchesAny(filter, hash, [el])) { bf++; console.log('member missed'); }
|
|
const key = keyFromBlockHash(hash); const F = BigInt(n) * M;
|
|
const setVals = new Set(inSet.map((e) => refRange(key, e, F)));
|
|
const probes = Array.from({ length: 500 }, () => randomBytes(25));
|
|
if (round % 2) probes[Math.floor(Math.random() * probes.length)] = inSet[Math.floor(Math.random() * n)];
|
|
const expectAny = probes.some((p) => setVals.has(refRange(key, p, F)));
|
|
const got = filterMatchesAny(filter, hash, probes);
|
|
if (got !== expectAny) { bf++; console.log('verdict mismatch'); }
|
|
got ? pos++ : neg++;
|
|
const expectWhich = probes.filter((p) => setVals.has(refRange(key, p, F))).map((p) => p.toString('hex')).sort();
|
|
const gotWhich = filterMatches(filter, hash, probes).map((p) => p.toString('hex')).sort();
|
|
if (expectWhich.join() !== gotWhich.join()) { bf++; console.log('filterMatches mismatch'); }
|
|
}
|
|
mfails += bf;
|
|
console.log(`matcher[${backend}]: ${bf === 0 ? 'OK' : bf + ' MISMATCHES'} (60 filters; ${pos} any-hit, ${neg} no-hit rounds)`);
|
|
}
|
|
|
|
// 3. Throughput: 3,300 watched scripts vs 200 filters of ~40 elements (a typical DGB block).
|
|
const scripts = Array.from({ length: 3300 }, () => randomBytes(25));
|
|
const filters = Array.from({ length: 200 }, () => { const h = randomBytes(32).toString('hex'); return { h, f: buildFilter(h, Array.from({ length: 40 }, () => randomBytes(25))) }; });
|
|
for (const backend of backends) {
|
|
gcsUseJs(backend === 'js');
|
|
const t0 = performance.now();
|
|
for (const { h, f } of filters) filterMatchesAny(f, h, scripts);
|
|
const ms = (performance.now() - t0) / filters.length;
|
|
console.log(`throughput[${backend}]: ${(1000 / ms).toFixed(0)} filters/s with 3,300 scripts (${ms.toFixed(2)} ms/filter)`);
|
|
}
|
|
let t0 = performance.now();
|
|
for (const { h } of filters.slice(0, 10)) { const key = keyFromBlockHash(h); const F = BigInt(40) * M; for (const s of scripts) refRange(key, s, F); }
|
|
console.log(`throughput[bigint reference, hashing only]: ${((performance.now() - t0) / 10).toFixed(1)} ms/filter`);
|
|
process.exit(fails + mfails === 0 ? 0 : 1);
|