From 690959a9834ef4b1adb90636a4d9ea57ad8ef4ea Mon Sep 17 00:00:00 2001 From: "Digibyte.X" Date: Sun, 11 Oct 2026 23:37:28 +0200 Subject: [PATCH] =?UTF-8?q?p2p:=20BIP158=20matching=20in=20WebAssembly=20(?= =?UTF-8?q?gcs.c),=20Number-based=20JS=20fallback;=20Blockbook=20upstream?= =?UTF-8?q?=20=E2=86=92=20Atomic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daemon's wallet scan hashed every watched script against every block filter with a BigInt SipHash: ~3 filters/s on the VPS once the gap limits went to 200/100 (~3,300 scripts), i.e. days for a one-year window. The matcher now lives in dgbpow.wasm next to the PoW hashes — scripts are uploaded once per scan, each filter is one call, the set is decoded once and each target binary-searched — ~5,500 filters/s locally, ~2,000/s on the VPS with block fetches. The JS path was rewritten with uint32 lanes and exact 32×32 partial products as a fallback (module missing or filter beyond its buffers). scripts/gcs-check.ts cross-checks both against a BigInt reference on random filters; `npm run gcs:test -w @dgb-wallet/p2p`. NOWNodes stopped accepting the proxy's API key ("Unknown API_key" on every endpoint), which is why the wallet showed "Blockbook HTTP 401" for every path. The vhost mirror now proxies /blockbook/ to Atomic Wallet's public Blockbook; ops/blockbook-upstream.sh switches the live vhost either way. --- ops/README.md | 3 +- ops/blockbook-upstream.sh | 50 +++++++ ops/nginx-digibyte-x.conf | 26 ++-- packages/p2p/native/build.sh | 6 +- packages/p2p/native/dgbpow.wasm | Bin 153904 -> 156185 bytes packages/p2p/native/gcs.c | 144 ++++++++++++++++++ packages/p2p/package.json | 3 +- packages/p2p/scripts/gcs-check.ts | 111 ++++++++++++++ packages/p2p/src/gcs.ts | 239 ++++++++++++++++++++++-------- packages/p2p/src/pow.ts | 7 +- packages/p2p/src/siphash.ts | 160 ++++++++++++-------- 11 files changed, 604 insertions(+), 145 deletions(-) create mode 100644 ops/blockbook-upstream.sh create mode 100644 packages/p2p/native/gcs.c create mode 100644 packages/p2p/scripts/gcs-check.ts diff --git a/ops/README.md b/ops/README.md index 8ca9a0e..7c5e6c4 100644 --- a/ops/README.md +++ b/ops/README.md @@ -7,7 +7,7 @@ web wallet needs: | Piece | Where | Notes | |---|---|---| -| nginx vhost | `/etc/nginx/sites-enabled/digibyte-x` — mirrored in [`nginx-digibyte-x.conf`](nginx-digibyte-x.conf) | SNI-selected on `digibyte.x`, self-signed cert pinned on-chain via the BNS `tls` record. Serves `/web/` (wallet), proxies `/p2p/*` → daemon and `/blockbook/*` → NOWNodes (API key injected server-side from `/etc/dgb-core/nownodes.env`), strict Content-Security-Policy on `/web/`. | +| nginx vhost | `/etc/nginx/sites-enabled/digibyte-x` — mirrored in [`nginx-digibyte-x.conf`](nginx-digibyte-x.conf) | SNI-selected on `digibyte.x`, self-signed cert pinned on-chain via the BNS `tls` record. Serves `/web/` (wallet), proxies `/p2p/*` → daemon and `/blockbook/*` → a public DGB Blockbook (Atomic Wallet's since 2026-10-11; NOWNodes before that, API key injected server-side from `/etc/dgb-core/nownodes.env` — switch with [`blockbook-upstream.sh`](blockbook-upstream.sh)), strict Content-Security-Policy on `/web/`. | | `dgb-daemon.service` | `/opt/dgb-wallet`, runs `packages/p2p/bin/dgb-daemon.ts` via tsx, binds `127.0.0.1:5174` | The wallet's only server-side component: header sync with reorg handling, BIP157/158 filter cache (packed 1000/file under `/var/lib/dgb-wallet/.dgb-wallet/p2p/packs/`), incremental balance scans remembered in `scans.json`, Dandelion++ broadcast with relay-echo verification. | | **No DigiByte Core, no Fulcrum** | — | Removed 2026-09-06. A full node with indexes needs ≥40 GB disk and ≥4 GB RAM; the VPS has neither, and the daemon does not use it. | @@ -125,6 +125,7 @@ If you'd rather understand each piece, see [MANUAL.md](MANUAL.md) — same steps |---|---| | `push.sh` | **Routine deploy from the dev box** — build, ship web bundle and/or daemon sources, restart, verify (see Production layout) | | `nginx-digibyte-x.conf` | Mirror of the live nginx vhost for digibyte.x (`@NOWNODES_API_KEY@` placeholder; render with `sed` as the file header shows) | +| `blockbook-upstream.sh` | `atomic` or `nownodes` — repoint the live `/blockbook/` proxy, test, reload, verify | | `setup.sh` | RPC-VPS bootstrap (Core + Caddy in front of it) — optional Core backend, not deployed | | `deploy-wallet.sh` | Wallet-site VPS bootstrap (web bundle at `/` + P2P daemon reverse-proxy) | | `deploy-wallet-web.sh` | Wallet-subpath bootstrap — plain production build (relative `./` base), ships bundle to `/web/`, renders Caddy import snippet | diff --git a/ops/blockbook-upstream.sh b/ops/blockbook-upstream.sh new file mode 100644 index 0000000..a971353 --- /dev/null +++ b/ops/blockbook-upstream.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Point the live digibyte.x vhost's /blockbook/ proxy at a Blockbook +# upstream, test the config, reload nginx and prove the route works. +# +# ops/blockbook-upstream.sh atomic # Atomic Wallet's public Blockbook (no key) +# ops/blockbook-upstream.sh nownodes # NOWNodes, key from /etc/dgb-core/nownodes.env on the VPS +# +# Runs over `ssh silentmode`; the previous vhost is kept under +# /etc/nginx/sites-enabled-stale-baks/. The repo mirror +# ops/nginx-digibyte-x.conf documents the same two blocks. +set -euo pipefail +target="${1:-}" +case "$target" in + atomic) host=digibyte.atomicwallet.io; keyhdr=0 ;; + nownodes) host=dgbbook.nownodes.io; keyhdr=1 ;; + *) echo "usage: $0 atomic|nownodes" >&2; exit 2 ;; +esac + +ssh silentmode "HOST=$host KEYHDR=$keyhdr bash -s" <<'REMOTE' +set -euo pipefail +F=/etc/nginx/sites-enabled/digibyte-x +mkdir -p /etc/nginx/sites-enabled-stale-baks +cp "$F" "/etc/nginx/sites-enabled-stale-baks/digibyte-x.bak.$(date +%s)" +KEY="" +if [ "$KEYHDR" = 1 ]; then + KEY="$(. /etc/dgb-core/nownodes.env; printf %s "$NOWNODES_API_KEY" | tr -d '\r\n ')" + [ -n "$KEY" ] || { echo "no NOWNODES_API_KEY in /etc/dgb-core/nownodes.env" >&2; exit 1; } +fi +HOST="$HOST" KEY="$KEY" python3 - "$F" <<'PY' +import os, re, sys +p = sys.argv[1]; s = open(p).read() +host, key = os.environ['HOST'], os.environ['KEY'] +a = s.index(' location /blockbook/ {'); b = s.index('\n }\n', a) + 7 +blk = s[a:b] +blk = re.sub(r'proxy_pass https://[^/]+/api/v2/;', f'proxy_pass https://{host}/api/v2/;', blk) +blk = re.sub(r'proxy_ssl_name [^;]+;', f'proxy_ssl_name {host};', blk) +blk = re.sub(r'proxy_set_header Host [^;]+;', f'proxy_set_header Host {host};', blk) +blk = re.sub(r'\n proxy_set_header api-key "[^"]*";', '', blk) +if key: + blk = blk.replace(' proxy_set_header X-Real-IP "";\n', f' proxy_set_header X-Real-IP "";\n proxy_set_header api-key "{key}";\n') +open(p, 'w').write(s[:a] + blk + s[b:]) +print('vhost now proxies /blockbook/ ->', host, '(api-key header)' if key else '(no key)') +PY +nginx -t +systemctl reload nginx +sleep 1 +code=$(curl -sk -o /dev/null -w '%{http_code}' --resolve digibyte.x:443:127.0.0.1 'https://digibyte.x/blockbook/') +echo "GET /blockbook/ via vhost -> HTTP $code" +[ "$code" = 200 ] || { echo "upstream not answering 200 — check the key / host" >&2; exit 1; } +REMOTE diff --git a/ops/nginx-digibyte-x.conf b/ops/nginx-digibyte-x.conf index 35f08c1..4fb0c2d 100644 --- a/ops/nginx-digibyte-x.conf +++ b/ops/nginx-digibyte-x.conf @@ -97,19 +97,25 @@ server { # individual browsers. (This VPS operator still can — the # wallet's Blockbook backend toggle carries an inline note.) location /blockbook/ { - # Upstream: NOWNodes DGB Blockbook (fully synced; api-key - # injected server-side so the browser never sees the credential). - # Same posture as Atomic — proxy hides individual user IPs from - # the upstream operator. Both Atomic and NOWNodes' Blockbook - # fork rejects raw bech32 address queries; `/xpub` works for - # both (server-side derivation covers all script kinds). - proxy_pass https://dgbbook.nownodes.io/api/v2/; + # Upstream: Atomic Wallet's public DGB Blockbook. The proxy hides + # individual user IPs from the upstream operator. Every public DGB + # Blockbook rejects raw bech32 address queries; `/xpub` works + # (server-side derivation covers all script kinds). + # + # NOWNodes (dgbbook.nownodes.io, api-key injected here from + # /etc/dgb-core/nownodes.env) was the upstream 2026-09-02..10-11 + # until the key stopped being accepted ("Unknown API_key" on every + # endpoint). To switch back with a fresh key: + # proxy_pass https://dgbbook.nownodes.io/api/v2/; + # proxy_ssl_name dgbbook.nownodes.io; + # proxy_set_header Host dgbbook.nownodes.io; + # proxy_set_header api-key "@NOWNODES_API_KEY@"; + proxy_pass https://digibyte.atomicwallet.io/api/v2/; proxy_ssl_server_name on; - proxy_ssl_name dgbbook.nownodes.io; - proxy_set_header Host dgbbook.nownodes.io; + proxy_ssl_name digibyte.atomicwallet.io; + proxy_set_header Host digibyte.atomicwallet.io; proxy_set_header X-Forwarded-For ""; proxy_set_header X-Real-IP ""; - proxy_set_header api-key "@NOWNODES_API_KEY@"; proxy_http_version 1.1; proxy_read_timeout 60s; proxy_hide_header Access-Control-Allow-Origin; diff --git a/packages/p2p/native/build.sh b/packages/p2p/native/build.sh index b912b7c..a678c86 100644 --- a/packages/p2p/native/build.sh +++ b/packages/p2p/native/build.sh @@ -14,10 +14,10 @@ MSYS_NO_PATHCONV=1 docker run --rm -v "$here":/work -w /work "$IMG" bash -c ' set -e emcc -O3 -DNDEBUG -fno-exceptions -fno-rtti \ -s STANDALONE_WASM=1 -s PURE_WASI=0 --no-entry \ - -s EXPORTED_FUNCTIONS=_in_ptr,_out_ptr,_hash_qubit,_hash_skein512,_hash_groestl512,_hash_odo \ - -s INITIAL_MEMORY=1048576 -s ALLOW_MEMORY_GROWTH=0 -s STACK_SIZE=262144 \ + -s EXPORTED_FUNCTIONS=_in_ptr,_out_ptr,_hash_qubit,_hash_skein512,_hash_groestl512,_hash_odo,_gcs_scripts_ptr,_gcs_offsets_ptr,_gcs_key_ptr,_gcs_filter_ptr,_gcs_matches_ptr,_gcs_limits_ptr,_gcs_set_count,_gcs_match \ + -s INITIAL_MEMORY=33554432 -s ALLOW_MEMORY_GROWTH=0 -s STACK_SIZE=262144 \ -Isrc \ - dgbpow.c odo_glue.cpp \ + dgbpow.c gcs.c odo_glue.cpp \ src/luffa.cpp src/cubehash.cpp src/shavite.cpp src/simd.cpp src/echo.cpp \ src/skein.cpp src/groestl.cpp src/odocrypt.cpp src/KeccakP-800-reference.cpp \ -o dgbpow.wasm diff --git a/packages/p2p/native/dgbpow.wasm b/packages/p2p/native/dgbpow.wasm index b56e948b1a106d1320a63cb76c155febcf1024f1..ebcd8e91f5313e9c227e57c143a33438d5f884db 100644 GIT binary patch delta 3612 zcmai1YiwLc6`r|s_wKImKJM;%y=yy(=3ZPB^C%BN0tgz8_5*$@Q(sgNg)N%QnhLgeBiTlWy|oR%vl~D$3tC zHa$E&Hg)7geR}vreTpSDa{SPt>G2kla!F@m{Ht1?&e819k)!qTDGhXKgJYxhvBTp{ z7p_)0dgR!VmWNDMuQFaA9y|V(FV|T%O>;64hgoj%*+JgB`0pLt`_f6y6n8o6`u%H3 z?8|kSI`;~vVZE`7-{e*Lq)=>^CDkS7->J?uCiu1#G%)olD-ZGgoL5%@N7>KDl>3NG}+s&dQuZM@Q0`qB#M=v_y7 zzVX|;#(1^y(Yrsnz1o-IuC&!PCX*9hwNEn1yu4)U@64+CkPwhY;?+@Ypw1qLYcZ(Pd9U>i3sbQVngItDZPw>*CD`$KA$ohTS8g_J0 zF_(LqiFoFFpEv6^{KsyMy0PuD8k+E$3{u(R)(jV01~XhN3}(34Hkje!gux6KI|g&2 zHXA3~+?wIyl)(%arwwMfxXWOMi(P{mF3uRN%|$BC8qjcY&R~X%^9D0q>>12(alv4Q zi;D&`TwEe7P1#=~cI9v1pXtH*AO1zn;pN63e!0%eD?j?xast?W7r!f_9#om@5zT0t z)ikGRUQR4Ov5C$bwK7(WfEvh2+%C zMad_Y{4@b#2vV4wL@bL|?SqUQMAIrcc~bGoy;n$$~1?0&j~~Z`11%XQw>K(@em7VQp87F*120&R4zE7_>~9`>@P*~K z*u0<7?M-+j!arX=fAzW7U-;?k5M*UKeBqrN+8=yUX(=QCag^37w3O6-v={T0s@f90fh< zfytn-7q4;FPsoh#$Q)WU?c0jcehP|DU@1Cy zun~3k%zPkb0vBw7SO5?@p?5|m=AmeZ@(eVinIJ{d0!b5L`HZy5m)n%W<2-`VR={m7 zm#L<39)TplB5GT9%_BUHFB4oUh1+l**cE0j0`FN0Ke6B^=}x=;SN7nlQW2lXC7pyV zEY6ZHZnRSEBg_duydaZvz5x>9&aD8V-wKGZx)rcOZ!5sVR2xXP)?ssggf%KMAIj5| zMw!Isb&9CQkQkkjc2@Pa-Ij(UQ`mMh&&Yf;ojR4sp^*6WLvkqdMujw zC?dn~8lRUqthPBBYPpW6>73+k59FH8Hz8A-!vABuDQsJsmD`q4hjstgl**$OHW`an z5l~6!c*SOs%`__td!lM2O|Vox$g(qwY+u7q{uJIMJ8JWZSuLj;oz*s{XJkTl%_7B3 z?H0IOq;Oz%3xrfI%+Xy?85AjB=uRp7MRW%}m5QH~XuBTt9*(@nJag!LDeepk{lO^S zK-nX~6{%|RVBj(J9KF%u_ z{O=uHA6vyZ7Y?4hj=}y|lkF@GWYuotIlUnp(GvY}E!Fo_z+i3ORR;8$*2XRLb zDfrDIB3=X)5%H@*L;b*lq9B4jiN%9GD5wWp@T-ELZ#L#Z`0sE2GjHbQ&D*{3EPiJw z-pjV`+YO8{Xo?jL-)BXJKk$oy$bD^SMFS_n=k;}97y!UAgqtXlC^yUq44@q`0fdNJ z;DbCfp0;C;z&`8k4SFCC>0l6M&#H~bfJv}{C60~7>I`N_nN;sVKSqP6Fkd)XmrJ(H zaOEh=ZICLMf<_Bh;7~ZFMxztJDGl)BDG;SUVn~1!+|FfDU*_6y=69}EFG$Wi-+&9Q ztUT90UldWRqaKru@s?SicZim$^*DTqeIVA%e`7VO;%s z%R7v`(X$fuOV1*(g6X}T`O4Z>LxCsR3Wjl#3Cm^L$r&~dEl*DAU4q{I7-JPTVNTnM zQA`cfXb6sO?SurTGSks1FgNL89wYb%hqgJ8!R<{s!Pc1tOej%JsHv|VW4ik`;vV<< zKDT{(4>&btd3ny-Ig{hzc#_6%^*GNzS$c9bc(r}F8OxYkU=;8qx4G3Q%x&q%s_J8! z-4;(@L#4n;r;)3;zc196rYbnsw7_wMbP{H>MHkZ zu={MGH5qCu)S6J8P(`TeP-{c23$;Gf3~EMBJ^-(Y$*`I8dM zj)z7j83vYij{WMaiR0MtTtn*Ch?n;)h7Jf#9-4T+@$$nq_^=+F@}(oNWa%9I${4Qc w diff --git a/packages/p2p/native/gcs.c b/packages/p2p/native/gcs.c new file mode 100644 index 0000000..0d566b2 --- /dev/null +++ b/packages/p2p/native/gcs.c @@ -0,0 +1,144 @@ +// BIP158 compact-filter matching, compiled to WebAssembly alongside the +// proof-of-work hashes (same module, dgbpow.wasm). +// +// Why native: a wallet scan tests every watched script (a few thousand +// with gap limits 200/100 over ~10 derivation profiles) against every +// block filter, and the SipHash key is per block, so nothing can be +// precomputed across blocks. In JavaScript that is ~3 µs per script +// even with int32 lanes; here it is well under 100 ns, which turns a +// one-year scan (2.1M DigiByte blocks) from hours into minutes. +// +// Interface (pointer-free for the caller, fixed static buffers): +// gcs_scripts_ptr() → byte area the JS side packs the scripts into +// gcs_offsets_ptr() → uint32[n+1] start offsets of each script +// gcs_set_count(n) → declares how many scripts are uploaded +// gcs_key_ptr() → 16-byte SipHash key (block hash LE, first 16 bytes) +// gcs_filter_ptr() → the GCS body (after the varint N), ≤ GCS_FILTER_BYTES +// gcs_match(n_set, body_len, want_all) +// → number of matching scripts (0 = none); with want_all=0 stops +// at the first match. Matching script indices are written to +// gcs_matches_ptr() (uint32[]), in no particular order. +// gcs_limits_ptr() → uint32[3] { max scripts, script bytes, filter bytes } +// +// Build: see build.sh (emscripten in docker). + +#include +#include + +#define EXPORT __attribute__((used, visibility("default"))) + +#define GCS_MAX_SCRIPTS 65536 +#define GCS_SCRIPT_BYTES (4u * 1024 * 1024) +#define GCS_FILTER_BYTES (2u * 1024 * 1024) +#define GCS_SET_CAP (1u << 20) /* decoded set values kept for binary search */ +#define GCS_P 19 +#define GCS_M 784931ull + +static uint8_t g_scripts[GCS_SCRIPT_BYTES]; +static uint32_t g_offsets[GCS_MAX_SCRIPTS + 1]; +static uint32_t g_count = 0; +static uint8_t g_key[16]; +static uint8_t g_filter[GCS_FILTER_BYTES]; +static uint32_t g_matches[GCS_MAX_SCRIPTS]; +static uint64_t g_set[GCS_SET_CAP]; +static uint32_t g_limits[3] = { GCS_MAX_SCRIPTS, GCS_SCRIPT_BYTES, GCS_FILTER_BYTES }; + +EXPORT uint8_t* gcs_scripts_ptr(void) { return g_scripts; } +EXPORT uint32_t* gcs_offsets_ptr(void) { return g_offsets; } +EXPORT uint8_t* gcs_key_ptr(void) { return g_key; } +EXPORT uint8_t* gcs_filter_ptr(void) { return g_filter; } +EXPORT uint32_t* gcs_matches_ptr(void) { return g_matches; } +EXPORT uint32_t* gcs_limits_ptr(void) { return g_limits; } +EXPORT void gcs_set_count(uint32_t n) { g_count = n > GCS_MAX_SCRIPTS ? GCS_MAX_SCRIPTS : n; } + +// ---- SipHash-2-4 (Bitcoin Core src/crypto/siphash.cpp semantics) ---- +#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) +#define SIPROUND do { \ + v0 += v1; v1 = ROTL(v1, 13); v1 ^= v0; v0 = ROTL(v0, 32); \ + v2 += v3; v3 = ROTL(v3, 16); v3 ^= v2; \ + v0 += v3; v3 = ROTL(v3, 21); v3 ^= v0; \ + v2 += v1; v1 = ROTL(v1, 17); v1 ^= v2; v2 = ROTL(v2, 32); \ + } while (0) + +static inline uint64_t rd64(const uint8_t* p) { + uint64_t v; memcpy(&v, p, 8); return v; // wasm is little-endian +} + +static inline uint64_t siphash24(uint64_t k0, uint64_t k1, const uint8_t* m, uint32_t len) { + uint64_t v0 = k0 ^ 0x736f6d6570736575ull; + uint64_t v1 = k1 ^ 0x646f72616e646f6dull; + uint64_t v2 = k0 ^ 0x6c7967656e657261ull; + uint64_t v3 = k1 ^ 0x7465646279746573ull; + uint32_t full = len & ~7u, i; + for (i = 0; i < full; i += 8) { + uint64_t w = rd64(m + i); + v3 ^= w; SIPROUND; SIPROUND; v0 ^= w; + } + uint64_t t = ((uint64_t)(len & 0xff)) << 56; + switch (len & 7) { + case 7: t |= ((uint64_t)m[i + 6]) << 48; /* fallthrough */ + case 6: t |= ((uint64_t)m[i + 5]) << 40; /* fallthrough */ + case 5: t |= ((uint64_t)m[i + 4]) << 32; /* fallthrough */ + case 4: t |= ((uint64_t)m[i + 3]) << 24; /* fallthrough */ + case 3: t |= ((uint64_t)m[i + 2]) << 16; /* fallthrough */ + case 2: t |= ((uint64_t)m[i + 1]) << 8; /* fallthrough */ + case 1: t |= ((uint64_t)m[i]); + } + v3 ^= t; SIPROUND; SIPROUND; v0 ^= t; + v2 ^= 0xff; + SIPROUND; SIPROUND; SIPROUND; SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} + +// Decode the Golomb-Rice set (unary quotient, P-bit remainder, +// cumulative) into g_set. Returns the number of values decoded (short +// on a truncated body). +static uint32_t decode_set(uint32_t n_set, uint32_t body_len) { + const uint8_t* buf = g_filter; + uint32_t bits = body_len * 8, pos = 0, s; + uint64_t last = 0; + for (s = 0; s < n_set; s++) { + uint64_t q = 0; + while (pos < bits && ((buf[pos >> 3] >> (7 - (pos & 7))) & 1)) { q++; pos++; } + if (pos + 1 + GCS_P > bits) return s; + pos++; + uint64_t r = 0; int k; + for (k = 0; k < GCS_P; k++, pos++) r = (r << 1) | ((buf[pos >> 3] >> (7 - (pos & 7))) & 1); + last += (q << GCS_P) | r; + g_set[s] = last; + } + return s; +} + +static inline int in_set(uint64_t v, uint32_t n) { + uint32_t lo = 0, hi = n; + while (lo < hi) { + uint32_t mid = lo + ((hi - lo) >> 1); + if (g_set[mid] < v) lo = mid + 1; else hi = mid; + } + return lo < n && g_set[lo] == v; +} + +// Decode the set once (it is already sorted), then hash each script and +// binary-search its target. No sort of the script targets is needed, +// and a typical DigiByte block (tens of elements) is a 5-6 step search. +// Returns -1 when the filter is beyond the static buffers (the caller +// falls back to the JS matcher), never a silently partial answer. +EXPORT int32_t gcs_match(uint32_t n_set, uint32_t body_len, int32_t want_all) { + if (g_count == 0 || n_set == 0) return 0; + if (n_set > GCS_SET_CAP || body_len > GCS_FILTER_BYTES) return -1; + uint32_t n = decode_set(n_set, body_len); + if (n == 0) return 0; + uint64_t k0 = rd64(g_key), k1 = rd64(g_key + 8); + uint64_t F = (uint64_t)n_set * GCS_M; + uint32_t i, found = 0; + for (i = 0; i < g_count; i++) { + uint64_t h = siphash24(k0, k1, g_scripts + g_offsets[i], g_offsets[i + 1] - g_offsets[i]); + uint64_t target = (uint64_t)(((unsigned __int128)h * F) >> 64); + if (in_set(target, n)) { + g_matches[found++] = i; + if (!want_all) return (int32_t)found; + } + } + return (int32_t)found; +} diff --git a/packages/p2p/package.json b/packages/p2p/package.json index c66fd1d..6f423ef 100644 --- a/packages/p2p/package.json +++ b/packages/p2p/package.json @@ -24,7 +24,8 @@ "blocks": "tsx scripts/block-fetch-demo.ts", "cache": "tsx scripts/cache-demo.ts", "daemon": "tsx bin/dgb-daemon.ts", - "pow:test": "tsx scripts/pow-vectors.ts" + "pow:test": "tsx scripts/pow-vectors.ts", + "gcs:test": "tsx scripts/gcs-check.ts" }, "dependencies": { "@bitcoinerlab/secp256k1": "^1.1.1", diff --git a/packages/p2p/scripts/gcs-check.ts b/packages/p2p/scripts/gcs-check.ts new file mode 100644 index 0000000..f315e5e --- /dev/null +++ b/packages/p2p/scripts/gcs-check.ts @@ -0,0 +1,111 @@ +// Cross-check the Number-based SipHash / GCS matcher against a straight +// BigInt reference (the previous implementation), on random filters and +// elements, and report throughput for a realistic watched-script set. +// +// npx tsx packages/p2p/scripts/gcs-check.ts +import { randomBytes } from 'node:crypto'; +import { filterMatchesAny, filterMatches, keyFromBlockHash, gcsBackend, gcsUseJs, P, M } from '../src/gcs.js'; +import { siphash24 } from '../src/siphash.js'; + +// ---- reference (BigInt) implementation ---- +const MASK64 = 0xffffffffffffffffn; +const rot = (v: bigint, n: number): bigint => ((v << BigInt(n)) & MASK64) | (v >> BigInt(64 - n)); +function sipRound(s: bigint[]): void { + let [v0, v1, v2, v3] = s; + v0 = (v0 + v1) & MASK64; v1 = rot(v1, 13); v1 ^= v0; v0 = rot(v0, 32); + v2 = (v2 + v3) & MASK64; v3 = rot(v3, 16); v3 ^= v2; + v0 = (v0 + v3) & MASK64; v3 = rot(v3, 21); v3 ^= v0; + v2 = (v2 + v1) & MASK64; v1 = rot(v1, 17); v1 ^= v2; v2 = rot(v2, 32); + s[0] = v0; s[1] = v1; s[2] = v2; s[3] = v3; +} +function refSiphash(key: Buffer, msg: Buffer): bigint { + const k0 = key.readBigUInt64LE(0), k1 = key.readBigUInt64LE(8); + const s = [k0 ^ 0x736f6d6570736575n, k1 ^ 0x646f72616e646f6dn, k0 ^ 0x6c7967656e657261n, k1 ^ 0x7465646279746573n]; + const full = Math.floor(msg.length / 8); + for (let i = 0; i < full; i++) { const m = msg.readBigUInt64LE(i * 8); s[3] ^= m; sipRound(s); sipRound(s); s[0] ^= m; } + const tail = Buffer.alloc(8); msg.subarray(full * 8).copy(tail); tail[7] = msg.length & 0xff; + const m = tail.readBigUInt64LE(0); s[3] ^= m; sipRound(s); sipRound(s); s[0] ^= m; s[2] ^= 0xffn; + sipRound(s); sipRound(s); sipRound(s); sipRound(s); + return (s[0] ^ s[1] ^ s[2] ^ s[3]) & MASK64; +} +const refRange = (key: Buffer, el: Buffer, F: bigint): bigint => (refSiphash(key, el) * F) >> 64n; + +// ---- BIP158 encoder (for building test filters) ---- +function encodeVarint(n: number): Buffer { + if (n < 0xfd) return Buffer.from([n]); + if (n <= 0xffff) { const b = Buffer.alloc(3); b[0] = 0xfd; b.writeUInt16LE(n, 1); return b; } + const b = Buffer.alloc(5); b[0] = 0xfe; b.writeUInt32LE(n, 1); return b; +} +function buildFilter(blockHash: string, elements: Buffer[]): Buffer { + const key = keyFromBlockHash(blockHash); + const F = BigInt(elements.length) * M; + const vals = [...new Set(elements.map((e) => refRange(key, e, F)))].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + const bits: number[] = []; + let last = 0n; + for (const v of vals) { + const d = v - last; last = v; + const q = d >> BigInt(P), r = d & ((1n << BigInt(P)) - 1n); + for (let i = 0n; i < q; i++) bits.push(1); + bits.push(0); + for (let i = P - 1; i >= 0; i--) bits.push(Number((r >> BigInt(i)) & 1n)); + } + const body = Buffer.alloc(Math.ceil(bits.length / 8)); + bits.forEach((b, i) => { if (b) body[i >> 3] |= 0x80 >> (i & 7); }); + // N is the element count the encoder hashed with (duplicates collapse + // in the set but N·M stays the range), exactly as Core writes it. + return Buffer.concat([encodeVarint(elements.length), body]); +} + +// 1. SipHash agreement on random keys / lengths 0..40 (incl. the 7-byte tail cases). +let fails = 0; +for (let i = 0; i < 2000; i++) { + const key = randomBytes(16), msg = randomBytes(i % 41); + if (siphash24(key, msg) !== refSiphash(key, msg)) { fails++; if (fails < 5) console.log('siphash mismatch len', msg.length); } +} +console.log(`siphash: ${fails === 0 ? 'OK' : fails + ' MISMATCHES'} (2000 random messages)`); + +// 2. Matcher agreement: random filters (1..3000 elements), random +// probes, plus probes that deliberately include members so the +// positive path is exercised. Run for each backend available. +let mfails = 0; +const backends: Array<'wasm' | 'js'> = gcsBackend() === 'wasm' ? ['wasm', 'js'] : ['js']; +if (backends[0] === 'js') console.log('note: dgbpow.wasm without gcs_match — only the JS path is checked'); +for (const backend of backends) { + gcsUseJs(backend === 'js'); + let bf = 0, pos = 0, neg = 0; + for (let round = 0; round < 60; round++) { + const n = 1 + Math.floor(Math.random() * (round < 50 ? 60 : 3000)); + const inSet = Array.from({ length: n }, () => randomBytes(20 + Math.floor(Math.random() * 15))); + const hash = randomBytes(32).toString('hex'); + const filter = buildFilter(hash, inSet); + for (const el of inSet.slice(0, 30)) if (!filterMatchesAny(filter, hash, [el])) { bf++; console.log('member missed'); } + const key = keyFromBlockHash(hash); const F = BigInt(n) * M; + const setVals = new Set(inSet.map((e) => refRange(key, e, F))); + const probes = Array.from({ length: 500 }, () => randomBytes(25)); + if (round % 2) probes[Math.floor(Math.random() * probes.length)] = inSet[Math.floor(Math.random() * n)]; + const expectAny = probes.some((p) => setVals.has(refRange(key, p, F))); + const got = filterMatchesAny(filter, hash, probes); + if (got !== expectAny) { bf++; console.log('verdict mismatch'); } + got ? pos++ : neg++; + const expectWhich = probes.filter((p) => setVals.has(refRange(key, p, F))).map((p) => p.toString('hex')).sort(); + const gotWhich = filterMatches(filter, hash, probes).map((p) => p.toString('hex')).sort(); + if (expectWhich.join() !== gotWhich.join()) { bf++; console.log('filterMatches mismatch'); } + } + mfails += bf; + console.log(`matcher[${backend}]: ${bf === 0 ? 'OK' : bf + ' MISMATCHES'} (60 filters; ${pos} any-hit, ${neg} no-hit rounds)`); +} + +// 3. Throughput: 3,300 watched scripts vs 200 filters of ~40 elements (a typical DGB block). +const scripts = Array.from({ length: 3300 }, () => randomBytes(25)); +const filters = Array.from({ length: 200 }, () => { const h = randomBytes(32).toString('hex'); return { h, f: buildFilter(h, Array.from({ length: 40 }, () => randomBytes(25))) }; }); +for (const backend of backends) { + gcsUseJs(backend === 'js'); + const t0 = performance.now(); + for (const { h, f } of filters) filterMatchesAny(f, h, scripts); + const ms = (performance.now() - t0) / filters.length; + console.log(`throughput[${backend}]: ${(1000 / ms).toFixed(0)} filters/s with 3,300 scripts (${ms.toFixed(2)} ms/filter)`); +} +let t0 = performance.now(); +for (const { h } of filters.slice(0, 10)) { const key = keyFromBlockHash(h); const F = BigInt(40) * M; for (const s of scripts) refRange(key, s, F); } +console.log(`throughput[bigint reference, hashing only]: ${((performance.now() - t0) / 10).toFixed(1)} ms/filter`); +process.exit(fails + mfails === 0 ? 0 : 1); diff --git a/packages/p2p/src/gcs.ts b/packages/p2p/src/gcs.ts index 1ef5a60..1181d12 100644 --- a/packages/p2p/src/gcs.ts +++ b/packages/p2p/src/gcs.ts @@ -1,10 +1,26 @@ -import { siphash24 } from './siphash.js'; +import { sipKey, siphash24Pair, type SipKey } from './siphash.js'; import { decodeVarint } from './varint.js'; +import { loadPowWasm } from './pow.js'; // BIP158 Golomb-Coded Set filter operations. // Parameters for the "basic" filter type (0x00): export const P = 19; export const M = 784931n; +const M_NUM = 784931; + +// Two implementations of the same thing: +// +// • native — gcs.c inside ../native/dgbpow.wasm. The watched scripts +// are uploaded into the module once per script set (the scan driver +// passes the same array for a whole scan) and each filter is then +// one call. ~30 ns per script; a few thousand scripts × a year of +// DigiByte blocks is minutes, not hours. +// • JS — Number arithmetic throughout (a filter's range N·M is far +// below 2^53, and the one overflowing step, (h·F) >> 64, is done +// with 32×32-bit partial products). ~3 µs per script. Used when the +// module is missing or the script set exceeds its static buffers. +// +// Both are checked against a BigInt reference by scripts/gcs-check.ts. // Derive the SipHash key from a block hash: first 16 bytes of the // block hash's LITTLE-endian byte order. @@ -16,55 +32,145 @@ export function keyFromBlockHash(blockHashHex: string): Buffer { return Buffer.from(le.subarray(0, 16)); } -// Map an element to the filter's [0, N*M) range: fastReduce of the -// SipHash output over N*M. Matches BIP158's `hashToRange`. -function hashToRange(key: Buffer, element: Buffer, F: bigint): bigint { - const h = siphash24(key, element); - // (h * F) >> 64 — do it in bigint to avoid precision loss. - return (h * F) >> 64n; +// ---------------------------------------------------------------- native +interface Native { + mem: Uint8Array; + scriptsPtr: number; offsetsPtr: number; keyPtr: number; filterPtr: number; matchesPtr: number; + maxScripts: number; scriptBytes: number; filterBytes: number; + setCount: (n: number) => void; + match: (nSet: number, bodyLen: number, wantAll: number) => number; +} +let native: Native | null | undefined; // undefined = not tried yet +let forceJs = false; +/** Test hook: `gcsUseJs(true)` disables the native path for this process. */ +export function gcsUseJs(on: boolean): void { forceJs = on; uploaded = null; } +export function gcsBackend(): 'wasm' | 'js' { return !forceJs && getNative() ? 'wasm' : 'js'; } + +function getNative(): Native | null { + if (native !== undefined) return native; + try { + const w = loadPowWasm(); + const x = w.exports; + if (typeof x.gcs_match !== 'function') { native = null; return null; } + const limitsPtr = (x.gcs_limits_ptr as () => number)(); + const limits = new Uint32Array(w.memory.buffer, limitsPtr, 3); + native = { + mem: w.mem, + scriptsPtr: (x.gcs_scripts_ptr as () => number)(), + offsetsPtr: (x.gcs_offsets_ptr as () => number)(), + keyPtr: (x.gcs_key_ptr as () => number)(), + filterPtr: (x.gcs_filter_ptr as () => number)(), + matchesPtr: (x.gcs_matches_ptr as () => number)(), + maxScripts: limits[0], scriptBytes: limits[1], filterBytes: limits[2], + setCount: x.gcs_set_count as (n: number) => void, + match: x.gcs_match as (nSet: number, bodyLen: number, wantAll: number) => number, + }; + } catch { + native = null; + } + return native; } -// Read one Golomb-Rice-coded value: unary prefix (`q` ones, then 0) -// followed by P bits of remainder. Value = q*2^P + r. -class BitReader { - private byte = 0; - private bit = 0; - constructor(private buf: Buffer) {} - readBit(): number { - if (this.byte >= this.buf.length) throw new Error('BitReader: past end of buffer'); - const b = (this.buf[this.byte] >> (7 - this.bit)) & 1; - if (++this.bit === 8) { - this.bit = 0; - this.byte++; - } - return b; - } - readBits(n: number): bigint { - let out = 0n; - for (let i = 0; i < n; i++) out = (out << 1n) | BigInt(this.readBit()); - return out; +// The script set currently uploaded into the module, by array identity +// and length (the scan driver never mutates its array mid-scan, but a +// grown array is re-uploaded anyway). +let uploaded: { arr: Buffer[]; len: number } | null = null; +function ensureUploaded(n: Native, elements: Buffer[]): boolean { + if (uploaded && uploaded.arr === elements && uploaded.len === elements.length) return true; + if (elements.length > n.maxScripts) return false; + let total = 0; + for (const e of elements) total += e.length; + if (total > n.scriptBytes) return false; + const offsets = new Uint32Array(n.mem.buffer, n.offsetsPtr, elements.length + 1); + let off = 0; + for (let i = 0; i < elements.length; i++) { + offsets[i] = off; + n.mem.set(elements[i], n.scriptsPtr + off); + off += elements[i].length; } + offsets[elements.length] = off; + n.setCount(elements.length); + uploaded = { arr: elements, len: elements.length }; + return true; } -// Decode all values in a Golomb-coded set into a sorted array. -// The set body is what `cfilter.filter` returns AFTER the varint N -// prefix has been consumed. -function decodeGcsValues(body: Buffer, n: number): bigint[] { - const reader = new BitReader(body); - const values: bigint[] = new Array(n); - let last = 0n; +function nativeMatch(n: Native, filterBody: Buffer, blockHash: string, elements: Buffer[], wantAll: boolean): number | null { + if (!ensureUploaded(n, elements)) return null; + const { value: nBig, bytesRead } = decodeVarint(filterBody, 0); + const nSet = Number(nBig); + if (nSet === 0) return 0; + const body = filterBody.subarray(bytesRead); + if (body.length > n.filterBytes) return null; + n.mem.set(keyFromBlockHash(blockHash), n.keyPtr); + n.mem.set(body, n.filterPtr); + const r = n.match(nSet, body.length, wantAll ? 1 : 0); + return r < 0 ? null : r; // -1: filter too big for the module's buffers → JS +} + +// -------------------------------------------------------------------- JS +// Exact 32×32 → 64-bit product as [hi, lo] uint32 (via 16-bit halves). +const mulOut = new Float64Array(2); +function mul32(a: number, b: number): void { + const al = a & 0xffff, ah = a >>> 16, bl = b & 0xffff, bh = b >>> 16; + const ll = al * bl, mid = al * bh + ah * bl, hh = ah * bh; + const lo = ll + (mid % 65536) * 65536; + const carry = Math.floor(lo / 4294967296); + mulOut[1] = lo - carry * 4294967296; + mulOut[0] = hh + Math.floor(mid / 65536) + carry; +} + +// Map an element to the filter's [0, N·M) range: fastReduce of the +// SipHash output over N·M, i.e. (h · F) >> 64. Matches BIP158's +// `hashToRange`. `F` is split once per filter into fh·2^32 + fl. +const hashOut = new Uint32Array(2); +function hashToRange(key: SipKey, element: Buffer, fh: number, fl: number): number { + siphash24Pair(key, element, hashOut); + const hi = hashOut[0], lo = hashOut[1]; + // h·F = hi·fh·2^64 + (hi·fl + lo·fh)·2^32 + lo·fl + mul32(lo, fl); const zHi = mulOut[0]; + mul32(hi, fl); const xHi = mulOut[0], xLo = mulOut[1]; + mul32(lo, fh); const yHi = mulOut[0], yLo = mulOut[1]; + const l = xLo + yLo + zHi; + const c = Math.floor(l / 4294967296); + return hi * fh + xHi + yHi + c; +} + +function splitF(n: number): { fh: number; fl: number } { + const F = n * M_NUM; + const fh = Math.floor(F / 4294967296); + return { fh, fl: F - fh * 4294967296 }; +} + +// Decode all values of a Golomb-Rice-coded set (unary quotient, P-bit +// remainder, cumulative) into a sorted Float64Array. The body is what +// `cfilter.filter` returns AFTER the varint N prefix has been consumed. +function decodeGcsValues(body: Buffer, n: number): Float64Array { + const values = new Float64Array(n); + const bitLen = body.length * 8; + let pos = 0; + let last = 0; + const bit = (p: number): number => (body[p >>> 3] >> (7 - (p & 7))) & 1; for (let i = 0; i < n; i++) { - // Unary quotient: count 1-bits until we see a 0. - let q = 0n; - while (reader.readBit() === 1) q++; - const r = reader.readBits(P); - const delta = (q << BigInt(P)) | r; - last += delta; + let q = 0; + while (pos < bitLen && bit(pos) === 1) { q++; pos++; } + if (pos + 1 + P > bitLen) throw new Error('GCS: past end of buffer'); + pos++; // the terminating 0 of the unary prefix + let r = 0; + for (let k = 0; k < P; k++) r = r * 2 + bit(pos++); + last += q * 524288 + r; // q·2^P + r values[i] = last; } return values; } +function targetsFor(key: SipKey, elements: Buffer[], n: number): Float64Array { + const { fh, fl } = splitF(n); + const t = new Float64Array(elements.length); + for (let i = 0; i < elements.length; i++) t[i] = hashToRange(key, elements[i], fh, fl); + return t.sort(); +} + +// ------------------------------------------------------------------ API // Test whether ANY of `elements` are in the filter. Returns true on // the first match; false if none matched. This is the primary // wallet-side operation: pass every scriptPubKey (and any input @@ -75,22 +181,22 @@ export function filterMatchesAny( elements: Buffer[], ): boolean { if (elements.length === 0) return false; - // Parse N (element count) as a varint at the start of the body. + if (!forceJs) { + const n = getNative(); + if (n) { + const r = nativeMatch(n, filterBody, blockHash, elements, false); + if (r !== null) return r > 0; + } + } const { value: nBig, bytesRead } = decodeVarint(filterBody, 0); const n = Number(nBig); if (n === 0) return false; - const setBody = filterBody.subarray(bytesRead); - - const F = BigInt(n) * M; - const key = keyFromBlockHash(blockHash); - const targets = elements - .map((el) => hashToRange(key, el, F)) - .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); - - // Two-pointer sweep over sorted values + sorted targets. - const setValues = decodeGcsValues(setBody, n); + const key = sipKey(keyFromBlockHash(blockHash)); + const targets = targetsFor(key, elements, n); + const setValues = decodeGcsValues(filterBody.subarray(bytesRead), n); let ti = 0; - for (const v of setValues) { + for (let i = 0; i < n; i++) { + const v = setValues[i]; while (ti < targets.length && targets[ti] < v) ti++; if (ti === targets.length) return false; if (targets[ti] === v) return true; @@ -107,26 +213,31 @@ export function filterMatches( elements: Buffer[], ): Buffer[] { if (elements.length === 0) return []; + if (!forceJs) { + const nat = getNative(); + if (nat) { + const r = nativeMatch(nat, filterBody, blockHash, elements, true); + if (r !== null) { + const idx = new Uint32Array(nat.mem.buffer, nat.matchesPtr, r); + return Array.from(idx, (i) => elements[i]); + } + } + } const { value: nBig, bytesRead } = decodeVarint(filterBody, 0); const n = Number(nBig); if (n === 0) return []; - const setBody = filterBody.subarray(bytesRead); - - const F = BigInt(n) * M; - const key = keyFromBlockHash(blockHash); - const withHash = elements.map((el) => ({ el, target: hashToRange(key, el, F) })); - withHash.sort((a, b) => (a.target < b.target ? -1 : a.target > b.target ? 1 : 0)); - - const setValues = decodeGcsValues(setBody, n); + const key = sipKey(keyFromBlockHash(blockHash)); + const { fh, fl } = splitF(n); + const withHash = elements.map((el) => ({ el, target: hashToRange(key, el, fh, fl) })); + withHash.sort((a, b) => a.target - b.target); + const setValues = decodeGcsValues(filterBody.subarray(bytesRead), n); const matched: Buffer[] = []; let ti = 0; - for (const v of setValues) { + for (let i = 0; i < n; i++) { + const v = setValues[i]; while (ti < withHash.length && withHash[ti].target < v) ti++; if (ti === withHash.length) break; - if (withHash[ti].target === v) { - matched.push(withHash[ti].el); - ti++; - } + while (ti < withHash.length && withHash[ti].target === v) matched.push(withHash[ti++].el); } return matched; } diff --git a/packages/p2p/src/pow.ts b/packages/p2p/src/pow.ts index 093fbf3..7980bd5 100644 --- a/packages/p2p/src/pow.ts +++ b/packages/p2p/src/pow.ts @@ -62,8 +62,11 @@ export function workFromBits(bits: number): bigint { } // ---- WebAssembly module (lazy, loaded once) ---- -interface Wasm { +export interface Wasm { mem: Uint8Array; + memory: WebAssembly.Memory; + /** Every export of the module — the GCS matcher (gcs.ts) picks its own. */ + exports: Record; inPtr: number; outPtr: number; hash_qubit: () => void; hash_skein512: () => void; hash_groestl512: () => void; hash_odo: (key: number) => void; } @@ -77,6 +80,8 @@ export function loadPowWasm(path = new URL('../native/dgbpow.wasm', import.meta. const memory = x.memory as WebAssembly.Memory; wasm = { mem: new Uint8Array(memory.buffer), + memory, + exports: x, inPtr: (x.in_ptr as () => number)(), outPtr: (x.out_ptr as () => number)(), hash_qubit: x.hash_qubit as () => void, diff --git a/packages/p2p/src/siphash.ts b/packages/p2p/src/siphash.ts index c9f09c1..69ce5f0 100644 --- a/packages/p2p/src/siphash.ts +++ b/packages/p2p/src/siphash.ts @@ -1,71 +1,101 @@ // SipHash-2-4 for BIP158 element hashing. Pure JS, no deps. -// Implementation follows Jean-Philippe Aumasson's reference and -// matches Bitcoin Core's src/crypto/siphash.cpp bit-for-bit. +// Matches Bitcoin Core's src/crypto/siphash.cpp bit-for-bit. // -// Public API: `siphash24(key: Buffer, msg: Buffer): bigint` — returns -// the 64-bit output as a bigint. BIP158 feeds this into a -// map-to-range step before checking against the Golomb-coded set. +// The 64-bit lanes are kept as pairs of uint32 Numbers rather than +// BigInts: a wallet watching a few thousand scripts hashes every one of +// them against every block filter (the key is per-block), and BigInt +// arithmetic made that ~100× slower than it needs to be — the 1-year +// scan dropped to ~3 filters/s once the gap limits went to 200/100. +// +// Public API: +// siphash24(key, msg): bigint — 64-bit output (kept for callers/tests) +// siphash24Pair(key, msg, out): void — writes [hi, lo] uint32 into `out` +// SipKey / sipKey(key) — pre-split key for the hot loop -const MASK64 = 0xffffffffffffffffn; +export interface SipKey { k0h: number; k0l: number; k1h: number; k1l: number } -function rot(v: bigint, n: number): bigint { - return ((v << BigInt(n)) & MASK64) | (v >> BigInt(64 - n)); -} - -// One SipRound: mixes four 64-bit lanes. -function sipRound(state: bigint[]): void { - let [v0, v1, v2, v3] = state; - v0 = (v0 + v1) & MASK64; - v1 = rot(v1, 13); - v1 ^= v0; - v0 = rot(v0, 32); - v2 = (v2 + v3) & MASK64; - v3 = rot(v3, 16); - v3 ^= v2; - v0 = (v0 + v3) & MASK64; - v3 = rot(v3, 21); - v3 ^= v0; - v2 = (v2 + v1) & MASK64; - v1 = rot(v1, 17); - v1 ^= v2; - v2 = rot(v2, 32); - state[0] = v0; - state[1] = v1; - state[2] = v2; - state[3] = v3; -} - -export function siphash24(key: Buffer, msg: Buffer): bigint { +export function sipKey(key: Buffer): SipKey { if (key.length !== 16) throw new Error('SipHash key must be 16 bytes'); - const k0 = key.readBigUInt64LE(0); - const k1 = key.readBigUInt64LE(8); - const state = [ - k0 ^ 0x736f6d6570736575n, - k1 ^ 0x646f72616e646f6dn, - k0 ^ 0x6c7967656e657261n, - k1 ^ 0x7465646279746573n, - ]; - const fullBlocks = Math.floor(msg.length / 8); - for (let i = 0; i < fullBlocks; i++) { - const m = msg.readBigUInt64LE(i * 8); - state[3] ^= m; - sipRound(state); - sipRound(state); - state[0] ^= m; - } - // Final block: 0..7 remaining bytes, LE, plus msg length in top byte. - const tail = Buffer.alloc(8); - msg.subarray(fullBlocks * 8).copy(tail); - tail[7] = msg.length & 0xff; - const m = tail.readBigUInt64LE(0); - state[3] ^= m; - sipRound(state); - sipRound(state); - state[0] ^= m; - state[2] ^= 0xffn; - sipRound(state); - sipRound(state); - sipRound(state); - sipRound(state); - return (state[0] ^ state[1] ^ state[2] ^ state[3]) & MASK64; + return { + k0l: key.readUInt32LE(0), k0h: key.readUInt32LE(4), + k1l: key.readUInt32LE(8), k1h: key.readUInt32LE(12), + }; +} + +// Lanes: v0 = (a0,b0) … v3 = (a3,b3) with a = high word, b = low word. +let a0 = 0, b0 = 0, a1 = 0, b1 = 0, a2 = 0, b2 = 0, a3 = 0, b3 = 0; + +// One SipRound on the module-level lanes (avoids allocation per call). +function sipRound(): void { + let t: number; + // v0 += v1 + t = b0 + b1; b0 = t >>> 0; a0 = (a0 + a1 + (t >= 0x100000000 ? 1 : 0)) >>> 0; + // v1 = rotl(v1, 13) + t = (a1 << 13) | (b1 >>> 19); b1 = ((b1 << 13) | (a1 >>> 19)) >>> 0; a1 = t >>> 0; + // v1 ^= v0 + a1 = (a1 ^ a0) >>> 0; b1 = (b1 ^ b0) >>> 0; + // v0 = rotl(v0, 32) + t = a0; a0 = b0; b0 = t; + // v2 += v3 + t = b2 + b3; b2 = t >>> 0; a2 = (a2 + a3 + (t >= 0x100000000 ? 1 : 0)) >>> 0; + // v3 = rotl(v3, 16) + t = (a3 << 16) | (b3 >>> 16); b3 = ((b3 << 16) | (a3 >>> 16)) >>> 0; a3 = t >>> 0; + // v3 ^= v2 + a3 = (a3 ^ a2) >>> 0; b3 = (b3 ^ b2) >>> 0; + // v0 += v3 + t = b0 + b3; b0 = t >>> 0; a0 = (a0 + a3 + (t >= 0x100000000 ? 1 : 0)) >>> 0; + // v3 = rotl(v3, 21) + t = (a3 << 21) | (b3 >>> 11); b3 = ((b3 << 21) | (a3 >>> 11)) >>> 0; a3 = t >>> 0; + // v3 ^= v0 + a3 = (a3 ^ a0) >>> 0; b3 = (b3 ^ b0) >>> 0; + // v2 += v1 + t = b2 + b1; b2 = t >>> 0; a2 = (a2 + a1 + (t >= 0x100000000 ? 1 : 0)) >>> 0; + // v1 = rotl(v1, 17) + t = (a1 << 17) | (b1 >>> 15); b1 = ((b1 << 17) | (a1 >>> 15)) >>> 0; a1 = t >>> 0; + // v1 ^= v2 + a1 = (a1 ^ a2) >>> 0; b1 = (b1 ^ b2) >>> 0; + // v2 = rotl(v2, 32) + t = a2; a2 = b2; b2 = t; +} + +/** SipHash-2-4; writes the 64-bit result as out[0] = high word, out[1] = low word. */ +export function siphash24Pair(key: SipKey, msg: Buffer, out: Uint32Array | number[]): void { + a0 = (key.k0h ^ 0x736f6d65) >>> 0; b0 = (key.k0l ^ 0x70736575) >>> 0; + a1 = (key.k1h ^ 0x646f7261) >>> 0; b1 = (key.k1l ^ 0x6e646f6d) >>> 0; + a2 = (key.k0h ^ 0x6c796765) >>> 0; b2 = (key.k0l ^ 0x6e657261) >>> 0; + a3 = (key.k1h ^ 0x74656462) >>> 0; b3 = (key.k1l ^ 0x79746573) >>> 0; + const len = msg.length; + const full = len - (len & 7); + let i = 0; + for (; i < full; i += 8) { + const ml = msg.readUInt32LE(i), mh = msg.readUInt32LE(i + 4); + a3 = (a3 ^ mh) >>> 0; b3 = (b3 ^ ml) >>> 0; + sipRound(); sipRound(); + a0 = (a0 ^ mh) >>> 0; b0 = (b0 ^ ml) >>> 0; + } + // Final block: remaining 0..7 bytes LE, message length in the top byte. + let ml = 0, mh = 0; + switch (len & 7) { + case 7: mh |= msg[i + 6] << 16; // falls through + case 6: mh |= msg[i + 5] << 8; // falls through + case 5: mh |= msg[i + 4]; // falls through + case 4: ml |= msg[i + 3] << 24; // falls through + case 3: ml |= msg[i + 2] << 16; // falls through + case 2: ml |= msg[i + 1] << 8; // falls through + case 1: ml |= msg[i]; + } + mh = (mh | ((len & 0xff) << 24)) >>> 0; ml >>>= 0; + a3 = (a3 ^ mh) >>> 0; b3 = (b3 ^ ml) >>> 0; + sipRound(); sipRound(); + a0 = (a0 ^ mh) >>> 0; b0 = (b0 ^ ml) >>> 0; + b2 = (b2 ^ 0xff) >>> 0; + sipRound(); sipRound(); sipRound(); sipRound(); + out[0] = (a0 ^ a1 ^ a2 ^ a3) >>> 0; + out[1] = (b0 ^ b1 ^ b2 ^ b3) >>> 0; +} + +const scratch = new Uint32Array(2); +export function siphash24(key: Buffer, msg: Buffer): bigint { + siphash24Pair(sipKey(key), msg, scratch); + return (BigInt(scratch[0]) << 32n) | BigInt(scratch[1]); }