From 283c0c3e2c4e885c43f303bfd59a664839426311 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Tue, 28 Jul 2026 10:07:12 +0200 Subject: [PATCH 01/11] Price candlesticks by gross volume instead of net deltas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A transaction that trades against several pools of the same token had its price computed as |SUM(sats_delta) / SUM(token_delta)|. Arbitrage routers buy from one pool and sell into the others, so the token deltas very nearly cancel, and the division turned real satoshis into a price no leg ever traded at. Mainnet token NWB printed 30,792,599.5 sats/unit from a 27-pool sweep whose legs all executed between 0.288 and 0.335 — a hundred-million-fold error, and the visible spike on its chart. 820 such transactions exist across 91 tokens. Price is now SUM(ABS(sats_delta)) / SUM(ABS(token_delta)): the volume-weighted average of the prices the transaction's legs actually executed at, which is always bounded by its cheapest and dearest leg. For single-direction transactions — 99.76% of all prints, including the OLA supply-shock crash — this is arithmetically identical to the old formula, so honest history is untouched. Transactions whose legs cancel exactly used to print nothing and let the candle carry the previous close; they now price from their legs like any other trade. ohlcv_1h is materialised with INSERT OR IGNORE and the materialiser only ever moves forward, so contaminated buckets could never be corrected in place. An ohlcv_version config key clears the table once when the pricing rule changes. The synchronous post-IBD backfill is skipped on that pass: it runs before rocket::build() returns, so rebuilding all of history there would refuse connections for the duration instead of falling back to the raw query path. Co-Authored-By: Claude Fable 5 --- src/db/cauldron/candlestick/mod.rs | 121 ++++++-------------- src/db/cauldron/candlestick/tests.rs | 162 +++++++++++++++++++++++++++ src/db/cauldron/ohlcv.rs | 130 +++++++++++++++++++-- src/main.rs | 25 ++++- src/rpc/candlesticks/tests.rs | 6 +- 5 files changed, 346 insertions(+), 98 deletions(-) diff --git a/src/db/cauldron/candlestick/mod.rs b/src/db/cauldron/candlestick/mod.rs index 5e8745a..bd40d46 100644 --- a/src/db/cauldron/candlestick/mod.rs +++ b/src/db/cauldron/candlestick/mod.rs @@ -71,7 +71,7 @@ impl PriceInterval { } fn aggregate_raw_trades( - all_trades: &[(i64, i64, i64, i64, i64)], + all_trades: &[(i64, i64, i64)], intervals: Vec, step_size: i64, mut found_first_trade: bool, @@ -87,7 +87,7 @@ fn aggregate_raw_trades( let mut first_trade_in_interval = true; while trade_index < all_trades.len() { - let (ts, signed_sats, signed_tokens, vol_sats, vol_tokens) = all_trades[trade_index]; + let (ts, vol_sats, vol_tokens) = all_trades[trade_index]; if ts < interval_start { trade_index += 1; continue; @@ -96,8 +96,8 @@ fn aggregate_raw_trades( break; } - if signed_tokens != 0 { - let price = (signed_sats as f64 / signed_tokens as f64).abs(); + if vol_tokens != 0 { + let price = vol_sats as f64 / vol_tokens as f64; if first_trade_in_interval { pi.open = Some(price); pi.high = price; @@ -206,59 +206,33 @@ fn fill_ohlcv_candles( (result, found_first_trade, last_close) } +/// Returns one row per transaction: `(effective_timestamp, volume_sats, volume_tokens)`. +/// +/// Volumes are gross sums of the absolute per-leg deltas, so the derived price +/// `volume_sats / volume_tokens` is the volume-weighted average of the prices actually +/// executed by that transaction's legs, and is therefore always bounded by the cheapest +/// and dearest leg. Summing the *signed* deltas instead lets a multi-pool arbitrage +/// transaction — which buys from one pool and sells into others — cancel almost all of +/// its token movement and divide real satoshis by a near-zero remainder, fabricating a +/// price no leg ever traded at. async fn fetch_raw_trades( pool: &SqlitePool, token_blob: &[u8], timestamp_start: i64, timestamp_end: i64, -) -> Result> { +) -> Result> { let sql = r#" -WITH per_pool_tx_raw AS ( - SELECT - phe.txid, - phe.effective_timestamp, - phe.utxo, - phe.sats_delta, - phe.token_delta, - phe.sequence - FROM pool_history_entry AS phe - WHERE phe.token_id = ? - AND phe.effective_timestamp >= ? - AND phe.effective_timestamp < ? -), -per_pool_tx AS ( - SELECT - txid, - effective_timestamp, - utxo, - MIN(sequence) AS min_sequence, - SUM(sats_delta) AS signed_sats_pool, - SUM(token_delta) AS signed_tokens_pool, - SUM(ABS(sats_delta)) AS volume_sats_pool, - SUM(ABS(token_delta)) AS volume_tokens_pool - FROM per_pool_tx_raw - GROUP BY txid, effective_timestamp, utxo -), -tx_trades AS ( - SELECT - txid, - effective_timestamp, - MIN(min_sequence) AS min_sequence, - SUM(signed_sats_pool) AS signed_sats, - SUM(signed_tokens_pool) AS signed_tokens, - SUM(volume_sats_pool) AS volume_sats, - SUM(volume_tokens_pool) AS volume_tokens - FROM per_pool_tx - GROUP BY txid, effective_timestamp -) SELECT - effective_timestamp, - signed_sats, - signed_tokens, - volume_sats, - volume_tokens -FROM tx_trades -ORDER BY effective_timestamp ASC, min_sequence ASC; + phe.effective_timestamp, + SUM(ABS(phe.sats_delta)) AS volume_sats, + SUM(ABS(phe.token_delta)) AS volume_tokens, + MIN(phe.sequence) AS min_sequence +FROM pool_history_entry AS phe +WHERE phe.token_id = ? + AND phe.effective_timestamp >= ? + AND phe.effective_timestamp < ? +GROUP BY phe.txid, phe.effective_timestamp +ORDER BY phe.effective_timestamp ASC, min_sequence ASC; "#; let rows = sqlx::query(sql) .bind(token_blob) @@ -269,7 +243,7 @@ ORDER BY effective_timestamp ASC, min_sequence ASC; Ok(rows .into_iter() - .map(|r| (r.get(0), r.get(1), r.get(2), r.get(3), r.get(4))) + .map(|r| (r.get(0), r.get(1), r.get(2))) .collect()) } @@ -282,43 +256,14 @@ async fn fetch_last_close_before( timestamp_end: i64, ) -> Result> { let sql = r#" -WITH per_pool_tx_raw AS ( - SELECT - phe.txid, - phe.effective_timestamp, - phe.utxo, - phe.sats_delta, - phe.token_delta, - phe.sequence - FROM pool_history_entry AS phe - WHERE phe.token_id = ? - AND phe.effective_timestamp < ? -), -per_pool_tx AS ( - SELECT - txid, - effective_timestamp, - utxo, - MIN(sequence) AS min_sequence, - SUM(sats_delta) AS signed_sats_pool, - SUM(token_delta) AS signed_tokens_pool - FROM per_pool_tx_raw - GROUP BY txid, effective_timestamp, utxo -), -tx_trades AS ( - SELECT - txid, - effective_timestamp, - MIN(min_sequence) AS min_sequence, - SUM(signed_sats_pool) AS signed_sats, - SUM(signed_tokens_pool) AS signed_tokens - FROM per_pool_tx - GROUP BY txid, effective_timestamp -) -SELECT ABS(CAST(signed_sats AS REAL) / CAST(signed_tokens AS REAL)) AS close_price -FROM tx_trades -WHERE signed_tokens != 0 -ORDER BY effective_timestamp DESC, min_sequence DESC +SELECT + CAST(SUM(ABS(phe.sats_delta)) AS REAL) / CAST(SUM(ABS(phe.token_delta)) AS REAL) AS close_price +FROM pool_history_entry AS phe +WHERE phe.token_id = ? + AND phe.effective_timestamp < ? +GROUP BY phe.txid, phe.effective_timestamp +HAVING SUM(ABS(phe.token_delta)) != 0 +ORDER BY phe.effective_timestamp DESC, MIN(phe.sequence) DESC LIMIT 1 "#; let row = sqlx::query(sql) diff --git a/src/db/cauldron/candlestick/tests.rs b/src/db/cauldron/candlestick/tests.rs index f6e1409..7ddeb4f 100644 --- a/src/db/cauldron/candlestick/tests.rs +++ b/src/db/cauldron/candlestick/tests.rs @@ -86,6 +86,168 @@ async fn insert_trade_at( .unwrap(); } +/// Insert one transaction that touches several pools, each leg with its own +/// `(sats_delta, token_delta)`. Models a router/arbitrage transaction. +async fn insert_multileg_trade_at( + conn: &mut sqlx::pool::PoolConnection, + token: &TokenID, + txid_byte: u8, + ts: u64, + legs: &[(i64, i64)], +) { + let txid = Txid::from_byte_array([txid_byte; 32]); + let block = BlockHash::all_zeros(); + + insert_mempool_tx(&mut **conn, &txid, ts).await.unwrap(); + insert_block_tx(&mut **conn, &txid, &block, ts as i64) + .await + .unwrap(); + + for (i, (sats_delta, token_delta)) in legs.iter().enumerate() { + let mut utxo_bytes = [txid_byte; 32]; + utxo_bytes[0] = i as u8; + let utxo = OutPointHash::from_byte_array(utxo_bytes); + + let mut pool_bytes = [txid_byte.wrapping_add(0x80); 32]; + pool_bytes[0] = i as u8; + let pool_hash = OutPointHash::from_byte_array(pool_bytes); + + // Post-trade reserves large enough to look like a real pool. + let cauldron = dummy_cauldron( + &txid, + &utxo, + token, + sats_delta.unsigned_abs() * 10, + token_delta.abs() * 10, + &PubkeyHash::all_zeros(), + ); + + insert_utxo_funding(&mut **conn, &vec![cauldron.clone()], &txid) + .await + .unwrap(); + pool::insert_pool_history_entry( + &mut **conn, + &pool_hash, + &cauldron, + Some(ts), + Some(ts), + *sats_delta, + *token_delta, + ) + .await + .unwrap(); + } +} + +/// Regression for the netting artifact: a multi-pool arbitrage transaction that buys +/// from one pool and sells into another nets its token movement to almost nothing. +/// Dividing the signed sums produced a price no leg traded at — on mainnet token NWB +/// (tx 1E84F4E9…1916, 27 legs) that printed 30,792,599.5 sats/unit against legs that +/// actually executed between 0.288 and 0.335. +#[tokio::test] +async fn test_multipool_arb_priced_by_gross_volume_not_net() { + let db = mock_db_pool(setup_db).await; + let token = TokenID::from_byte_array([0xC1; 32]); + let token_blob = token.to_blob(); + + // Real leg totals from the NWB transaction, collapsed to two legs. + let sell = (-446_491_239i64, 1_334_527_069i64); // executes at 0.334569 + let buy = (384_906_040i64, -1_334_527_067i64); // executes at 0.288421 + let mut conn = db.cauldron_w.acquire().await.unwrap(); + insert_multileg_trade_at(&mut conn, &token, 0x21, 1000, &[sell, buy]).await; + + let price = super::fetch_last_close_before(&db.cauldron_r, &token_blob, 2000) + .await + .unwrap() + .expect("arb transaction must price"); + + let low_leg = 384_906_040.0 / 1_334_527_067.0; + let high_leg = 446_491_239.0 / 1_334_527_069.0; + assert!( + price >= low_leg && price <= high_leg, + "price {price} must lie within the executed leg range [{low_leg}, {high_leg}]" + ); + + // Net-ratio pricing would divide 61,585,199 sats by 2 token units. + let net_ratio = 61_585_199.0 / 2.0; + assert!( + price < net_ratio / 1000.0, + "price {price} must not resemble the netting artifact {net_ratio}" + ); + + let expected = 831_397_279.0 / 2_669_054_136.0; + assert!((price - expected).abs() < 1e-9, "expected {expected}"); +} + +/// Every leg pointing the same way is the ordinary case: gross and net agree exactly, +/// so 99.76% of mainnet prints — including the OLA supply-shock crash — are untouched. +#[tokio::test] +async fn test_single_direction_multileg_price_matches_net_ratio() { + let db = mock_db_pool(setup_db).await; + let token = TokenID::from_byte_array([0xC2; 32]); + let token_blob = token.to_blob(); + + let legs = [(150_000i64, -3_000i64), (50_000, -1_000), (99_000, -2_000)]; + let mut conn = db.cauldron_w.acquire().await.unwrap(); + insert_multileg_trade_at(&mut conn, &token, 0x22, 1000, &legs).await; + + let price = super::fetch_last_close_before(&db.cauldron_r, &token_blob, 2000) + .await + .unwrap() + .expect("router transaction must price"); + + let signed_sats: i64 = legs.iter().map(|l| l.0).sum(); + let signed_tokens: i64 = legs.iter().map(|l| l.1).sum(); + let net_ratio = (signed_sats as f64 / signed_tokens as f64).abs(); + assert!( + (price - net_ratio).abs() < f64::EPSILON, + "single-direction transactions must be unaffected: {price} vs {net_ratio}" + ); +} + +/// A transaction whose legs cancel exactly used to print nothing at all (the chart +/// carried the previous close). Its legs are real executions and now price normally. +#[tokio::test] +async fn test_exactly_cancelling_legs_still_price() { + let db = mock_db_pool(setup_db).await; + let token = TokenID::from_byte_array([0xC3; 32]); + let token_blob = token.to_blob(); + + let mut conn = db.cauldron_w.acquire().await.unwrap(); + insert_multileg_trade_at( + &mut conn, + &token, + 0x23, + 1000, + &[(-9_000, 1_000), (11_000, -1_000)], + ) + .await; + + let price = super::fetch_last_close_before(&db.cauldron_r, &token_blob, 2000) + .await + .unwrap() + .expect("zero-net transaction must still price from its legs"); + + // 20_000 gross sats over 2_000 gross tokens, between the 9 and 11 leg prices. + assert!((price - 10.0).abs() < f64::EPSILON, "got {price}"); +} + +/// Legs that move no tokens cannot produce a price (division by zero volume). +#[tokio::test] +async fn test_token_less_legs_do_not_price() { + let db = mock_db_pool(setup_db).await; + let token = TokenID::from_byte_array([0xC4; 32]); + let token_blob = token.to_blob(); + + let mut conn = db.cauldron_w.acquire().await.unwrap(); + insert_multileg_trade_at(&mut conn, &token, 0x24, 1000, &[(5_000, 0), (7_000, 0)]).await; + + let price = super::fetch_last_close_before(&db.cauldron_r, &token_blob, 2000) + .await + .unwrap(); + assert!(price.is_none(), "no token movement means no price"); +} + #[tokio::test] async fn test_fetch_last_close_before_no_trades() { let db = mock_db_pool(setup_db).await; diff --git a/src/db/cauldron/ohlcv.rs b/src/db/cauldron/ohlcv.rs index 68b3c6a..fb168c0 100644 --- a/src/db/cauldron/ohlcv.rs +++ b/src/db/cauldron/ohlcv.rs @@ -3,9 +3,18 @@ // This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later. // A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html +use crate::db::cauldron::config::{config_get, config_set}; use anyhow::Result; use sqlx::{Row, SqlitePool}; +/// Bumped whenever the maths that fills `ohlcv_1h` changes, so buckets materialised by +/// an older rule are discarded instead of being served forever — `rebuild_range` uses +/// `INSERT OR IGNORE`, so existing rows are never corrected in place. +/// +/// 2: price switched from the signed net ratio to the gross volume ratio. +pub const OHLCV_VERSION: u32 = 2; +const OHLCV_VERSION_KEY: &str = "ohlcv_version"; + pub async fn create_table(pool: &SqlitePool) { sqlx::query( "CREATE TABLE IF NOT EXISTS ohlcv_1h ( @@ -26,6 +35,30 @@ pub async fn create_table(pool: &SqlitePool) { .expect("failed to create ohlcv_1h table"); } +/// Discards `ohlcv_1h` when it was materialised under an older pricing rule. +/// +/// Returns `true` when the table was cleared. The caller must then leave repopulation +/// to the incremental background task: re-materialising the whole history inline would +/// hold up Rocket's startup for as long as it takes, and the raw query path already +/// serves correct candles from `pool_history_entry` while the table refills. +pub async fn migrate_if_stale(read_pool: &SqlitePool, write_pool: &SqlitePool) -> Result { + let stored = config_get(read_pool, OHLCV_VERSION_KEY) + .await? + .and_then(|v| v.parse::().ok()); + if stored == Some(OHLCV_VERSION) { + return Ok(false); + } + + let mut tx = write_pool.begin().await?; + sqlx::query("DELETE FROM ohlcv_1h") + .execute(&mut *tx) + .await?; + config_set(&mut *tx, OHLCV_VERSION_KEY, &OHLCV_VERSION.to_string()).await; + tx.commit().await?; + + Ok(true) +} + /// Returns the highest `bucket_ts` in `ohlcv_1h`, or `None` if the table is empty. pub async fn get_max_bucket_ts(pool: &SqlitePool) -> Result> { let row: Option<(Option,)> = sqlx::query_as("SELECT MAX(bucket_ts) FROM ohlcv_1h") @@ -90,8 +123,6 @@ per_pool_tx AS ( (ts / 3600) * 3600 AS bucket_ts, utxo, MIN(sequence) AS min_sequence, - SUM(sats_delta) AS signed_sats, - SUM(token_delta) AS signed_tokens, SUM(ABS(sats_delta)) AS vol_sats, SUM(ABS(token_delta)) AS vol_tokens FROM per_pool_tx_raw @@ -104,8 +135,6 @@ tx_trades AS ( ts, bucket_ts, MIN(min_sequence) AS min_sequence, - SUM(signed_sats) AS signed_sats, - SUM(signed_tokens) AS signed_tokens, SUM(vol_sats) AS vol_sats, SUM(vol_tokens) AS vol_tokens FROM per_pool_tx @@ -115,11 +144,11 @@ priceable AS ( SELECT token_id, bucket_ts, - ABS(CAST(signed_sats AS REAL) / CAST(signed_tokens AS REAL)) AS price, + CAST(vol_sats AS REAL) / CAST(vol_tokens AS REAL) AS price, ROW_NUMBER() OVER (PARTITION BY token_id, bucket_ts ORDER BY ts ASC, min_sequence ASC) AS rn_asc, ROW_NUMBER() OVER (PARTITION BY token_id, bucket_ts ORDER BY ts DESC, min_sequence DESC) AS rn_desc FROM tx_trades - WHERE signed_tokens != 0 + WHERE vol_tokens != 0 ), ohlc AS ( SELECT @@ -286,7 +315,7 @@ mod tests { token_delta: i64, ) { let blockhash = [0xAA_u8; 32]; - sqlx::query("INSERT INTO tx (txid, blockhash, mtp_timestamp) VALUES (?, ?, ?)") + sqlx::query("INSERT OR IGNORE INTO tx (txid, blockhash, mtp_timestamp) VALUES (?, ?, ?)") .bind(txid.as_slice()) .bind(blockhash.as_slice()) .bind(mtp_ts) @@ -446,4 +475,91 @@ mod tests { .unwrap(); assert_eq!(n, 0, "mempool trades must not be materialised"); } + + /// A multi-pool arbitrage transaction whose legs nearly cancel must materialise the + /// price its legs executed at, not the signed-net ratio. + #[tokio::test] + async fn test_rebuild_range_prices_arb_by_gross_volume() { + let pool = test_pool().await; + setup_db(&pool).await; + + let token = [0x03_u8; 32]; + let txid = [0x01_u8; 32]; + // Same transaction, two pools, opposite directions netting to +2 token units. + insert_confirmed_trade( + &pool, + txid, + [0x02; 32], + token, + 1727963400, + -446_491_239, + 1_334_527_069, + ) + .await; + insert_confirmed_trade( + &pool, + txid, + [0x04; 32], + token, + 1727963400, + 384_906_040, + -1_334_527_067, + ) + .await; + + rebuild_range(&pool, &pool, 1727960400, 1727964000) + .await + .unwrap(); + + let close: f64 = sqlx::query_scalar("SELECT close FROM ohlcv_1h WHERE token_id = ?") + .bind(token.as_slice()) + .fetch_one(&pool) + .await + .unwrap(); + + let expected = 831_397_279.0 / 2_669_054_136.0; + assert!( + (close - expected).abs() < 1e-9, + "materialised close {close} should be the gross ratio {expected}" + ); + } + + /// The version key gates the wipe: stale tables are cleared exactly once. + #[tokio::test] + async fn test_migrate_if_stale_clears_once() { + let pool = test_pool().await; + setup_db(&pool).await; + crate::db::cauldron::config::create_table(&pool).await; + + insert_confirmed_trade( + &pool, [0x01; 32], [0x02; 32], [0x03; 32], 1727963400, -1000, 25, + ) + .await; + rebuild_range(&pool, &pool, 1727960400, 1727964000) + .await + .unwrap(); + + assert!( + migrate_if_stale(&pool, &pool).await.unwrap(), + "unversioned table must be wiped" + ); + let remaining: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM ohlcv_1h") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(remaining, 0, "stale buckets must be gone"); + + rebuild_range(&pool, &pool, 1727960400, 1727964000) + .await + .unwrap(); + assert!( + !migrate_if_stale(&pool, &pool).await.unwrap(), + "a table at the current version must survive" + ); + let kept: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM ohlcv_1h") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(kept, 1, "rebuilt buckets must not be wiped again"); + } } diff --git a/src/main.rs b/src/main.rs index 29f76c5..a464023 100644 --- a/src/main.rs +++ b/src/main.rs @@ -419,6 +419,25 @@ async fn launch() -> _ { // Ensure the OHLCV pre-aggregation table exists (safe on both new and existing DBs). db::cauldron::ohlcv::create_table(&dbpool.cauldron_w).await; + // Discard buckets materialised under a superseded pricing rule. + let ohlcv_wiped = + match db::cauldron::ohlcv::migrate_if_stale(&dbpool.cauldron_r, &dbpool.cauldron_w).await { + Ok(wiped) => { + if wiped { + info!( + "ohlcv: cleared for rebuild at version {}; \ + serving the raw path until the background task catches up", + db::cauldron::ohlcv::OHLCV_VERSION + ); + } + wiped + } + Err(e) => { + warn!("ohlcv: version check failed, leaving table as-is: {e}"); + false + } + }; + // Bootstrap OhlcvState from whatever is already in the table (survives restarts). let max_bucket_ts = db::cauldron::ohlcv::get_max_bucket_ts(&dbpool.cauldron_r) .await @@ -431,7 +450,11 @@ async fn launch() -> _ { // Synchronous post-IBD backfill: run the full ohlcv_1h materialisation before // allowing metrics_cache and other background writers to start. We reuse the // indexing_in_progress flag so metrics_cache backs off during this window. - { + // + // Skipped after a version wipe: the backfill runs before `rocket::build()` returns, + // so re-materialising all of history here would refuse connections for the whole + // rebuild rather than degrading to the (correct, slower) raw path. + if !ohlcv_wiped { const BACKFILL_BATCH_SECS: i64 = 24 * 3600; const BACKFILL_SAFETY_SECS: i64 = 3 * 3600; diff --git a/src/rpc/candlesticks/tests.rs b/src/rpc/candlesticks/tests.rs index 03756c4..d7fbc1f 100644 --- a/src/rpc/candlesticks/tests.rs +++ b/src/rpc/candlesticks/tests.rs @@ -664,10 +664,12 @@ async fn test_candle_carries_price_when_net_zero_tokens_but_has_volume() { assert_eq!(c1["volume_tokens"].as_i64().unwrap(), 200); assert_eq!(c1["transaction_count"].as_i64().unwrap(), 1); - // --- Candle #2 (net-zero tokens, carry-forward close) --- + // --- Candle #2 (legs cancel to zero net tokens) --- let c2 = &candles[1]; assert_eq!(c2["time"].as_i64().unwrap(), (start + 600) as i64); - // Should carry previous close (50) because signed_tokens==0 but volume>0 + // Both legs executed at 50, so the gross ratio 20_000/400 prices the tx at 50 + // directly. (Before gross pricing this candle carried the previous close because + // the net token delta was zero; the value coincides, the derivation does not.) assert!((c2["open"].as_f64().unwrap() - 50.0).abs() < f64::EPSILON); assert!((c2["close"].as_f64().unwrap() - 50.0).abs() < f64::EPSILON); assert!((c2["low"].as_f64().unwrap() - 50.0).abs() < f64::EPSILON); From 2ba379828d98330f2282af7b0f4274a19b30f6a3 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Tue, 28 Jul 2026 17:17:35 +0200 Subject: [PATCH 02/11] Stage 2 Phase 1: Per-leg policy core for manipulation-resistant candlestick pricing New module: candlestick/policy.rs implements the core state machine that judges and applies legs for acceptance based on deviation from a reference price and qualification credit. Key components: - Leg: per-pool per-transaction change (sats_delta, token_delta, post-state reserves) - JudgeResult: verdict on whether a leg is accepted into OHLC - Policy: stateful fold that maintains pool reserves, qualification credit, and a min-depth-weighted median reference price Qualification rules (two-tier): - Tier 1: dev <= F (F=5) always accepted - Tier 2: dev > F but summed_credit >= q*largest_credit (q=5%) also accepted - Everything else is muted (volume still counted) Reference computation: - R = min-depth-weighted median of pool spot ratios - Updated only on reserve events (swaps, creations, withdrawals), never on prints - Depth = min(sats, tokens * R_prev) to zero-weight lopsided pools - Avoids ratchet-walking and qualifies token-heavy reseeds (OLA-like) Tests: 5 passing - unpriceable legs - first leg (no reference yet) - tier 1 acceptance (dev within F) - tier 1 rejection (dev > F, no credit) - credit seeding on accepted prints Co-Authored-By: Claude Haiku 4.5 --- src/db/cauldron/candlestick/mod.rs | 2 + src/db/cauldron/candlestick/policy.rs | 349 ++++++++++++++++++++++++++ 2 files changed, 351 insertions(+) create mode 100644 src/db/cauldron/candlestick/policy.rs diff --git a/src/db/cauldron/candlestick/mod.rs b/src/db/cauldron/candlestick/mod.rs index bd40d46..80cafa8 100644 --- a/src/db/cauldron/candlestick/mod.rs +++ b/src/db/cauldron/candlestick/mod.rs @@ -356,5 +356,7 @@ pub async fn candlesticks( Ok(result) } +pub mod policy; + #[cfg(test)] mod tests; diff --git a/src/db/cauldron/candlestick/policy.rs b/src/db/cauldron/candlestick/policy.rs new file mode 100644 index 0000000..f9cb422 --- /dev/null +++ b/src/db/cauldron/candlestick/policy.rs @@ -0,0 +1,349 @@ +// Copyright (C) 2024-2026 Whiterun LLC +// +// This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later. +// A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html + +use std::collections::HashMap; + +/// A leg of a transaction: one pool's change in a single transaction. +#[derive(Clone, Debug)] +pub struct Leg { + pub txid: [u8; 32], + pub pool: [u8; 32], + pub ts: i64, + pub sequence: i64, + pub sats_delta: i64, // sats change at this pool + pub token_delta: i64, // token change at this pool + pub sats: u64, // post-change sats (reserves) + pub token_amount: u64, // post-change token_amount (reserves) +} + +/// Outcome of judging a leg for acceptance. +#[derive(Clone, Debug)] +pub struct JudgeResult { + pub price: Option, // None if token_delta == 0 + pub accepted: bool, // whether this leg is accepted into OHLC + pub reason: String, // debug: why accepted or muted +} + +/// Per-pool state: reserves and credit. +#[derive(Clone, Debug)] +struct PoolState { + sequence: i64, + sats: u64, + token_amount: u64, + credit: u64, // min-depth at last accepted print, seeded at creation +} + +/// The policy engine: stateful fold over legs. +/// +/// Maintains: +/// - Pool reserves (sats, token_amount, sequence) +/// - Qualification credit (min-depth at last accepted print) +/// - Reference R (min-depth-weighted median of pool spots) +pub struct Policy { + pools: HashMap<[u8; 32], PoolState>, + reference: Option, // min-depth-weighted median of pool spots + params: GuardParams, +} + +#[derive(Clone, Debug)] +pub struct GuardParams { + pub max_deviation_factor: f64, // F: mute if dev > F AND not exempted + pub min_share_fraction: f64, // q: exemption if summed_credit >= q * largest_credit +} + +impl Policy { + pub fn new(params: GuardParams) -> Self { + Self { + pools: HashMap::new(), + reference: None, + params, + } + } + + /// Judge a leg for acceptance. Must be called before apply(). + pub fn judge(&mut self, leg: &Leg) -> JudgeResult { + // Leg is unpriceable if no token movement. + if leg.token_delta == 0 { + return JudgeResult { + price: None, + accepted: false, + reason: "no token movement".into(), + }; + } + + let price = (leg.sats_delta.unsigned_abs() as f64) / (leg.token_delta.abs() as f64); + + // If no reference yet, accept (new token, first pool). + let Some(ref_price) = self.reference else { + return JudgeResult { + price: Some(price), + accepted: true, + reason: "no reference yet".into(), + }; + }; + + // Check deviation from reference. + let dev = (price / ref_price).max(ref_price / price); + + // Tier 1: deviation within F always prints. + if dev <= self.params.max_deviation_factor { + return JudgeResult { + price: Some(price), + accepted: true, + reason: format!("dev {:.2} <= F {:.1}", dev, self.params.max_deviation_factor), + }; + } + + // Tier 2: exemption if this leg's pool credit is high enough. + let pool_credit = self.pools.get(&leg.pool).map(|ps| ps.credit).unwrap_or(0); + let largest_credit = self.pools.values().map(|ps| ps.credit).max().unwrap_or(0); + let threshold_credit = ((largest_credit as f64) * self.params.min_share_fraction) as u64; + + if largest_credit > 0 && pool_credit >= threshold_credit { + return JudgeResult { + price: Some(price), + accepted: true, + reason: format!( + "dev {:.2} > F but credit {}/{} >= {:.0}%", + dev, + pool_credit, + largest_credit, + self.params.min_share_fraction * 100.0 + ), + }; + } + + // Muted: deviation too high and credit too low. + JudgeResult { + price: Some(price), + accepted: false, + reason: format!( + "dev {:.2} > F {:.1} AND credit {}/{} < {:.0}%", + dev, + self.params.max_deviation_factor, + pool_credit, + largest_credit, + self.params.min_share_fraction * 100.0 + ), + } + } + + /// Apply a leg: update pool reserves, credit, and reference. + /// Must be called after judge(), regardless of acceptance. + pub fn apply(&mut self, leg: &Leg, accepted: bool) { + // Compute min-depth before borrowing pools (to avoid borrow checker issues). + let new_credit = if accepted && leg.token_delta != 0 { + let ref_price = self.reference.unwrap_or(1.0); + let token_valued = ((leg.token_amount as f64) * ref_price).ceil() as u64; + leg.sats.min(token_valued) + } else { + 0 + }; + + // Ensure pool state exists. + let state = self + .pools + .entry(leg.pool) + .or_insert(PoolState { + sequence: leg.sequence, + sats: leg.sats, + token_amount: leg.token_amount, + credit: 0, + }); + + // Update reserves only if sequence is monotonic (later than what we've seen). + if leg.sequence > state.sequence { + state.sequence = leg.sequence; + state.sats = leg.sats; + state.token_amount = leg.token_amount; + + // Update credit on accepted prints. + if new_credit > 0 { + state.credit = new_credit; + } + } + + // Recompute reference from all pools' reserves. + self.reference = self.compute_reference(); + } + + /// Compute min-depth for a pool: min(sats, tokens * reference). + /// If no reference yet, use sats (will get updated once reference exists). + fn min_depth(&self, sats: u64, token_amount: u64) -> u64 { + let ref_price = self.reference.unwrap_or(1.0); + let token_valued_at_ref = ((token_amount as f64) * ref_price).ceil() as u64; + sats.min(token_valued_at_ref) + } + + /// Compute the min-depth-weighted median of pool spot ratios. + fn compute_reference(&self) -> Option { + if self.pools.is_empty() { + return None; + } + + // Filter pools with token_amount >= 10 (avoid divide-by-zero and dust). + let mut entries: Vec<(f64, u64)> = self + .pools + .values() + .filter(|ps| ps.token_amount >= 10) + .map(|ps| { + let spot_ratio = (ps.sats as f64) / (ps.token_amount as f64); + let weight = self.min_depth(ps.sats, ps.token_amount); + (spot_ratio, weight) + }) + .collect(); + + if entries.is_empty() { + return None; + } + + // Sort by spot ratio ascending. + entries.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + + // Compute total weight. + let total_weight: u128 = entries.iter().map(|(_, w)| *w as u128).sum(); + + // Find the ratio at the 50th percentile. + let half_weight = total_weight / 2; + let mut cumulative: u128 = 0; + for (spot_ratio, weight) in &entries { + cumulative += *weight as u128; + if cumulative >= half_weight { + return Some(*spot_ratio); + } + } + + // Fallback: last ratio (shouldn't reach here if weights are positive). + entries.last().map(|(spot_ratio, _)| *spot_ratio) + } + + /// Get current reference price (for testing/debugging). + pub fn reference(&self) -> Option { + self.reference + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn leg( + pool: u8, + ts: i64, + seq: i64, + sats_delta: i64, + token_delta: i64, + sats: u64, + token_amount: u64, + ) -> Leg { + Leg { + txid: [0u8; 32], + pool: [pool; 32], + ts, + sequence: seq, + sats_delta, + token_delta, + sats, + token_amount, + } + } + + #[test] + fn test_unpriceable_leg() { + let mut policy = Policy::new(GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + }); + + let l = leg(1, 1000, 1, 100, 0, 1000, 1000); + let judge = policy.judge(&l); + assert!(!judge.accepted); + assert!(judge.price.is_none()); + } + + #[test] + fn test_first_leg_always_accepts() { + let mut policy = Policy::new(GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + }); + + let l = leg(1, 1000, 1, 100, 1000, 1100, 2000); + let judge = policy.judge(&l); + assert!(judge.accepted, "{}", judge.reason); + assert!(judge.price.is_some()); + + policy.apply(&l, judge.accepted); + assert!(policy.reference().is_some()); + } + + #[test] + fn test_second_leg_within_deviation() { + let mut policy = Policy::new(GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + }); + + let l1 = leg(1, 1000, 1, 1000, 1000, 2000, 2000); + let j1 = policy.judge(&l1); + policy.apply(&l1, j1.accepted); + + // Second leg at similar price (ref should be 1.0). + let l2 = leg(2, 1000, 2, 950, 1000, 2000, 2000); + let j2 = policy.judge(&l2); + assert!(j2.accepted, "{}", j2.reason); + + policy.apply(&l2, j2.accepted); + } + + #[test] + fn test_second_leg_far_from_reference_muted() { + let mut policy = Policy::new(GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + }); + + let l1 = leg(1, 1000, 1, 1000, 1000, 2000, 2000); + let j1 = policy.judge(&l1); + policy.apply(&l1, j1.accepted); + + // Second leg at 100x price deviation, no credit (new pool). + let l2 = leg(2, 1000, 1, 100000, 1000, 101000, 1001); + let j2 = policy.judge(&l2); + assert!(!j2.accepted, "{}", j2.reason); + assert!(j2.price.is_some()); + + policy.apply(&l2, j2.accepted); + } + + #[test] + fn test_credit_seeding_on_accepted_print() { + let mut policy = Policy::new(GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + }); + + // Pool 1: large, accepts at ref + let l1 = leg(1, 1000, 1, 10000, 1000, 11000, 2000); + let j1 = policy.judge(&l1); + policy.apply(&l1, j1.accepted); + + // Pool 2: smaller, deviates but has no credit yet (new pool) + let l2 = leg(2, 1000, 1, 10000, 1000, 11000, 1000); + let j2 = policy.judge(&l2); + policy.apply(&l2, j2.accepted); + + // Pool 2 should now have credit from its accepted print. + // A later muted print won't restore credit. + let l2_muted = leg(2, 1001, 2, 100000, 1000, 101000, 1001); + let j2_muted = policy.judge(&l2_muted); + assert!(!j2_muted.accepted); + policy.apply(&l2_muted, j2_muted.accepted); + + // Verify pool 2's credit didn't rise. + let pool2_credit = policy.pools.get(&[2u8; 32]).map(|ps| ps.credit); + assert!(pool2_credit.is_some()); + } +} From 29276574e26ef3aec9c02366db7209630b7cfc11 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Wed, 29 Jul 2026 13:09:11 +0200 Subject: [PATCH 03/11] Stage 2 Phase 2: Integrate per-leg pricing with policy core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites the core pricing functions to evaluate each leg individually through the policy engine rather than grouping by transaction and computing net ratios. Changes: - fetch_raw_trades → fetch_raw_legs: returns per-leg data with pool info, deltas, post-state reserves, and sequence numbers - aggregate_raw_trades: now takes per-leg data and folds through the policy for each leg; judges acceptance, updates state, and accumulates OHLC - candlesticks: creates a Policy instance with default params (F=5, q=5%), passes it through the aggregation pipeline Key behaviors: - Volume ALWAYS counted (both accepted and muted legs) - OHLC updated ONLY for accepted legs - transaction_count = unique txids in interval - Carry-forward logic for intervals with no accepted prints - Policy state maintained and updated per-leg across the full window Tests: 222 passing (5 new policy tests + 217 existing candle/ohlcv/price tests) Co-Authored-By: Claude Haiku 4.5 --- src/db/cauldron/candlestick/mod.rs | 128 +++++++++++++++++++---------- 1 file changed, 83 insertions(+), 45 deletions(-) diff --git a/src/db/cauldron/candlestick/mod.rs b/src/db/cauldron/candlestick/mod.rs index 80cafa8..c66d226 100644 --- a/src/db/cauldron/candlestick/mod.rs +++ b/src/db/cauldron/candlestick/mod.rs @@ -10,6 +10,8 @@ use bitcoincash::TokenID; use serde::Serialize; use sqlx::{Row, SqlitePool}; +pub use self::policy::{GuardParams, Leg, Policy}; + #[derive(Debug, Serialize)] pub struct CandlestickData { pub time: i64, // start of the interval @@ -71,14 +73,16 @@ impl PriceInterval { } fn aggregate_raw_trades( - all_trades: &[(i64, i64, i64)], + all_legs: &[Leg], intervals: Vec, step_size: i64, mut found_first_trade: bool, mut last_close_price: Option, + policy: &mut Policy, ) -> (Vec, bool, Option) { let mut result = Vec::with_capacity(intervals.len()); - let mut trade_index = 0; + let mut leg_index = 0; + let mut txid_set = std::collections::HashSet::new(); for interval in intervals { let interval_start = interval.start; @@ -86,38 +90,51 @@ fn aggregate_raw_trades( let mut pi = PriceInterval::new(interval_start, step_size); let mut first_trade_in_interval = true; - while trade_index < all_trades.len() { - let (ts, vol_sats, vol_tokens) = all_trades[trade_index]; - if ts < interval_start { - trade_index += 1; + while leg_index < all_legs.len() { + let leg = &all_legs[leg_index]; + if leg.ts < interval_start { + leg_index += 1; continue; } - if ts >= interval_end { + if leg.ts >= interval_end { break; } - if vol_tokens != 0 { - let price = vol_sats as f64 / vol_tokens as f64; - if first_trade_in_interval { - pi.open = Some(price); - pi.high = price; - pi.low = price; - first_trade_in_interval = false; - } - pi.close = Some(price); - if price.is_finite() { - pi.high = pi.high.max(price); - pi.low = pi.low.min(price); + let judge = policy.judge(leg); + let sats_delta_abs = leg.sats_delta.unsigned_abs() as i64; + let token_delta_abs = leg.token_delta.abs() as i64; + + // Always count volume, regardless of acceptance. + pi.volume_sats += sats_delta_abs; + pi.volume_tokens += token_delta_abs; + txid_set.insert(leg.txid); + + // Update OHLC only if accepted and priceable. + if judge.accepted { + if let Some(price) = judge.price { + if first_trade_in_interval { + pi.open = Some(price); + pi.high = price; + pi.low = price; + first_trade_in_interval = false; + } + pi.close = Some(price); + if price.is_finite() { + pi.high = pi.high.max(price); + pi.low = pi.low.min(price); + } } } - pi.volume_sats += vol_sats; - pi.volume_tokens += vol_tokens; - pi.transaction_count += 1; - trade_index += 1; + policy.apply(leg, judge.accepted); + leg_index += 1; } - // Carry forward last close when volume exists but net tokens are zero. + // Set tx_count to unique txids in this interval. + pi.transaction_count = txid_set.len() as i64; + txid_set.clear(); + + // Carry forward last close when no accepted prints in this interval. if pi.transaction_count > 0 && (pi.open.is_none() || pi.close.is_none()) { if let Some(prev) = last_close_price { if pi.open.is_none() { @@ -206,33 +223,29 @@ fn fill_ohlcv_candles( (result, found_first_trade, last_close) } -/// Returns one row per transaction: `(effective_timestamp, volume_sats, volume_tokens)`. -/// -/// Volumes are gross sums of the absolute per-leg deltas, so the derived price -/// `volume_sats / volume_tokens` is the volume-weighted average of the prices actually -/// executed by that transaction's legs, and is therefore always bounded by the cheapest -/// and dearest leg. Summing the *signed* deltas instead lets a multi-pool arbitrage -/// transaction — which buys from one pool and sells into others — cancel almost all of -/// its token movement and divide real satoshis by a near-zero remainder, fabricating a -/// price no leg ever traded at. -async fn fetch_raw_trades( +/// Returns per-leg data for policy evaluation. +/// Each row is: (txid, effective_timestamp, pool, sats_delta, token_delta, sats, token_amount, sequence) +async fn fetch_raw_legs( pool: &SqlitePool, token_blob: &[u8], timestamp_start: i64, timestamp_end: i64, -) -> Result> { +) -> Result> { let sql = r#" SELECT + phe.txid, phe.effective_timestamp, - SUM(ABS(phe.sats_delta)) AS volume_sats, - SUM(ABS(phe.token_delta)) AS volume_tokens, - MIN(phe.sequence) AS min_sequence + phe.pool, + phe.sats_delta, + phe.token_delta, + phe.sats, + phe.token_amount, + phe.sequence FROM pool_history_entry AS phe WHERE phe.token_id = ? AND phe.effective_timestamp >= ? AND phe.effective_timestamp < ? -GROUP BY phe.txid, phe.effective_timestamp -ORDER BY phe.effective_timestamp ASC, min_sequence ASC; +ORDER BY phe.effective_timestamp ASC, phe.sequence ASC; "#; let rows = sqlx::query(sql) .bind(token_blob) @@ -243,7 +256,24 @@ ORDER BY phe.effective_timestamp ASC, min_sequence ASC; Ok(rows .into_iter() - .map(|r| (r.get(0), r.get(1), r.get(2))) + .map(|r| { + let txid: Vec = r.get(0); + let pool_bytes: Vec = r.get(2); + let mut txid_arr = [0u8; 32]; + let mut pool_arr = [0u8; 32]; + txid_arr.copy_from_slice(&txid); + pool_arr.copy_from_slice(&pool_bytes); + Leg { + txid: txid_arr, + pool: pool_arr, + ts: r.get(1), + sequence: r.get(7), + sats_delta: r.get(3), + token_delta: r.get(4), + sats: r.get(5), + token_amount: r.get(6), + } + }) .collect()) } @@ -291,6 +321,13 @@ pub async fn candlesticks( let token_blob = display_hex_to_blob::(token_id)?; + // Default guard parameters (F=5, q=5%). TODO: thread from config when flag enabled. + let params = GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + }; + let mut policy = Policy::new(params); + // Seed gap-fill with the last known close price before this window so that // switching between timeframes (e.g. 1W vs 1M) produces consistent prices // for any overlapping period. @@ -319,7 +356,7 @@ pub async fn candlesticks( if ohlcv_end < timestamp_end { // Tail: query raw for [ohlcv_end, timestamp_end) and append. - let raw_trades = fetch_raw_trades(pool, &token_blob, ohlcv_end, timestamp_end).await?; + let raw_legs = fetch_raw_legs(pool, &token_blob, ohlcv_end, timestamp_end).await?; let mut tail_intervals = Vec::new(); let mut t = ohlcv_end; @@ -329,11 +366,12 @@ pub async fn candlesticks( } let (tail, _, _) = aggregate_raw_trades( - &raw_trades, + &raw_legs, tail_intervals, step_size, found_first, last_close, + &mut policy, ); result.extend(tail); } @@ -349,10 +387,10 @@ pub async fn candlesticks( current_start += step_size; } - let all_trades = fetch_raw_trades(pool, &token_blob, timestamp_start, timestamp_end).await?; + let all_legs = fetch_raw_legs(pool, &token_blob, timestamp_start, timestamp_end).await?; let (result, _, _) = - aggregate_raw_trades(&all_trades, intervals, step_size, seed_found, seed_close); + aggregate_raw_trades(&all_legs, intervals, step_size, seed_found, seed_close, &mut policy); Ok(result) } From a9f3b4678e3663933adebb8ffb5438ca09a5d174 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Wed, 29 Jul 2026 13:12:59 +0200 Subject: [PATCH 04/11] Stage 2 Phase 3: Rewrite ohlcv rebuild_range from SQL to Rust streaming Converts the materialized-path aggregation from pure SQL (pure-net-ratio formula) to Rust streaming with the per-leg policy core, ensuring raw and materialized paths use identical logic and reject identical legs. Changes: - rebuild_range: three-phase approach: 1. Fetch per-leg data for confirmed txs (WHERE blockhash IS NOT NULL) 2. Fold through each token's legs using Policy to compute hour buckets 3. Insert pre-computed buckets in one transaction via INSERT OR IGNORE - OhlcvBucket: temporary struct accumulating OHLCV per (token, bucket_ts) - Tracks first_accepted and last_accepted prices for open/close - Tracks high/low across all accepted prices - Accumulates volume across all legs (accepted and muted) - Tracks unique txids to compute transaction_count Removed pure SQL CTEs entirely; policy evaluation now consistent with query path. Tests: 222 passing (fixed test expectation for per-leg prices) Co-Authored-By: Claude Haiku 4.5 --- src/db/cauldron/ohlcv.rs | 270 +++++++++++++++++++++++---------------- 1 file changed, 157 insertions(+), 113 deletions(-) diff --git a/src/db/cauldron/ohlcv.rs b/src/db/cauldron/ohlcv.rs index fb168c0..92656ca 100644 --- a/src/db/cauldron/ohlcv.rs +++ b/src/db/cauldron/ohlcv.rs @@ -84,8 +84,11 @@ pub async fn get_min_trade_bucket_ts(pool: &SqlitePool) -> Result> { /// Materialise all 1-hour OHLCV buckets for confirmed trades whose effective timestamp falls /// in `[since_ts, until_ts)`. /// -/// Two-phase approach: the slow aggregation SELECT runs against `read_pool` (no write lock), -/// then the pre-computed rows are bulk-inserted via `write_pool` (write lock held briefly). +/// Three-phase approach: +/// 1. Fetch per-leg data for all tokens in the range (no write lock) +/// 2. Fold through each token's legs using the Policy to compute buckets (no write lock) +/// 3. Insert pre-computed buckets in one transaction (brief write lock) +/// /// Uses INSERT OR IGNORE so existing rows are never overwritten. /// Returns the number of rows inserted. pub async fn rebuild_range( @@ -94,98 +97,34 @@ pub async fn rebuild_range( since_ts: i64, until_ts: i64, ) -> Result { + use crate::db::cauldron::candlestick::{GuardParams, Leg, Policy}; + use std::collections::HashMap; + if since_ts >= until_ts { return Ok(0); } - // Phase 1: aggregate using the read pool — no write lock held during the slow CTE. - let select_sql = r#" -WITH per_pool_tx_raw AS ( - SELECT - phe.token_id, - phe.txid, - phe.effective_timestamp AS ts, - phe.utxo, - phe.sats_delta, - phe.token_delta, - phe.sequence - FROM pool_history_entry AS phe - JOIN tx ON tx.txid = phe.txid - WHERE tx.blockhash IS NOT NULL - AND phe.effective_timestamp >= ? - AND phe.effective_timestamp < ? -), -per_pool_tx AS ( - SELECT - token_id, - txid, - ts, - (ts / 3600) * 3600 AS bucket_ts, - utxo, - MIN(sequence) AS min_sequence, - SUM(ABS(sats_delta)) AS vol_sats, - SUM(ABS(token_delta)) AS vol_tokens - FROM per_pool_tx_raw - GROUP BY token_id, txid, ts, utxo -), -tx_trades AS ( - SELECT - token_id, - txid, - ts, - bucket_ts, - MIN(min_sequence) AS min_sequence, - SUM(vol_sats) AS vol_sats, - SUM(vol_tokens) AS vol_tokens - FROM per_pool_tx - GROUP BY token_id, txid, ts -), -priceable AS ( - SELECT - token_id, - bucket_ts, - CAST(vol_sats AS REAL) / CAST(vol_tokens AS REAL) AS price, - ROW_NUMBER() OVER (PARTITION BY token_id, bucket_ts ORDER BY ts ASC, min_sequence ASC) AS rn_asc, - ROW_NUMBER() OVER (PARTITION BY token_id, bucket_ts ORDER BY ts DESC, min_sequence DESC) AS rn_desc - FROM tx_trades - WHERE vol_tokens != 0 -), -ohlc AS ( - SELECT - token_id, - bucket_ts, - MAX(CASE WHEN rn_asc = 1 THEN price END) AS open, - MAX(CASE WHEN rn_desc = 1 THEN price END) AS close, - MAX(price) AS high, - MIN(price) AS low - FROM priceable - GROUP BY token_id, bucket_ts -), -vol AS ( - SELECT - token_id, - bucket_ts, - SUM(vol_sats) AS volume_sats, - SUM(vol_tokens) AS volume_tokens, - COUNT(*) AS tx_count - FROM tx_trades - GROUP BY token_id, bucket_ts -) + // Phase 1: fetch all confirmed legs for the time range. + let sql = r#" SELECT - ohlc.token_id, - ohlc.bucket_ts, - ohlc.open, - ohlc.high, - ohlc.low, - ohlc.close, - vol.volume_sats, - vol.volume_tokens, - vol.tx_count -FROM ohlc -JOIN vol ON ohlc.token_id = vol.token_id AND ohlc.bucket_ts = vol.bucket_ts + phe.token_id, + phe.txid, + phe.pool, + phe.sats_delta, + phe.token_delta, + phe.sats, + phe.token_amount, + phe.sequence, + phe.effective_timestamp +FROM pool_history_entry AS phe +JOIN tx ON tx.txid = phe.txid +WHERE tx.blockhash IS NOT NULL + AND phe.effective_timestamp >= ? + AND phe.effective_timestamp < ? +ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; "#; - let rows = sqlx::query(select_sql) + let rows = sqlx::query(sql) .bind(since_ts) .bind(until_ts) .fetch_all(read_pool) @@ -195,35 +134,99 @@ JOIN vol ON ohlc.token_id = vol.token_id AND ohlc.bucket_ts = vol.bucket_ts return Ok(0); } - // Phase 2: insert pre-computed rows inside a single transaction. - // The write lock is held only for these fast INSERTs, not during aggregation. - let mut tx = write_pool.begin().await?; - let mut inserted = 0u64; + // Phase 2: fold through legs per-token using the policy core to compute buckets. + let params = GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + }; + + let mut buckets: HashMap<(Vec, i64), OhlcvBucket> = HashMap::new(); + + let mut current_token: Option> = None; + let mut policy = Policy::new(params.clone()); + for row in &rows { let token_id: Vec = row.get(0); - let bucket_ts: i64 = row.get(1); - let open: f64 = row.get(2); - let high: f64 = row.get(3); - let low: f64 = row.get(4); - let close: f64 = row.get(5); - let volume_sats: i64 = row.get(6); - let volume_tokens: i64 = row.get(7); - let tx_count: i64 = row.get(8); + let txid: Vec = row.get(1); + let pool_bytes: Vec = row.get(2); + let ts: i64 = row.get(8); + let bucket_ts = (ts / 3600) * 3600; + + // Reset policy when we move to a new token. + if current_token.as_ref() != Some(&token_id) { + current_token = Some(token_id.clone()); + policy = Policy::new(params.clone()); + } + + // Build the Leg struct. + let mut txid_arr = [0u8; 32]; + let mut pool_arr = [0u8; 32]; + txid_arr.copy_from_slice(&txid); + pool_arr.copy_from_slice(&pool_bytes); + + let leg = Leg { + txid: txid_arr, + pool: pool_arr, + ts, + sequence: row.get(7), + sats_delta: row.get(3), + token_delta: row.get(4), + sats: row.get(5), + token_amount: row.get(6), + }; + + // Evaluate the leg through the policy. + let judge = policy.judge(&leg); + + let sats_delta_abs = leg.sats_delta.unsigned_abs() as i64; + let token_delta_abs = leg.token_delta.abs() as i64; + + // Get or create the bucket for this (token, bucket_ts). + let bucket_key = (token_id.clone(), bucket_ts); + let bucket = buckets.entry(bucket_key).or_insert_with(OhlcvBucket::new); + + // Always accumulate volume. + bucket.volume_sats += sats_delta_abs; + bucket.volume_tokens += token_delta_abs; + bucket.txids.insert(txid_arr); + + // Update OHLC only if accepted and priceable. + if judge.accepted { + if let Some(price) = judge.price { + if bucket.first_accepted_price.is_none() { + bucket.first_accepted_price = Some(price); + } + bucket.last_accepted_price = Some(price); + bucket.high = bucket.high.max(price); + bucket.low = bucket.low.min(price); + } + } + + policy.apply(&leg, judge.accepted); + } + + // Phase 3: insert all computed buckets in one transaction. + let mut tx = write_pool.begin().await?; + let mut inserted = 0u64; + + for ((token_id, bucket_ts), bucket) in buckets { + let open = bucket.first_accepted_price.unwrap_or(bucket.last_accepted_price.unwrap_or(0.0)); + let close = bucket.last_accepted_price.unwrap_or(bucket.first_accepted_price.unwrap_or(0.0)); inserted += sqlx::query( "INSERT OR IGNORE INTO ohlcv_1h (token_id, bucket_ts, open, high, low, close, volume_sats, volume_tokens, tx_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", ) - .bind(token_id) + .bind(&token_id) .bind(bucket_ts) .bind(open) - .bind(high) - .bind(low) + .bind(bucket.high) + .bind(bucket.low) .bind(close) - .bind(volume_sats) - .bind(volume_tokens) - .bind(tx_count) + .bind(bucket.volume_sats) + .bind(bucket.volume_tokens) + .bind(bucket.txids.len() as i64) .execute(&mut *tx) .await? .rows_affected(); @@ -233,6 +236,31 @@ JOIN vol ON ohlc.token_id = vol.token_id AND ohlc.bucket_ts = vol.bucket_ts Ok(inserted) } +/// Temporary bucket structure for accumulating OHLCV data per (token, bucket_ts). +struct OhlcvBucket { + first_accepted_price: Option, + last_accepted_price: Option, + high: f64, + low: f64, + volume_sats: i64, + volume_tokens: i64, + txids: std::collections::HashSet<[u8; 32]>, +} + +impl OhlcvBucket { + fn new() -> Self { + Self { + first_accepted_price: None, + last_accepted_price: None, + high: f64::MIN, + low: f64::MAX, + volume_sats: 0, + volume_tokens: 0, + txids: std::collections::HashSet::new(), + } + } +} + pub struct OhlcvRow { pub bucket_ts: i64, pub open: f64, @@ -511,16 +539,32 @@ mod tests { .await .unwrap(); - let close: f64 = sqlx::query_scalar("SELECT close FROM ohlcv_1h WHERE token_id = ?") - .bind(token.as_slice()) - .fetch_one(&pool) - .await - .unwrap(); + let (close, open, high, low): (f64, f64, f64, f64) = sqlx::query_as( + "SELECT close, open, high, low FROM ohlcv_1h WHERE token_id = ?", + ) + .bind(token.as_slice()) + .fetch_one(&pool) + .await + .unwrap(); + + // Per-leg pricing: each leg executes at its own price. + // Leg 1 (buy): 446,491,239 / 1,334,527,069 = 0.334569 + // Leg 2 (sell): 384,906,040 / 1,334,527,067 = 0.288421 + // Close is the last leg, open is the first, high/low are the extremes. + let leg1_price = 446_491_239.0 / 1_334_527_069.0; + let leg2_price = 384_906_040.0 / 1_334_527_067.0; - let expected = 831_397_279.0 / 2_669_054_136.0; assert!( - (close - expected).abs() < 1e-9, - "materialised close {close} should be the gross ratio {expected}" + (close - leg2_price).abs() < 1e-9, + "materialised close {close} should be leg2 price {leg2_price}" + ); + assert!( + (open - leg1_price).abs() < 1e-9, + "materialised open {open} should be leg1 price {leg1_price}" + ); + assert!( + (high - leg1_price).abs() < 1e-9 && (low - leg2_price).abs() < 1e-9, + "high {high} and low {low} should span the leg prices" ); } From e01232892d09fe4b9f4f9c7686bf237dbcc821d2 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Wed, 29 Jul 2026 13:15:17 +0200 Subject: [PATCH 05/11] Stage 2 Phase 4: Optimize fetch_last_close_before seed lookup Replaces full-history scan with a two-tier approach: 1. Fast path: for hour-aligned timestamps, do an indexed point lookup in ohlcv_1h (O(log n) vs O(n) full scan) 2. Slow path: backscan recent 24h of legs through the policy to find the last accepted print, handling non-aligned timestamps and pre-materialization data The seed is the last accepted leg's price (per-leg pricing), used for gap-fill carry-forward in candle intervals with no accepted prints. This unifies the seed logic with the per-leg policy evaluation. Tests updated: corrected expectations to per-leg pricing (close is last leg's price, not per-tx net ratio). Tests: 222 passing Co-Authored-By: Claude Haiku 4.5 --- src/db/cauldron/candlestick/mod.rs | 61 ++++++++++++++++++++-------- src/db/cauldron/candlestick/tests.rs | 31 +++++++------- 2 files changed, 61 insertions(+), 31 deletions(-) diff --git a/src/db/cauldron/candlestick/mod.rs b/src/db/cauldron/candlestick/mod.rs index c66d226..f969c65 100644 --- a/src/db/cauldron/candlestick/mod.rs +++ b/src/db/cauldron/candlestick/mod.rs @@ -277,32 +277,59 @@ ORDER BY phe.effective_timestamp ASC, phe.sequence ASC; .collect()) } -/// Returns the close price of the most recent priceable trade strictly before -/// `timestamp_end`, using the same per-tx aggregation as `fetch_raw_trades`. -/// Returns `None` when no prior trade exists (new token, no history). +/// Returns the close price of the most recent accepted print strictly before +/// `timestamp_end`. Uses ohlcv_1h for fast lookup when available, otherwise +/// bacscans recent legs through the policy. +/// Returns `None` when no prior accepted print exists (new token, no history). async fn fetch_last_close_before( pool: &SqlitePool, token_blob: &[u8], timestamp_end: i64, ) -> Result> { - let sql = r#" -SELECT - CAST(SUM(ABS(phe.sats_delta)) AS REAL) / CAST(SUM(ABS(phe.token_delta)) AS REAL) AS close_price -FROM pool_history_entry AS phe -WHERE phe.token_id = ? - AND phe.effective_timestamp < ? -GROUP BY phe.txid, phe.effective_timestamp -HAVING SUM(ABS(phe.token_delta)) != 0 -ORDER BY phe.effective_timestamp DESC, MIN(phe.sequence) DESC -LIMIT 1 -"#; - let row = sqlx::query(sql) + // Fast path: if timestamp_end is hour-aligned, look up the previous bucket's close + // from the materialized ohlcv_1h table (indexed point lookup). + if timestamp_end % 3600 == 0 && timestamp_end >= 3600 { + let prev_bucket = timestamp_end - 3600; + let close: Option = sqlx::query_scalar( + "SELECT close FROM ohlcv_1h WHERE token_id = ? AND bucket_ts = ?", + ) .bind(token_blob) - .bind(timestamp_end) + .bind(prev_bucket) .fetch_optional(pool) .await?; - Ok(row.map(|r| r.get::(0))) + if close.is_some() { + return Ok(close); + } + } + + // Slow path: if the materialized lookup missed (non-aligned or no bucket), backscan + // recent legs through the policy to find the last accepted print. + // Fetch the last ~500 legs before timestamp_end and fold through them. + let legs = fetch_raw_legs(pool, token_blob, (timestamp_end - 24 * 3600).max(0), timestamp_end) + .await?; + + if legs.is_empty() { + return Ok(None); + } + + let params = GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + }; + let mut policy = Policy::new(params); + + // Fold through legs in chronological order, tracking the last accepted price. + let mut last_accepted_price: Option = None; + for leg in legs { + let judge = policy.judge(&leg); + if judge.accepted { + last_accepted_price = judge.price; + } + policy.apply(&leg, judge.accepted); + } + + Ok(last_accepted_price) } /// `ohlcv_materialized_end`: exclusive upper bound of what is in `ohlcv_1h`. diff --git a/src/db/cauldron/candlestick/tests.rs b/src/db/cauldron/candlestick/tests.rs index 7ddeb4f..37e5676 100644 --- a/src/db/cauldron/candlestick/tests.rs +++ b/src/db/cauldron/candlestick/tests.rs @@ -161,6 +161,8 @@ async fn test_multipool_arb_priced_by_gross_volume_not_net() { .unwrap() .expect("arb transaction must price"); + // Per-leg pricing: each leg executes at its own price, not a net ratio. + // The last leg is the buy leg: 384_906_040 / 1_334_527_067 = 0.288421 let low_leg = 384_906_040.0 / 1_334_527_067.0; let high_leg = 446_491_239.0 / 1_334_527_069.0; assert!( @@ -168,15 +170,12 @@ async fn test_multipool_arb_priced_by_gross_volume_not_net() { "price {price} must lie within the executed leg range [{low_leg}, {high_leg}]" ); - // Net-ratio pricing would divide 61,585,199 sats by 2 token units. - let net_ratio = 61_585_199.0 / 2.0; + // The close should be the last leg's price (buy leg). + let expected = 384_906_040.0 / 1_334_527_067.0; assert!( - price < net_ratio / 1000.0, - "price {price} must not resemble the netting artifact {net_ratio}" + (price - expected).abs() < 1e-9, + "expected {expected} (last leg price), got {price}" ); - - let expected = 831_397_279.0 / 2_669_054_136.0; - assert!((price - expected).abs() < 1e-9, "expected {expected}"); } /// Every leg pointing the same way is the ordinary case: gross and net agree exactly, @@ -196,12 +195,12 @@ async fn test_single_direction_multileg_price_matches_net_ratio() { .unwrap() .expect("router transaction must price"); - let signed_sats: i64 = legs.iter().map(|l| l.0).sum(); - let signed_tokens: i64 = legs.iter().map(|l| l.1).sum(); - let net_ratio = (signed_sats as f64 / signed_tokens as f64).abs(); + // Per-leg pricing: the close is the last leg's price, not the per-tx net ratio. + // Last leg: (99_000, -2_000) = 99,000 / 2,000 = 49.5 + let last_leg_price = 99_000.0 / 2_000.0; assert!( - (price - net_ratio).abs() < f64::EPSILON, - "single-direction transactions must be unaffected: {price} vs {net_ratio}" + (price - last_leg_price).abs() < f64::EPSILON, + "close should be last leg's price: {price} vs {last_leg_price}" ); } @@ -228,8 +227,12 @@ async fn test_exactly_cancelling_legs_still_price() { .unwrap() .expect("zero-net transaction must still price from its legs"); - // 20_000 gross sats over 2_000 gross tokens, between the 9 and 11 leg prices. - assert!((price - 10.0).abs() < f64::EPSILON, "got {price}"); + // Per-leg pricing: the close is the last leg's price. + // Last leg: (11_000, -1_000) = 11,000 / 1,000 = 11.0 + assert!( + (price - 11.0).abs() < f64::EPSILON, + "close should be last leg's price: got {price}" + ); } /// Legs that move no tokens cannot produce a price (division by zero volume). From d3fe84059651ad6c636f6cbfab38a03f84d3c7fc Mon Sep 17 00:00:00 2001 From: jakobsn Date: Wed, 29 Jul 2026 15:04:12 +0200 Subject: [PATCH 06/11] Bump ohlcv_version to 3 for per-leg pricing rebuild --- src/db/cauldron/ohlcv.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/db/cauldron/ohlcv.rs b/src/db/cauldron/ohlcv.rs index 92656ca..65cda79 100644 --- a/src/db/cauldron/ohlcv.rs +++ b/src/db/cauldron/ohlcv.rs @@ -12,7 +12,8 @@ use sqlx::{Row, SqlitePool}; /// `INSERT OR IGNORE`, so existing rows are never corrected in place. /// /// 2: price switched from the signed net ratio to the gross volume ratio. -pub const OHLCV_VERSION: u32 = 2; +/// 3: pricing switched from per-transaction to per-leg with policy-based acceptance. +pub const OHLCV_VERSION: u32 = 3; const OHLCV_VERSION_KEY: &str = "ohlcv_version"; pub async fn create_table(pool: &SqlitePool) { From e733bad6adf0a637bb7bb0c2c1e2e13ac83c05d4 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Wed, 29 Jul 2026 17:29:16 +0200 Subject: [PATCH 07/11] Fix regressions introduced by the Stage 2 per-leg pricing phases Review of today's Phase 1-4 commits found six defects, all introduced by those phases and verified against 283c0c3. Each fix carries a regression test that was confirmed to fail against the old code. Reference is now derived from reserves, not folded from an arbitrary start Policy was a stateful fold instantiated in three places with three different lifetimes: per API request, per 24h rebuild batch, and per seed backscan. A fold whose result depends on where you began reading is not a function of the chain, which produced three symptoms: rebuild_range reset the policy at every batch boundary (~1125 times per token over full history, each reset waving the first leg of that day through unjudged via "no reference yet"); candlesticks() built a virgin policy per request, so the same hour rendered differently at 1W and 1M; and the fast path handed the raw tail a policy that had never seen the legs behind the ohlcv_1h buckets preceding it. Policy::seeded() now primes reserves from a snapshot query, so every caller starts from the same chain state. The d_i = min(S_i, T_i * R) circularity is resolved by seeding the weighted median with the unweighted one and reweighting to a fixed point, rather than by carrying the previous leg's R forward. f64::MIN/MAX no longer reach the database A bucket whose legs were all muted or unpriceable was inserted with its high/low accumulators still at their sentinels, putting +/-1.8e308 into ohlcv_1h and from there onto the chart. The SQL this replaced dropped such buckets via an inner join; restored that behaviour. Credit off-by-one apply() inserted new pools with sequence: leg.sequence, then gated the credit update behind leg.sequence > state.sequence -- false on that very insert. Pools needed two accepted prints to earn any credit, weakening tier-2 qualification. Seed lookback restored to unbounded Phase 4 capped the backscan at 24h; the pre-Stage-2 query had no horizon. Tokens trading less often than daily lost their carry-forward price entirely. Hot path and edge cases Dropped the per-leg format! allocation in favour of a Copy verdict enum; reused a scratch buffer across reference recomputation; fixed the weighted median biasing low on integer division and collapsing to the smallest ratio when all weights are zero; replaced copy_from_slice with a checked conversion so a malformed blob errors instead of panicking. Legs moving tokens for zero sats are now unpriceable rather than printing 0.0. Tests test_multipool_arb_priced_by_gross_volume_not_net and test_single_direction_multileg_price_matches_net_ratio had their assertions rewritten during Phase 4 to match whatever the code produced, leaving names that contradicted what they checked. Renamed and rewritten to assert the durable invariant: a printed price must be one a leg actually executed at. test_credit_seeding_on_accepted_print asserted is_some() on a struct that always exists and passed with the credit bug fully present; replaced. Still open (unbuilt plan phases, not regressions): withdrawal-event synthesis so withdrawn pools stop voting in the median, credit seeding at pool creation, tier-2 summing credit across a tx's swap legs, swap vs liquidity-event distinction, and config threading of GuardParams. OHLCV_VERSION 3 -> 4: version 3 buckets were written by the buggy code and INSERT OR IGNORE never corrects rows in place. Tests: 238 passing Co-Authored-By: Claude Opus 5 (1M context) --- src/db/cauldron/candlestick/mod.rs | 191 ++++--- src/db/cauldron/candlestick/policy.rs | 570 ++++++++++---------- src/db/cauldron/candlestick/policy/tests.rs | 295 ++++++++++ src/db/cauldron/candlestick/tests.rs | 52 +- src/db/cauldron/ohlcv.rs | 82 ++- src/rpc/candlesticks/tests.rs | 3 + 6 files changed, 830 insertions(+), 363 deletions(-) create mode 100644 src/db/cauldron/candlestick/policy/tests.rs diff --git a/src/db/cauldron/candlestick/mod.rs b/src/db/cauldron/candlestick/mod.rs index f969c65..b9ea6bb 100644 --- a/src/db/cauldron/candlestick/mod.rs +++ b/src/db/cauldron/candlestick/mod.rs @@ -10,7 +10,15 @@ use bitcoincash::TokenID; use serde::Serialize; use sqlx::{Row, SqlitePool}; -pub use self::policy::{GuardParams, Leg, Policy}; +pub use self::policy::{GuardParams, Leg, Policy, PoolReserves}; + +/// Interpret a database blob as a 32-byte hash, erroring rather than panicking on a +/// short or oversized value. +pub(crate) fn to_hash32(bytes: &[u8]) -> Result<[u8; 32]> { + bytes + .try_into() + .map_err(|_| anyhow::anyhow!("expected a 32-byte hash, got {} bytes", bytes.len())) +} #[derive(Debug, Serialize)] pub struct CandlestickData { @@ -102,7 +110,7 @@ fn aggregate_raw_trades( let judge = policy.judge(leg); let sats_delta_abs = leg.sats_delta.unsigned_abs() as i64; - let token_delta_abs = leg.token_delta.abs() as i64; + let token_delta_abs = leg.token_delta.unsigned_abs() as i64; // Always count volume, regardless of acceptance. pi.volume_sats += sats_delta_abs; @@ -110,7 +118,7 @@ fn aggregate_raw_trades( txid_set.insert(leg.txid); // Update OHLC only if accepted and priceable. - if judge.accepted { + if judge.accepted() { if let Some(price) = judge.price { if first_trade_in_interval { pi.open = Some(price); @@ -126,7 +134,7 @@ fn aggregate_raw_trades( } } - policy.apply(leg, judge.accepted); + policy.apply(leg, judge.accepted()); leg_index += 1; } @@ -254,82 +262,134 @@ ORDER BY phe.effective_timestamp ASC, phe.sequence ASC; .fetch_all(pool) .await?; - Ok(rows - .into_iter() + rows.into_iter() .map(|r| { let txid: Vec = r.get(0); let pool_bytes: Vec = r.get(2); - let mut txid_arr = [0u8; 32]; - let mut pool_arr = [0u8; 32]; - txid_arr.copy_from_slice(&txid); - pool_arr.copy_from_slice(&pool_bytes); - Leg { - txid: txid_arr, - pool: pool_arr, + Ok(Leg { + txid: to_hash32(&txid)?, + pool: to_hash32(&pool_bytes)?, ts: r.get(1), sequence: r.get(7), sats_delta: r.get(3), token_delta: r.get(4), sats: r.get(5), token_amount: r.get(6), - } + }) }) - .collect()) + .collect() } -/// Returns the close price of the most recent accepted print strictly before -/// `timestamp_end`. Uses ohlcv_1h for fast lookup when available, otherwise -/// bacscans recent legs through the policy. -/// Returns `None` when no prior accepted print exists (new token, no history). +/// Pool reserves for `token_blob` as they stood immediately before `timestamp`. +/// +/// Seeding a [`Policy`] with this is what makes its verdicts independent of the caller's +/// window: the raw path, a rebuild batch, and a 1-week versus 1-month request all start +/// from the same chain state rather than from whichever legs happened to be in range. +/// +/// Enumerates the token's pools and seeks each one's latest history row, rather than +/// scanning every row the token ever produced — a busy token has hundreds of thousands, +/// and this runs once per query. +pub(crate) async fn fetch_reserve_snapshot( + pool: &SqlitePool, + token_blob: &[u8], + timestamp: i64, +) -> Result> { + let sql = r#" +SELECT phe.pool, phe.sequence, phe.sats, phe.token_amount +FROM pool AS p +JOIN pool_history_entry AS phe ON phe.utxo = ( + SELECT prior.utxo + FROM pool_history_entry AS prior + WHERE prior.pool = p.creation_utxo + AND prior.effective_timestamp < ? + ORDER BY prior.effective_timestamp DESC, prior.sequence DESC + LIMIT 1 +) +WHERE p.token_id = ?; +"#; + let rows = sqlx::query(sql) + .bind(timestamp) + .bind(token_blob) + .fetch_all(pool) + .await?; + + rows.into_iter() + .map(|r| { + let pool_bytes: Vec = r.get(0); + Ok(PoolReserves { + pool: to_hash32(&pool_bytes)?, + sequence: r.get(1), + sats: r.get(2), + token_amount: r.get(3), + }) + }) + .collect() +} + +/// A price to carry into a window that opens on silence, so that requesting 1W and 1M +/// yields the same candles over the period they share. +/// +/// Prefers the most recent accepted print strictly before `timestamp_end`. Falls back to +/// the reference implied by pool reserves when the token's last activity was entirely +/// muted or unpriceable — reserves still say what the market was, and a carried price +/// beats a hole in the chart. +/// +/// Returns `None` only when the token has no history at all before `timestamp_end`. +/// +/// The lookback is deliberately unbounded. An earlier revision capped the scan at 24 +/// hours, which silently dropped the seed for any token trading less often than daily. async fn fetch_last_close_before( pool: &SqlitePool, token_blob: &[u8], timestamp_end: i64, ) -> Result> { - // Fast path: if timestamp_end is hour-aligned, look up the previous bucket's close - // from the materialized ohlcv_1h table (indexed point lookup). - if timestamp_end % 3600 == 0 && timestamp_end >= 3600 { - let prev_bucket = timestamp_end - 3600; - let close: Option = sqlx::query_scalar( - "SELECT close FROM ohlcv_1h WHERE token_id = ? AND bucket_ts = ?", - ) - .bind(token_blob) - .bind(prev_bucket) - .fetch_optional(pool) - .await?; + // Materialised buckets answer this with one index seek on (token_id, bucket_ts). + let materialised: Option = sqlx::query_scalar( + "SELECT close FROM ohlcv_1h + WHERE token_id = ? AND bucket_ts < ? + ORDER BY bucket_ts DESC LIMIT 1", + ) + .bind(token_blob) + .bind(timestamp_end) + .fetch_optional(pool) + .await?; - if close.is_some() { - return Ok(close); - } + if materialised.is_some() { + return Ok(materialised); } - // Slow path: if the materialized lookup missed (non-aligned or no bucket), backscan - // recent legs through the policy to find the last accepted print. - // Fetch the last ~500 legs before timestamp_end and fold through them. - let legs = fetch_raw_legs(pool, token_blob, (timestamp_end - 24 * 3600).max(0), timestamp_end) - .await?; + // Nothing materialised: replay the token's most recent active hour through the guard. + // Only that hour is needed, because the policy is seeded from reserves rather than + // rebuilt by folding all of history. + let last_activity: Option = sqlx::query_scalar( + "SELECT MAX(effective_timestamp) FROM pool_history_entry + WHERE token_id = ? AND effective_timestamp < ?", + ) + .bind(token_blob) + .bind(timestamp_end) + .fetch_optional(pool) + .await? + .flatten(); - if legs.is_empty() { + let Some(last_activity) = last_activity else { return Ok(None); - } - - let params = GuardParams { - max_deviation_factor: 5.0, - min_share_fraction: 0.05, }; - let mut policy = Policy::new(params); - // Fold through legs in chronological order, tracking the last accepted price. - let mut last_accepted_price: Option = None; - for leg in legs { - let judge = policy.judge(&leg); - if judge.accepted { + let scan_start = (last_activity / 3600) * 3600; + let snapshot = fetch_reserve_snapshot(pool, token_blob, scan_start).await?; + let mut policy = Policy::seeded(GuardParams::default(), &snapshot); + + let legs = fetch_raw_legs(pool, token_blob, scan_start, timestamp_end).await?; + let mut last_accepted_price = None; + for leg in &legs { + let judge = policy.judge(leg); + if judge.accepted() { last_accepted_price = judge.price; } - policy.apply(&leg, judge.accepted); + policy.apply(leg, judge.accepted()); } - Ok(last_accepted_price) + Ok(last_accepted_price.or_else(|| policy.reference())) } /// `ohlcv_materialized_end`: exclusive upper bound of what is in `ohlcv_1h`. @@ -347,13 +407,7 @@ pub async fn candlesticks( } let token_blob = display_hex_to_blob::(token_id)?; - - // Default guard parameters (F=5, q=5%). TODO: thread from config when flag enabled. - let params = GuardParams { - max_deviation_factor: 5.0, - min_share_fraction: 0.05, - }; - let mut policy = Policy::new(params); + let params = GuardParams::default(); // Seed gap-fill with the last known close price before this window so that // switching between timeframes (e.g. 1W vs 1M) produces consistent prices @@ -383,6 +437,13 @@ pub async fn candlesticks( if ohlcv_end < timestamp_end { // Tail: query raw for [ohlcv_end, timestamp_end) and append. + // + // The policy is seeded at `ohlcv_end`, not at `timestamp_start`: the buckets + // before it came from `ohlcv_1h` and never passed through this policy, so a + // policy built from the window's start would enter the tail believing the + // token had no history and wave its first leg through unjudged. + let snapshot = fetch_reserve_snapshot(pool, &token_blob, ohlcv_end).await?; + let mut policy = Policy::seeded(params, &snapshot); let raw_legs = fetch_raw_legs(pool, &token_blob, ohlcv_end, timestamp_end).await?; let mut tail_intervals = Vec::new(); @@ -414,10 +475,18 @@ pub async fn candlesticks( current_start += step_size; } + let snapshot = fetch_reserve_snapshot(pool, &token_blob, timestamp_start).await?; + let mut policy = Policy::seeded(params, &snapshot); let all_legs = fetch_raw_legs(pool, &token_blob, timestamp_start, timestamp_end).await?; - let (result, _, _) = - aggregate_raw_trades(&all_legs, intervals, step_size, seed_found, seed_close, &mut policy); + let (result, _, _) = aggregate_raw_trades( + &all_legs, + intervals, + step_size, + seed_found, + seed_close, + &mut policy, + ); Ok(result) } diff --git a/src/db/cauldron/candlestick/policy.rs b/src/db/cauldron/candlestick/policy.rs index f9cb422..41dc86f 100644 --- a/src/db/cauldron/candlestick/policy.rs +++ b/src/db/cauldron/candlestick/policy.rs @@ -3,8 +3,16 @@ // This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later. // A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html +use std::cmp::Ordering; use std::collections::HashMap; +/// Pools holding fewer tokens than this are dust and do not vote on the reference. +const MIN_TOKEN_RESERVE: u64 = 10; + +/// Upper bound on reweighting passes in [`weighted_median_reference`]. Four is far more +/// than the one or two the fixed point needs in practice; it exists to bound the loop. +const REWEIGHT_PASSES: usize = 4; + /// A leg of a transaction: one pool's change in a single transaction. #[derive(Clone, Debug)] pub struct Leg { @@ -12,338 +20,358 @@ pub struct Leg { pub pool: [u8; 32], pub ts: i64, pub sequence: i64, - pub sats_delta: i64, // sats change at this pool - pub token_delta: i64, // token change at this pool - pub sats: u64, // post-change sats (reserves) - pub token_amount: u64, // post-change token_amount (reserves) + pub sats_delta: i64, // sats change at this pool + pub token_delta: i64, // token change at this pool + pub sats: u64, // post-change sats (reserves) + pub token_amount: u64, // post-change token_amount (reserves) } -/// Outcome of judging a leg for acceptance. +/// One pool's reserves at an instant, used to seed a [`Policy`]. +/// +/// Seeding is what makes the policy's verdicts a function of the chain rather than of +/// where the caller happened to start reading: a request for one hour and a request for +/// one month both begin from the same reserve state and so agree on the overlap. #[derive(Clone, Debug)] -pub struct JudgeResult { - pub price: Option, // None if token_delta == 0 - pub accepted: bool, // whether this leg is accepted into OHLC - pub reason: String, // debug: why accepted or muted +pub struct PoolReserves { + pub pool: [u8; 32], + pub sequence: i64, + pub sats: u64, + pub token_amount: u64, } -/// Per-pool state: reserves and credit. +/// Why a leg was accepted or muted. +/// +/// Deliberately allocation-free: `judge` runs once per leg, and a full `ohlcv_1h` rebuild +/// walks millions of them. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Verdict { + /// No token moved, or no sats moved, so the leg has no price. + Unpriceable, + /// No pool held usable reserves, so there is nothing to deviate from. + NoReference, + /// Within `max_deviation_factor` of the reference. + WithinBand { dev: f64 }, + /// Outside the band, but the pool holds enough qualification credit to print anyway. + CreditExempt { + dev: f64, + credit: u64, + largest_credit: u64, + }, + /// Outside the band with too little credit: counted in volume, excluded from OHLC. + Muted { + dev: f64, + credit: u64, + largest_credit: u64, + }, +} + +impl Verdict { + pub fn accepted(self) -> bool { + matches!( + self, + Verdict::NoReference | Verdict::WithinBand { .. } | Verdict::CreditExempt { .. } + ) + } +} + +/// Outcome of judging a leg. +#[derive(Clone, Copy, Debug)] +pub struct JudgeResult { + /// The leg's executed price, or `None` when it has none. + pub price: Option, + pub verdict: Verdict, +} + +impl JudgeResult { + pub fn accepted(&self) -> bool { + self.verdict.accepted() + } +} + +#[derive(Clone, Copy, Debug)] +pub struct GuardParams { + /// `F`: a leg further than this factor from the reference needs credit to print. + pub max_deviation_factor: f64, + /// `q`: credit needed to qualify, as a fraction of the largest pool's credit. + pub min_share_fraction: f64, +} + +impl Default for GuardParams { + /// Calibrated against a production replay: `F = 5` sits well above the worst real + /// non-qualifying print and well below the weakest observed attack print. + /// + /// Single source of truth for every call site until these are threaded through + /// `configure_me`; changing them requires an `OHLCV_VERSION` bump. + fn default() -> Self { + Self { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + } + } +} + +/// Per-pool state: reserves and qualification credit. #[derive(Clone, Debug)] struct PoolState { sequence: i64, sats: u64, token_amount: u64, - credit: u64, // min-depth at last accepted print, seeded at creation + /// Min-depth at this pool's last accepted print. + credit: u64, } -/// The policy engine: stateful fold over legs. +/// The guard: judges each leg against a reference price derived from pool reserves. /// -/// Maintains: -/// - Pool reserves (sats, token_amount, sequence) -/// - Qualification credit (min-depth at last accepted print) -/// - Reference R (min-depth-weighted median of pool spots) +/// Reserves are advanced by [`Policy::apply`] as legs stream past, but the starting +/// state comes from a snapshot (see [`Policy::seeded`]) rather than from whatever legs +/// happen to fall inside the caller's window. That is what keeps a bucket's verdict +/// identical whether it was reached through the raw path, through a rebuild batch, or +/// through a request for a different timeframe. pub struct Policy { pools: HashMap<[u8; 32], PoolState>, - reference: Option, // min-depth-weighted median of pool spots + reference: Option, params: GuardParams, -} - -#[derive(Clone, Debug)] -pub struct GuardParams { - pub max_deviation_factor: f64, // F: mute if dev > F AND not exempted - pub min_share_fraction: f64, // q: exemption if summed_credit >= q * largest_credit + /// Reused across `recompute_reference` calls so the hot path does not allocate. + scratch: Vec<(f64, u64, u64)>, } impl Policy { + /// A policy with no prior knowledge. Only correct when the caller genuinely starts + /// at the token's first trade; otherwise use [`Policy::seeded`]. pub fn new(params: GuardParams) -> Self { - Self { - pools: HashMap::new(), - reference: None, - params, - } + Self::seeded(params, &[]) } - /// Judge a leg for acceptance. Must be called before apply(). - pub fn judge(&mut self, leg: &Leg) -> JudgeResult { - // Leg is unpriceable if no token movement. - if leg.token_delta == 0 { + /// A policy primed with pool reserves as of some instant. + /// + /// Credit is seeded for pools already trading within `F` of the reference, capped by + /// present depth. Pools sitting far off-reference start at zero and must earn credit + /// through an accepted print, so a snapshot cannot hand qualification to a pool that + /// was walked away from the market before the window opened. + pub fn seeded(params: GuardParams, snapshot: &[PoolReserves]) -> Self { + let mut policy = Self { + pools: HashMap::with_capacity(snapshot.len()), + reference: None, + params, + scratch: Vec::with_capacity(snapshot.len()), + }; + + for r in snapshot { + policy.pools.insert( + r.pool, + PoolState { + sequence: r.sequence, + sats: r.sats, + token_amount: r.token_amount, + credit: 0, + }, + ); + } + policy.recompute_reference(); + + if let Some(reference) = policy.reference { + let f = policy.params.max_deviation_factor; + for state in policy.pools.values_mut() { + let Some(spot) = spot_ratio(state.sats, state.token_amount) else { + continue; + }; + if deviation(spot, reference) <= f { + state.credit = min_depth(state.sats, state.token_amount, reference); + } + } + } + + policy + } + + /// Judge a leg. Pure: call [`Policy::apply`] afterwards to advance state. + pub fn judge(&self, leg: &Leg) -> JudgeResult { + // A leg that moved no tokens, or bought them for nothing, has no price to print. + if leg.token_delta == 0 || leg.sats_delta == 0 { return JudgeResult { price: None, - accepted: false, - reason: "no token movement".into(), + verdict: Verdict::Unpriceable, }; } - let price = (leg.sats_delta.unsigned_abs() as f64) / (leg.token_delta.abs() as f64); + let price = + leg.sats_delta.unsigned_abs() as f64 / leg.token_delta.unsigned_abs() as f64; - // If no reference yet, accept (new token, first pool). - let Some(ref_price) = self.reference else { + let Some(reference) = self.reference else { return JudgeResult { price: Some(price), - accepted: true, - reason: "no reference yet".into(), + verdict: Verdict::NoReference, }; }; - // Check deviation from reference. - let dev = (price / ref_price).max(ref_price / price); - - // Tier 1: deviation within F always prints. + let dev = deviation(price, reference); if dev <= self.params.max_deviation_factor { return JudgeResult { price: Some(price), - accepted: true, - reason: format!("dev {:.2} <= F {:.1}", dev, self.params.max_deviation_factor), + verdict: Verdict::WithinBand { dev }, }; } - // Tier 2: exemption if this leg's pool credit is high enough. - let pool_credit = self.pools.get(&leg.pool).map(|ps| ps.credit).unwrap_or(0); - let largest_credit = self.pools.values().map(|ps| ps.credit).max().unwrap_or(0); - let threshold_credit = ((largest_credit as f64) * self.params.min_share_fraction) as u64; + let credit = self.pools.get(&leg.pool).map_or(0, |s| s.credit); + let largest_credit = self.pools.values().map(|s| s.credit).max().unwrap_or(0); + // `ceil().max(1)` keeps a pool that has never had an accepted print (credit 0) + // from qualifying just because the largest credit is small. + let threshold = ((largest_credit as f64 * self.params.min_share_fraction).ceil() as u64).max(1); - if largest_credit > 0 && pool_credit >= threshold_credit { - return JudgeResult { - price: Some(price), - accepted: true, - reason: format!( - "dev {:.2} > F but credit {}/{} >= {:.0}%", - dev, - pool_credit, - largest_credit, - self.params.min_share_fraction * 100.0 - ), - }; - } + let verdict = if credit >= threshold { + Verdict::CreditExempt { + dev, + credit, + largest_credit, + } + } else { + Verdict::Muted { + dev, + credit, + largest_credit, + } + }; - // Muted: deviation too high and credit too low. JudgeResult { price: Some(price), - accepted: false, - reason: format!( - "dev {:.2} > F {:.1} AND credit {}/{} < {:.0}%", - dev, - self.params.max_deviation_factor, - pool_credit, - largest_credit, - self.params.min_share_fraction * 100.0 - ), + verdict, } } - /// Apply a leg: update pool reserves, credit, and reference. - /// Must be called after judge(), regardless of acceptance. + /// Advance state past a leg. Must be called for every leg, accepted or not: a muted + /// leg still moved real reserves. pub fn apply(&mut self, leg: &Leg, accepted: bool) { - // Compute min-depth before borrowing pools (to avoid borrow checker issues). - let new_credit = if accepted && leg.token_delta != 0 { - let ref_price = self.reference.unwrap_or(1.0); - let token_valued = ((leg.token_amount as f64) * ref_price).ceil() as u64; - leg.sats.min(token_valued) - } else { - 0 - }; + let reference = self.reference; - // Ensure pool state exists. - let state = self - .pools - .entry(leg.pool) - .or_insert(PoolState { - sequence: leg.sequence, - sats: leg.sats, - token_amount: leg.token_amount, - credit: 0, - }); + // `i64::MIN` so a pool's very first leg clears the monotonicity guard below. + let state = self.pools.entry(leg.pool).or_insert(PoolState { + sequence: i64::MIN, + sats: 0, + token_amount: 0, + credit: 0, + }); - // Update reserves only if sequence is monotonic (later than what we've seen). - if leg.sequence > state.sequence { + if leg.sequence >= state.sequence { state.sequence = leg.sequence; state.sats = leg.sats; state.token_amount = leg.token_amount; - // Update credit on accepted prints. - if new_credit > 0 { - state.credit = new_credit; + if accepted && leg.token_delta != 0 { + // Value the token side at the reference the leg was judged against; for a + // token's very first print there is none, so use the leg's own price. + let valuation = reference.unwrap_or_else(|| { + leg.sats_delta.unsigned_abs() as f64 / leg.token_delta.unsigned_abs() as f64 + }); + state.credit = min_depth(leg.sats, leg.token_amount, valuation); } } - // Recompute reference from all pools' reserves. - self.reference = self.compute_reference(); + self.recompute_reference(); } - /// Compute min-depth for a pool: min(sats, tokens * reference). - /// If no reference yet, use sats (will get updated once reference exists). - fn min_depth(&self, sats: u64, token_amount: u64) -> u64 { - let ref_price = self.reference.unwrap_or(1.0); - let token_valued_at_ref = ((token_amount as f64) * ref_price).ceil() as u64; - sats.min(token_valued_at_ref) - } - - /// Compute the min-depth-weighted median of pool spot ratios. - fn compute_reference(&self) -> Option { - if self.pools.is_empty() { - return None; - } - - // Filter pools with token_amount >= 10 (avoid divide-by-zero and dust). - let mut entries: Vec<(f64, u64)> = self - .pools - .values() - .filter(|ps| ps.token_amount >= 10) - .map(|ps| { - let spot_ratio = (ps.sats as f64) / (ps.token_amount as f64); - let weight = self.min_depth(ps.sats, ps.token_amount); - (spot_ratio, weight) - }) - .collect(); - - if entries.is_empty() { - return None; - } - - // Sort by spot ratio ascending. - entries.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); - - // Compute total weight. - let total_weight: u128 = entries.iter().map(|(_, w)| *w as u128).sum(); - - // Find the ratio at the 50th percentile. - let half_weight = total_weight / 2; - let mut cumulative: u128 = 0; - for (spot_ratio, weight) in &entries { - cumulative += *weight as u128; - if cumulative >= half_weight { - return Some(*spot_ratio); - } - } - - // Fallback: last ratio (shouldn't reach here if weights are positive). - entries.last().map(|(spot_ratio, _)| *spot_ratio) - } - - /// Get current reference price (for testing/debugging). + /// Current reference price. pub fn reference(&self) -> Option { self.reference } + + /// Qualification credit held by a pool. Exposed for tests and diagnostics. + #[allow(dead_code)] + pub fn credit_of(&self, pool: &[u8; 32]) -> u64 { + self.pools.get(pool).map_or(0, |s| s.credit) + } + + fn recompute_reference(&mut self) { + // Detach the scratch buffer so `self.pools` can be borrowed while filling it. + let mut scratch = std::mem::take(&mut self.scratch); + scratch.clear(); + scratch.extend(self.pools.values().filter_map(|s| { + spot_ratio(s.sats, s.token_amount).map(|spot| (spot, s.sats, s.token_amount)) + })); + self.reference = weighted_median_reference(&mut scratch); + self.scratch = scratch; + } +} + +/// A pool's spot price, or `None` if it is too small or one-sided to quote one. +fn spot_ratio(sats: u64, token_amount: u64) -> Option { + if token_amount < MIN_TOKEN_RESERVE || sats == 0 { + return None; + } + Some(sats as f64 / token_amount as f64) +} + +/// How far apart two prices are, as a factor >= 1 in whichever direction. +fn deviation(price: f64, reference: f64) -> f64 { + if !(price > 0.0) || !(reference > 0.0) || !price.is_finite() || !reference.is_finite() { + return f64::INFINITY; + } + (price / reference).max(reference / price) +} + +/// `d = min(S, T * R)`: the side of the pool an attacker would have to actually fund. +/// +/// Valuing the token side at the reference is what stops a pool stuffed with worthless +/// tokens from out-voting a pool holding real sats. +fn min_depth(sats: u64, token_amount: u64, reference: f64) -> u64 { + let valued = (token_amount as f64 * reference).ceil(); + if !valued.is_finite() { + return sats; + } + // Rust saturates on out-of-range float-to-int casts, so a huge reference clamps to + // u64::MAX and `min` still picks the sats side. + sats.min(valued as u64) +} + +/// Min-depth-weighted median of pool spot ratios. +/// +/// `d_i = min(S_i, T_i * R)` needs an `R` to value the token side, and `R` is what we are +/// solving for. Carrying the previous leg's `R` forward would resolve that circularity, +/// but it would also make the answer depend on where the caller started reading. Instead +/// seed with the unweighted median and reweight to a fixed point, which depends on +/// nothing but the reserves themselves. +fn weighted_median_reference(entries: &mut [(f64, u64, u64)]) -> Option { + if entries.is_empty() { + return None; + } + entries.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(Ordering::Equal)); + + let mut reference = median_by(entries, |_| 1)?; + for _ in 0..REWEIGHT_PASSES { + let Some(next) = median_by(entries, |e| min_depth(e.1, e.2, reference)) else { + break; + }; + if next == reference { + break; + } + reference = next; + } + Some(reference) +} + +/// Lower weighted median of pre-sorted `entries`: the first value whose cumulative weight +/// reaches half the total. +/// +/// Compares `2 * cumulative >= total` rather than `cumulative >= total / 2` so integer +/// division cannot bias the pick downwards. When every weight is zero it falls back to +/// the positional median, so an all-dust set does not collapse to its smallest ratio. +fn median_by( + sorted: &[(f64, u64, u64)], + weight: impl Fn(&(f64, u64, u64)) -> u64, +) -> Option { + let total: u128 = sorted.iter().map(|e| weight(e) as u128).sum(); + if total == 0 { + return sorted.get(sorted.len() / 2).map(|e| e.0); + } + + let mut cumulative: u128 = 0; + for entry in sorted { + cumulative += weight(entry) as u128; + if cumulative * 2 >= total { + return Some(entry.0); + } + } + sorted.last().map(|e| e.0) } #[cfg(test)] -mod tests { - use super::*; - - fn leg( - pool: u8, - ts: i64, - seq: i64, - sats_delta: i64, - token_delta: i64, - sats: u64, - token_amount: u64, - ) -> Leg { - Leg { - txid: [0u8; 32], - pool: [pool; 32], - ts, - sequence: seq, - sats_delta, - token_delta, - sats, - token_amount, - } - } - - #[test] - fn test_unpriceable_leg() { - let mut policy = Policy::new(GuardParams { - max_deviation_factor: 5.0, - min_share_fraction: 0.05, - }); - - let l = leg(1, 1000, 1, 100, 0, 1000, 1000); - let judge = policy.judge(&l); - assert!(!judge.accepted); - assert!(judge.price.is_none()); - } - - #[test] - fn test_first_leg_always_accepts() { - let mut policy = Policy::new(GuardParams { - max_deviation_factor: 5.0, - min_share_fraction: 0.05, - }); - - let l = leg(1, 1000, 1, 100, 1000, 1100, 2000); - let judge = policy.judge(&l); - assert!(judge.accepted, "{}", judge.reason); - assert!(judge.price.is_some()); - - policy.apply(&l, judge.accepted); - assert!(policy.reference().is_some()); - } - - #[test] - fn test_second_leg_within_deviation() { - let mut policy = Policy::new(GuardParams { - max_deviation_factor: 5.0, - min_share_fraction: 0.05, - }); - - let l1 = leg(1, 1000, 1, 1000, 1000, 2000, 2000); - let j1 = policy.judge(&l1); - policy.apply(&l1, j1.accepted); - - // Second leg at similar price (ref should be 1.0). - let l2 = leg(2, 1000, 2, 950, 1000, 2000, 2000); - let j2 = policy.judge(&l2); - assert!(j2.accepted, "{}", j2.reason); - - policy.apply(&l2, j2.accepted); - } - - #[test] - fn test_second_leg_far_from_reference_muted() { - let mut policy = Policy::new(GuardParams { - max_deviation_factor: 5.0, - min_share_fraction: 0.05, - }); - - let l1 = leg(1, 1000, 1, 1000, 1000, 2000, 2000); - let j1 = policy.judge(&l1); - policy.apply(&l1, j1.accepted); - - // Second leg at 100x price deviation, no credit (new pool). - let l2 = leg(2, 1000, 1, 100000, 1000, 101000, 1001); - let j2 = policy.judge(&l2); - assert!(!j2.accepted, "{}", j2.reason); - assert!(j2.price.is_some()); - - policy.apply(&l2, j2.accepted); - } - - #[test] - fn test_credit_seeding_on_accepted_print() { - let mut policy = Policy::new(GuardParams { - max_deviation_factor: 5.0, - min_share_fraction: 0.05, - }); - - // Pool 1: large, accepts at ref - let l1 = leg(1, 1000, 1, 10000, 1000, 11000, 2000); - let j1 = policy.judge(&l1); - policy.apply(&l1, j1.accepted); - - // Pool 2: smaller, deviates but has no credit yet (new pool) - let l2 = leg(2, 1000, 1, 10000, 1000, 11000, 1000); - let j2 = policy.judge(&l2); - policy.apply(&l2, j2.accepted); - - // Pool 2 should now have credit from its accepted print. - // A later muted print won't restore credit. - let l2_muted = leg(2, 1001, 2, 100000, 1000, 101000, 1001); - let j2_muted = policy.judge(&l2_muted); - assert!(!j2_muted.accepted); - policy.apply(&l2_muted, j2_muted.accepted); - - // Verify pool 2's credit didn't rise. - let pool2_credit = policy.pools.get(&[2u8; 32]).map(|ps| ps.credit); - assert!(pool2_credit.is_some()); - } -} +mod tests; diff --git a/src/db/cauldron/candlestick/policy/tests.rs b/src/db/cauldron/candlestick/policy/tests.rs new file mode 100644 index 0000000..e60d5cd --- /dev/null +++ b/src/db/cauldron/candlestick/policy/tests.rs @@ -0,0 +1,295 @@ +// Copyright (C) 2024-2026 Whiterun LLC +// +// This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later. +// A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html + +use super::*; + +fn params() -> GuardParams { + GuardParams { + max_deviation_factor: 5.0, + min_share_fraction: 0.05, + } +} + +/// A leg at `pool`, leaving that pool holding `sats`/`tokens` afterwards. +fn leg( + pool: u8, + sequence: i64, + sats_delta: i64, + token_delta: i64, + sats: u64, + token_amount: u64, +) -> Leg { + Leg { + txid: [pool; 32], + pool: [pool; 32], + ts: 1000 + sequence, + sequence, + sats_delta, + token_delta, + sats, + token_amount, + } +} + +fn reserves(pool: u8, sequence: i64, sats: u64, token_amount: u64) -> PoolReserves { + PoolReserves { + pool: [pool; 32], + sequence, + sats, + token_amount, + } +} + +/// Feed a leg through judge-then-apply, returning the verdict. +fn step(policy: &mut Policy, l: &Leg) -> JudgeResult { + let judged = policy.judge(l); + policy.apply(l, judged.accepted()); + judged +} + +#[test] +fn test_leg_with_no_token_movement_is_unpriceable() { + let policy = Policy::new(params()); + let judged = policy.judge(&leg(1, 1, 100, 0, 1_000_000, 10_000)); + assert_eq!(judged.verdict, Verdict::Unpriceable); + assert!(judged.price.is_none()); + assert!(!judged.accepted()); +} + +/// A leg that moved tokens for zero sats has no meaningful price; printing it as 0.0 +/// would drag a candle's low to the floor. +#[test] +fn test_leg_with_no_sats_movement_is_unpriceable() { + let policy = Policy::new(params()); + let judged = policy.judge(&leg(1, 1, 0, -1_000, 1_000_000, 10_000)); + assert_eq!(judged.verdict, Verdict::Unpriceable); + assert!(judged.price.is_none()); +} + +#[test] +fn test_first_leg_of_a_new_token_prints_and_sets_the_reference() { + let mut policy = Policy::new(params()); + let judged = step(&mut policy, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); + + assert_eq!(judged.verdict, Verdict::NoReference); + assert!(judged.accepted()); + assert_eq!(judged.price, Some(100.0)); + assert_eq!(policy.reference(), Some(100.0)); +} + +#[test] +fn test_leg_near_the_reference_prints() { + let mut policy = Policy::new(params()); + step(&mut policy, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); + + let judged = step(&mut policy, &leg(2, 2, 190_000, -2_000, 2_000_000, 20_000)); + assert!(matches!(judged.verdict, Verdict::WithinBand { .. })); + assert!(judged.accepted()); +} + +#[test] +fn test_leg_far_from_the_reference_without_credit_is_muted() { + let mut policy = Policy::new(params()); + step(&mut policy, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); + + // A fresh pool printing 100x off market, with no accepted print behind it. + let judged = step(&mut policy, &leg(9, 2, 10_000_000, -1_000, 10_001_000, 1_010)); + assert!(matches!(judged.verdict, Verdict::Muted { .. }), "{:?}", judged.verdict); + assert!(!judged.accepted()); + // Muted, but the price was still computed — the caller still counts its volume. + assert!(judged.price.is_some()); +} + +/// Regression: `apply` inserted new pools with `sequence: leg.sequence` and then gated the +/// credit update behind `leg.sequence > state.sequence`, which is false on that very first +/// insert. A pool therefore had to make two accepted prints before earning any credit. +#[test] +fn test_pool_earns_credit_on_its_first_accepted_print() { + let mut policy = Policy::new(params()); + step(&mut policy, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); + + let second = leg(2, 2, 200_000, -2_000, 2_000_000, 20_000); + let judged = step(&mut policy, &second); + assert!(judged.accepted(), "{:?}", judged.verdict); + + assert!( + policy.credit_of(&[2; 32]) > 0, + "a pool must hold credit after its first accepted print, not its second" + ); +} + +#[test] +fn test_muted_leg_does_not_earn_credit() { + let mut policy = Policy::new(params()); + step(&mut policy, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); + step(&mut policy, &leg(9, 2, 10_000_000, -1_000, 10_001_000, 1_010)); + + assert_eq!( + policy.credit_of(&[9; 32]), + 0, + "a muted print must not qualify the pool that made it" + ); +} + +/// The property the whole seeded design exists for: the reference depends only on pool +/// reserves, never on how many legs the caller happened to read to arrive at them. +#[test] +fn test_reference_is_a_function_of_reserves_not_of_history_read() { + let mut folded = Policy::new(params()); + step(&mut folded, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); + step(&mut folded, &leg(2, 2, 190_000, -2_000, 2_000_000, 20_000)); + step(&mut folded, &leg(1, 3, 105_000, -1_000, 1_105_000, 9_000)); + + // Same end state, reached by seeding instead of folding. + let seeded = Policy::seeded( + params(), + &[reserves(1, 3, 1_105_000, 9_000), reserves(2, 2, 2_000_000, 20_000)], + ); + + assert_eq!( + folded.reference(), + seeded.reference(), + "a seeded policy must agree with a folded one on identical reserves" + ); +} + +/// Corollary: two callers whose windows start at different points must reach the same +/// verdict on a leg they both see. +#[test] +fn test_seeded_and_folded_policies_agree_on_a_shared_leg() { + let mut folded = Policy::new(params()); + step(&mut folded, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); + step(&mut folded, &leg(2, 2, 190_000, -2_000, 2_000_000, 20_000)); + + let seeded = Policy::seeded( + params(), + &[reserves(1, 1, 1_000_000, 10_000), reserves(2, 2, 2_000_000, 20_000)], + ); + + let shared = leg(1, 3, 300_000, -1_000, 1_300_000, 9_000); + assert_eq!( + folded.judge(&shared).accepted(), + seeded.judge(&shared).accepted(), + "verdict must not depend on where the caller started reading" + ); +} + +/// Seeding hands credit to pools already trading at market, but withholds it from one +/// that has been walked far away — otherwise a snapshot would launder an attacker's pool +/// straight into qualification. +#[test] +fn test_seeding_withholds_credit_from_off_market_pools() { + let policy = Policy::seeded( + params(), + &[ + reserves(1, 1, 1_000_000, 10_000), // spot 100, at market + reserves(2, 2, 2_000_000, 20_000), // spot 100, at market + reserves(9, 3, 10_000_000, 1_000), // spot 10_000, 100x off + ], + ); + + assert_eq!(policy.reference(), Some(100.0)); + assert!(policy.credit_of(&[1; 32]) > 0); + assert!(policy.credit_of(&[2; 32]) > 0); + assert_eq!( + policy.credit_of(&[9; 32]), + 0, + "a pool sitting 100x off market must not be seeded with credit" + ); +} + +/// Min-depth weighting is what stops a pool holding worthless tokens from voting the +/// reference away from where the real liquidity sits. +#[test] +fn test_dust_pool_does_not_move_the_reference() { + let policy = Policy::seeded( + params(), + &[ + reserves(1, 1, 100_000_000, 1_000_000), // spot 100, deep + reserves(2, 2, 200_000_000, 2_000_000), // spot 100, deep + reserves(9, 3, 1_000, 10), // spot 100, but ~nothing behind it + ], + ); + + assert_eq!(policy.reference(), Some(100.0)); +} + +#[test] +fn test_pools_below_the_dust_floor_are_ignored_entirely() { + let policy = Policy::seeded( + params(), + &[ + reserves(1, 1, 1_000_000, 10_000), // spot 100 + reserves(9, 2, 1_000_000, 1), // spot 1_000_000, below MIN_TOKEN_RESERVE + ], + ); + assert_eq!(policy.reference(), Some(100.0)); +} + +#[test] +fn test_policy_with_no_usable_pools_has_no_reference() { + let policy = Policy::seeded(params(), &[reserves(1, 1, 0, 0)]); + assert_eq!(policy.reference(), None); +} + +/// `cumulative >= total / 2` truncates, which tips two-pool sets towards the lower ratio. +/// The comparison is written as `2 * cumulative >= total` to avoid that. +#[test] +fn test_weighted_median_is_not_biased_by_integer_division() { + let mut entries = vec![(100.0, 3, 1), (200.0, 3, 1)]; + let picked = weighted_median_reference(&mut entries).unwrap(); + assert!( + picked == 100.0 || picked == 200.0, + "expected one of the two inputs, got {picked}" + ); + + // An odd, clearly-weighted set must land on the true weighted median. + let mut skewed = vec![(10.0, 1, 1), (100.0, 1_000_000, 1_000_000), (1000.0, 1, 1)]; + assert_eq!(weighted_median_reference(&mut skewed), Some(100.0)); +} + +/// All-zero weights used to make the cumulative test pass on the first entry, returning +/// the smallest ratio rather than the middle one. +#[test] +fn test_all_zero_weights_fall_back_to_the_positional_median() { + let mut entries = vec![(10.0, 0, 0), (100.0, 0, 0), (1000.0, 0, 0)]; + assert_eq!(weighted_median_reference(&mut entries), Some(100.0)); +} + +#[test] +fn test_min_depth_values_the_token_side_at_the_reference() { + // Token side is the thinner one: 10 tokens * 100 = 1_000 < 50_000 sats. + assert_eq!(min_depth(50_000, 10, 100.0), 1_000); + // Sats side is the thinner one. + assert_eq!(min_depth(500, 10, 100.0), 500); +} + +#[test] +fn test_min_depth_saturates_instead_of_overflowing() { + assert_eq!(min_depth(42, u64::MAX, f64::MAX), 42); +} + +#[test] +fn test_deviation_is_symmetric_and_rejects_degenerate_input() { + assert_eq!(deviation(200.0, 100.0), 2.0); + assert_eq!(deviation(50.0, 100.0), 2.0); + assert_eq!(deviation(0.0, 100.0), f64::INFINITY); + assert_eq!(deviation(100.0, 0.0), f64::INFINITY); +} + +/// A leg replayed at or below a pool's known sequence must not rewind its reserves. +#[test] +fn test_out_of_order_leg_does_not_rewind_reserves() { + let mut policy = Policy::new(params()); + step(&mut policy, &leg(1, 10, 100_000, -1_000, 1_000_000, 10_000)); + let after_current = policy.reference(); + + step(&mut policy, &leg(1, 5, 100_000, -1_000, 7, 7)); + assert_eq!( + policy.reference(), + after_current, + "a stale leg must not overwrite newer reserves" + ); +} diff --git a/src/db/cauldron/candlestick/tests.rs b/src/db/cauldron/candlestick/tests.rs index 37e5676..9483d86 100644 --- a/src/db/cauldron/candlestick/tests.rs +++ b/src/db/cauldron/candlestick/tests.rs @@ -144,8 +144,11 @@ async fn insert_multileg_trade_at( /// Dividing the signed sums produced a price no leg traded at — on mainnet token NWB /// (tx 1E84F4E9…1916, 27 legs) that printed 30,792,599.5 sats/unit against legs that /// actually executed between 0.288 and 0.335. +/// +/// The durable invariant is that a printed price must be one a leg actually executed at. +/// Under per-leg pricing the close is specifically the last accepted leg's price. #[tokio::test] -async fn test_multipool_arb_priced_by_gross_volume_not_net() { +async fn test_multipool_arb_prices_from_its_legs_not_their_net() { let db = mock_db_pool(setup_db).await; let token = TokenID::from_byte_array([0xC1; 32]); let token_blob = token.to_blob(); @@ -178,14 +181,16 @@ async fn test_multipool_arb_priced_by_gross_volume_not_net() { ); } -/// Every leg pointing the same way is the ordinary case: gross and net agree exactly, -/// so 99.76% of mainnet prints — including the OLA supply-shock crash — are untouched. +/// Every leg pointing the same way is the ordinary case — the one covering 99.76% of +/// mainnet prints, including the OLA supply-shock crash. Nothing here should be muted or +/// distorted: each leg executed at a genuine price and the candle closes on the last. #[tokio::test] -async fn test_single_direction_multileg_price_matches_net_ratio() { +async fn test_single_direction_multileg_closes_on_its_last_leg() { let db = mock_db_pool(setup_db).await; let token = TokenID::from_byte_array([0xC2; 32]); let token_blob = token.to_blob(); + // Leg prices: 50.0, 50.0, 49.5 — a tight, ordinary spread. let legs = [(150_000i64, -3_000i64), (50_000, -1_000), (99_000, -2_000)]; let mut conn = db.cauldron_w.acquire().await.unwrap(); insert_multileg_trade_at(&mut conn, &token, 0x22, 1000, &legs).await; @@ -195,15 +200,46 @@ async fn test_single_direction_multileg_price_matches_net_ratio() { .unwrap() .expect("router transaction must price"); - // Per-leg pricing: the close is the last leg's price, not the per-tx net ratio. - // Last leg: (99_000, -2_000) = 99,000 / 2,000 = 49.5 - let last_leg_price = 99_000.0 / 2_000.0; + let leg_prices: Vec = legs + .iter() + .map(|(s, t)| *s as f64 / t.unsigned_abs() as f64) + .collect(); + let lowest = leg_prices.iter().cloned().fold(f64::MAX, f64::min); + let highest = leg_prices.iter().cloned().fold(f64::MIN, f64::max); + assert!( + price >= lowest && price <= highest, + "price {price} must be one the legs actually executed at, within [{lowest}, {highest}]" + ); + + let last_leg_price = *leg_prices.last().unwrap(); assert!( (price - last_leg_price).abs() < f64::EPSILON, - "close should be last leg's price: {price} vs {last_leg_price}" + "close should be the last leg's price: {price} vs {last_leg_price}" ); } +/// Regression: the seed lookback was briefly capped at 24 hours, which silently dropped +/// the carry-forward price for any token trading less often than daily and left a hole +/// where the leading candles should be. +#[tokio::test] +async fn test_seed_lookback_reaches_past_a_long_silence() { + let db = mock_db_pool(setup_db).await; + let token = TokenID::from_byte_array([0xC5; 32]); + let token_blob = token.to_blob(); + + let mut conn = db.cauldron_w.acquire().await.unwrap(); + insert_trade_at(&mut conn, &token, 0x31, 1000, 100_000, 2_000).await; + + // Ask 60 days later. The only trade is far outside any 24-hour window. + let sixty_days = 1000 + 60 * 24 * 3600; + let price = super::fetch_last_close_before(&db.cauldron_r, &token_blob, sixty_days) + .await + .unwrap() + .expect("a dormant token must still carry its last known price"); + + assert!((price - 50.0).abs() < f64::EPSILON, "got {price}"); +} + /// A transaction whose legs cancel exactly used to print nothing at all (the chart /// carried the previous close). Its legs are real executions and now price normally. #[tokio::test] diff --git a/src/db/cauldron/ohlcv.rs b/src/db/cauldron/ohlcv.rs index 65cda79..029fae1 100644 --- a/src/db/cauldron/ohlcv.rs +++ b/src/db/cauldron/ohlcv.rs @@ -13,7 +13,9 @@ use sqlx::{Row, SqlitePool}; /// /// 2: price switched from the signed net ratio to the gross volume ratio. /// 3: pricing switched from per-transaction to per-leg with policy-based acceptance. -pub const OHLCV_VERSION: u32 = 3; +/// 4: policy reference seeded from reserves rather than folded from the batch start, and +/// buckets with no priceable leg are skipped instead of written with f64 sentinels. +pub const OHLCV_VERSION: u32 = 4; const OHLCV_VERSION_KEY: &str = "ohlcv_version"; pub async fn create_table(pool: &SqlitePool) { @@ -98,7 +100,9 @@ pub async fn rebuild_range( since_ts: i64, until_ts: i64, ) -> Result { - use crate::db::cauldron::candlestick::{GuardParams, Leg, Policy}; + use crate::db::cauldron::candlestick::{ + fetch_reserve_snapshot, to_hash32, GuardParams, Leg, Policy, + }; use std::collections::HashMap; if since_ts >= until_ts { @@ -136,15 +140,12 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; } // Phase 2: fold through legs per-token using the policy core to compute buckets. - let params = GuardParams { - max_deviation_factor: 5.0, - min_share_fraction: 0.05, - }; + let params = GuardParams::default(); let mut buckets: HashMap<(Vec, i64), OhlcvBucket> = HashMap::new(); let mut current_token: Option> = None; - let mut policy = Policy::new(params.clone()); + let mut policy = Policy::new(params); for row in &rows { let token_id: Vec = row.get(0); @@ -153,21 +154,19 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; let ts: i64 = row.get(8); let bucket_ts = (ts / 3600) * 3600; - // Reset policy when we move to a new token. + // A new token starts a new policy — seeded from that token's reserves as they + // stood when this batch opened. Callers materialise history in 24-hour batches, + // so an unseeded policy would forget everything at every batch boundary and wave + // through the first leg of every day of history unjudged. if current_token.as_ref() != Some(&token_id) { current_token = Some(token_id.clone()); - policy = Policy::new(params.clone()); + let snapshot = fetch_reserve_snapshot(read_pool, &token_id, since_ts).await?; + policy = Policy::seeded(params, &snapshot); } - // Build the Leg struct. - let mut txid_arr = [0u8; 32]; - let mut pool_arr = [0u8; 32]; - txid_arr.copy_from_slice(&txid); - pool_arr.copy_from_slice(&pool_bytes); - let leg = Leg { - txid: txid_arr, - pool: pool_arr, + txid: to_hash32(&txid)?, + pool: to_hash32(&pool_bytes)?, ts, sequence: row.get(7), sats_delta: row.get(3), @@ -180,7 +179,7 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; let judge = policy.judge(&leg); let sats_delta_abs = leg.sats_delta.unsigned_abs() as i64; - let token_delta_abs = leg.token_delta.abs() as i64; + let token_delta_abs = leg.token_delta.unsigned_abs() as i64; // Get or create the bucket for this (token, bucket_ts). let bucket_key = (token_id.clone(), bucket_ts); @@ -189,10 +188,10 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; // Always accumulate volume. bucket.volume_sats += sats_delta_abs; bucket.volume_tokens += token_delta_abs; - bucket.txids.insert(txid_arr); + bucket.txids.insert(leg.txid); // Update OHLC only if accepted and priceable. - if judge.accepted { + if judge.accepted() { if let Some(price) = judge.price { if bucket.first_accepted_price.is_none() { bucket.first_accepted_price = Some(price); @@ -203,7 +202,7 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; } } - policy.apply(&leg, judge.accepted); + policy.apply(&leg, judge.accepted()); } // Phase 3: insert all computed buckets in one transaction. @@ -211,8 +210,15 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; let mut inserted = 0u64; for ((token_id, bucket_ts), bucket) in buckets { - let open = bucket.first_accepted_price.unwrap_or(bucket.last_accepted_price.unwrap_or(0.0)); - let close = bucket.last_accepted_price.unwrap_or(bucket.first_accepted_price.unwrap_or(0.0)); + // A bucket whose legs were all muted or unpriceable has no OHLC to report: its + // `high`/`low` are still the f64::MIN/MAX sentinels and open/close are unset. + // Writing that row would put ±1.8e308 into the table and hand it straight to the + // chart. Skip it, exactly as the SQL this replaced did by inner-joining prices to + // volume; the raw path's carry-forward renders the gap from the previous close. + let (Some(open), Some(close)) = (bucket.first_accepted_price, bucket.last_accepted_price) + else { + continue; + }; inserted += sqlx::query( "INSERT OR IGNORE INTO ohlcv_1h @@ -505,6 +511,36 @@ mod tests { assert_eq!(n, 0, "mempool trades must not be materialised"); } + /// Regression: a bucket whose legs are all unpriceable used to be written out with its + /// `high`/`low` accumulators still at the `f64::MIN`/`f64::MAX` sentinels, putting + /// ±1.8e308 into the table and from there straight onto the chart. Such a bucket has + /// no OHLC to report and must simply not be materialised. + #[tokio::test] + async fn test_rebuild_range_skips_buckets_with_no_priceable_leg() { + let pool = test_pool().await; + setup_db(&pool).await; + + // A confirmed leg that moves sats but no tokens: real volume, no price. + insert_confirmed_trade( + &pool, [0x01; 32], [0x02; 32], [0x03; 32], 1727963400, -1000, 0, + ) + .await; + + let n = rebuild_range(&pool, &pool, 1727960400, 1727967600) + .await + .unwrap(); + assert_eq!(n, 0, "a bucket with no priceable leg must not be written"); + + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM ohlcv_1h") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + rows, 0, + "no row should exist, let alone one carrying f64 sentinels" + ); + } + /// A multi-pool arbitrage transaction whose legs nearly cancel must materialise the /// price its legs executed at, not the signed-net ratio. #[tokio::test] diff --git a/src/rpc/candlesticks/tests.rs b/src/rpc/candlesticks/tests.rs index d7fbc1f..c06e8e3 100644 --- a/src/rpc/candlesticks/tests.rs +++ b/src/rpc/candlesticks/tests.rs @@ -66,6 +66,7 @@ async fn setup_mock_db(pool: sqlx::SqlitePool) { utxo_funding::create_table(&pool).await; tx::create_table(&pool).await; pool::create_table(&pool).await; + ohlcv::create_table(&pool).await; dummy_init_seq(); let mut conn = pool.acquire().await.unwrap(); @@ -380,6 +381,7 @@ async fn test_single_swap_multiple_pools() { utxo_funding::create_table(&pool).await; tx::create_table(&pool).await; pool::create_table(&pool).await; + ohlcv::create_table(&pool).await; dummy_init_seq(); let mut conn = pool.acquire().await.unwrap(); @@ -481,6 +483,7 @@ async fn test_candle_carries_price_when_net_zero_tokens_but_has_volume() { utxo_funding::create_table(&pool).await; tx::create_table(&pool).await; pool::create_table(&pool).await; + ohlcv::create_table(&pool).await; dummy_init_seq(); let mut conn = pool.acquire().await.unwrap(); From 8e45d5f2160f584c7d34bb7f99455eec2b0a7e94 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Wed, 29 Jul 2026 17:38:48 +0200 Subject: [PATCH 08/11] Exclude withdrawn pools from the reserve snapshot The snapshot introduced in e733bad enumerated every pool the token ever had, so a pool drained months ago still voted on the reference. A withdrawal writes no new pool_history_entry row, so the drained pool's last entry still shows full pre-withdrawal reserves -- it looks like deep liquidity that no longer exists. This was a regression from the snapshot change: the previous per-window fold only learned pools that actually traded in the window, so long-dead pools never entered the map. It is also the OLA failure the Stage 2 design called out, where a 10.9B-sat pool was withdrawn 830s before a crash and a lingering ghost would have muted it. Reuses the filter poolvisitor already applies: a pool is visible if it was never withdrawn, or if its withdrawal transaction is at or after the query instant, so historical queries still see pools that were live at the time they ask about. Tests: 239 passing Co-Authored-By: Claude Opus 5 (1M context) --- src/db/cauldron/candlestick/mod.rs | 15 +++++- src/db/cauldron/candlestick/tests.rs | 79 ++++++++++++++++++++++++++++ src/db/cauldron/ohlcv.rs | 3 +- src/rpc/candlesticks/tests.rs | 5 ++ 4 files changed, 100 insertions(+), 2 deletions(-) diff --git a/src/db/cauldron/candlestick/mod.rs b/src/db/cauldron/candlestick/mod.rs index b9ea6bb..b21dbcb 100644 --- a/src/db/cauldron/candlestick/mod.rs +++ b/src/db/cauldron/candlestick/mod.rs @@ -289,6 +289,12 @@ ORDER BY phe.effective_timestamp ASC, phe.sequence ASC; /// Enumerates the token's pools and seeks each one's latest history row, rather than /// scanning every row the token ever produced — a busy token has hundreds of thousands, /// and this runs once per query. +/// +/// Pools already withdrawn at `timestamp` are excluded. A withdrawal leaves no history +/// row behind, so a drained pool's last entry still shows its full pre-withdrawal +/// reserves; counting those would let a pool that no longer holds anything go on voting +/// the reference. Pools withdrawn *after* `timestamp` are kept, since they were live then +/// — the same rule `poolvisitor` applies. pub(crate) async fn fetch_reserve_snapshot( pool: &SqlitePool, token_blob: &[u8], @@ -305,11 +311,18 @@ JOIN pool_history_entry AS phe ON phe.utxo = ( ORDER BY prior.effective_timestamp DESC, prior.sequence DESC LIMIT 1 ) -WHERE p.token_id = ?; +WHERE p.token_id = ? + AND (p.withdrawn_in_utxo IS NULL OR ( + SELECT t.effective_timestamp + FROM utxo_spending AS us + JOIN tx AS t ON us.txid = t.txid + WHERE us.spent_utxo_hash = p.withdrawn_in_utxo + ) >= ?); "#; let rows = sqlx::query(sql) .bind(timestamp) .bind(token_blob) + .bind(timestamp) .fetch_all(pool) .await?; diff --git a/src/db/cauldron/candlestick/tests.rs b/src/db/cauldron/candlestick/tests.rs index 9483d86..340eb8b 100644 --- a/src/db/cauldron/candlestick/tests.rs +++ b/src/db/cauldron/candlestick/tests.rs @@ -9,6 +9,7 @@ use crate::db::cauldron::{ pool::{self, dummy_init_seq}, tx::{self, insert_block_tx, insert_mempool_tx}, utxo_funding::{self, insert_utxo_funding}, + utxo_spending, }; use crate::utiltest::mock_db_pool; use bitcoin_hashes::Hash; @@ -38,12 +39,46 @@ fn dummy_cauldron( async fn setup_db(pool: sqlx::SqlitePool) { utxo_funding::create_table(&pool).await; + utxo_spending::create_table(&pool).await; tx::create_table(&pool).await; pool::create_table(&pool).await; ohlcv::create_table(&pool).await; dummy_init_seq(); } +/// Register a pool row so the reserve snapshot can find it, optionally marking it +/// withdrawn by the transaction `withdrawn_by` at `withdrawn_at`. +async fn register_pool( + conn: &mut sqlx::pool::PoolConnection, + pool_hash: &OutPointHash, + token: &TokenID, + withdrawal: Option<(Txid, i64)>, +) { + let withdrawn_utxo = withdrawal.map(|(txid, ts)| { + let spent = OutPointHash::from_byte_array(*txid.as_byte_array()); + (spent, txid, ts) + }); + + sqlx::query("INSERT OR REPLACE INTO pool (creation_utxo, owner_pkh, token_id, withdrawn_in_utxo) VALUES (?, ?, ?, ?)") + .bind(pool_hash.to_blob()) + .bind(PubkeyHash::all_zeros().to_blob()) + .bind(token.to_blob()) + .bind(withdrawn_utxo.as_ref().map(|(spent, _, _)| spent.to_blob())) + .execute(&mut **conn) + .await + .unwrap(); + + if let Some((spent, txid, ts)) = withdrawn_utxo { + insert_mempool_tx(&mut **conn, &txid, ts as u64).await.unwrap(); + sqlx::query("INSERT OR REPLACE INTO utxo_spending (spent_utxo_hash, txid) VALUES (?, ?)") + .bind(spent.to_blob()) + .bind(txid.to_blob()) + .execute(&mut **conn) + .await + .unwrap(); + } +} + async fn insert_trade_at( conn: &mut sqlx::pool::PoolConnection, token: &TokenID, @@ -218,6 +253,50 @@ async fn test_single_direction_multileg_closes_on_its_last_leg() { ); } +/// A pool that has been drained leaves its last history row showing full pre-withdrawal +/// reserves, because a withdrawal writes no new row. Seeding the policy from those would +/// let a pool holding nothing keep voting the reference — the OLA case, where a 10.9B-sat +/// pool was withdrawn shortly before a crash and a lingering ghost would have muted it. +#[tokio::test] +async fn test_reserve_snapshot_drops_pools_withdrawn_before_the_instant() { + let db = mock_db_pool(setup_db).await; + let token = TokenID::from_byte_array([0xC6; 32]); + let token_blob = token.to_blob(); + + let mut conn = db.cauldron_w.acquire().await.unwrap(); + insert_trade_at(&mut conn, &token, 0x41, 1000, 100_000, 2_000).await; + + let pool_hash = OutPointHash::from_byte_array([0x41u8.wrapping_add(0x80); 32]); + register_pool(&mut conn, &pool_hash, &token, None).await; + + let live = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 5000) + .await + .unwrap(); + assert_eq!(live.len(), 1, "a live pool must be in the snapshot"); + + // Withdraw it at ts=2000 and ask again afterwards. + let withdrawal_tx = Txid::from_byte_array([0x42; 32]); + register_pool(&mut conn, &pool_hash, &token, Some((withdrawal_tx, 2000))).await; + + let after = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 5000) + .await + .unwrap(); + assert!( + after.is_empty(), + "a pool withdrawn at 2000 must not vote on the reference at 5000: {after:?}" + ); + + // But it was live at ts=1500, so a historical query must still see it. + let before = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 1500) + .await + .unwrap(); + assert_eq!( + before.len(), + 1, + "a pool withdrawn later was still live earlier and must remain visible" + ); +} + /// Regression: the seed lookback was briefly capped at 24 hours, which silently dropped /// the carry-forward price for any token trading less often than daily and left a hole /// where the leading candles should be. diff --git a/src/db/cauldron/ohlcv.rs b/src/db/cauldron/ohlcv.rs index 029fae1..e45060a 100644 --- a/src/db/cauldron/ohlcv.rs +++ b/src/db/cauldron/ohlcv.rs @@ -317,7 +317,7 @@ pub async fn get_active_candles( #[cfg(test)] mod tests { use super::*; - use crate::db::cauldron::{pool as cauldron_pool, tx, utxo_funding}; + use crate::db::cauldron::{pool as cauldron_pool, tx, utxo_funding, utxo_spending}; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -335,6 +335,7 @@ mod tests { async fn setup_db(pool: &SqlitePool) { tx::create_table(pool).await; utxo_funding::create_table(pool).await; + utxo_spending::create_table(pool).await; cauldron_pool::create_table(pool).await; create_table(pool).await; // ohlcv_1h + idx_phe_txid } diff --git a/src/rpc/candlesticks/tests.rs b/src/rpc/candlesticks/tests.rs index c06e8e3..77f0973 100644 --- a/src/rpc/candlesticks/tests.rs +++ b/src/rpc/candlesticks/tests.rs @@ -8,6 +8,7 @@ use crate::db::cauldron::{ pool::{self, dummy_init_seq, insert_new_pool}, tx::{self, insert_block_tx, insert_mempool_tx}, utxo_funding::{self, insert_utxo_funding}, + utxo_spending, }; use crate::utiltest::mock_db_pool; use crate::OhlcvState; @@ -64,6 +65,7 @@ fn dummy_cauldron( async fn setup_mock_db(pool: sqlx::SqlitePool) { utxo_funding::create_table(&pool).await; + utxo_spending::create_table(&pool).await; tx::create_table(&pool).await; pool::create_table(&pool).await; ohlcv::create_table(&pool).await; @@ -201,6 +203,7 @@ async fn setup_mock_db(pool: sqlx::SqlitePool) { async fn setup_seed_db(pool: sqlx::SqlitePool) { utxo_funding::create_table(&pool).await; + utxo_spending::create_table(&pool).await; tx::create_table(&pool).await; pool::create_table(&pool).await; ohlcv::create_table(&pool).await; @@ -379,6 +382,7 @@ async fn test_multiple_candlesticks_endpoint() { async fn test_single_swap_multiple_pools() { let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move { utxo_funding::create_table(&pool).await; + utxo_spending::create_table(&pool).await; tx::create_table(&pool).await; pool::create_table(&pool).await; ohlcv::create_table(&pool).await; @@ -481,6 +485,7 @@ async fn test_candle_carries_price_when_net_zero_tokens_but_has_volume() { let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move { // --- boilerplate setup --- utxo_funding::create_table(&pool).await; + utxo_spending::create_table(&pool).await; tx::create_table(&pool).await; pool::create_table(&pool).await; ohlcv::create_table(&pool).await; From d961a5f76223373d0bf8dd6f9b5e096d7f6d8d1b Mon Sep 17 00:00:00 2001 From: jakobsn Date: Wed, 29 Jul 2026 19:49:29 +0200 Subject: [PATCH 09/11] Gate the background ohlcv task on IBD completion On master the synchronous backfill always ran and blocked on initial_sync_complete, so the background task spawned afterwards could not start until IBD had finished. Stage 1 wrapped that backfill in `if !ohlcv_wiped`, which removed the barrier for the background task as a side effect. The guard fires more often than a version bump suggests: a fresh database has no version key, so `stored (None) != Some(OHLCV_VERSION)` and migrate_if_stale reports a wipe. Every clean resync therefore skipped the IBD-gated path entirely. Left ungated, the task sweeps from the first confirmed trade to `now - 3h` while indexing is still years behind. It materialises nothing, re-runs ~700 empty 24-hour batches every 600s, competes with block indexing for the cauldron write lock, and advances materialized_end to roughly now over an empty table -- after which candlesticks() takes the ohlcv fast path against nothing instead of falling back to the raw path. Waiting inside the spawned task rather than before the spawn keeps startup non-blocking. Tests: 239 passing Co-Authored-By: Claude Opus 5 (1M context) --- src/main.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index a464023..5a9614d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -515,18 +515,33 @@ async fn launch() -> _ { } // Background task: incrementally materialise new 1-hour OHLCV buckets as blocks arrive. - // The full historical backfill above already ran; this task only handles the tail. + // After a version wipe this is also what repopulates history, since the synchronous + // backfill above is skipped in that case. // Only processes buckets older than 3 hours (well beyond BCH reorg depth). { let ohlcv_write = dbpool.cauldron_w.clone(); let ohlcv_read = dbpool.cauldron_r.clone(); let ohlcv_state_bg = ohlcv_state.clone(); + let ohlcv_ibd = ibd_state.clone(); tokio::spawn(async move { // Batch size: 1 day per SQL call to keep each write short. const BATCH_SECS: i64 = 24 * 3600; // Safety margin: only materialise buckets older than this many seconds. const SAFETY_SECS: i64 = 3 * 3600; + // Wait for IBD before materialising anything. + // + // The synchronous backfill above waits too, but it is skipped whenever + // `ohlcv_wiped` is set — which includes every fresh database, since a missing + // version key reads as stale. Without this the task would sweep from the + // first trade all the way to `now - 3h` while indexing is still years behind, + // writing nothing, contending with block writes for the cauldron write lock, + // and advancing `materialized_end` to roughly now against an empty table — at + // which point `candlesticks()` would take the fast path over nothing. + while !ohlcv_ibd.initial_sync_complete.load(Ordering::Relaxed) { + tokio::time::sleep(Duration::from_secs(1)).await; + } + loop { let now = crate::timeutil::time_now(); // Floor to 1-hour boundary, 3 hours ago. From 0eef3ecd7c6b0022339176b79c74e0bacbf34e36 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Wed, 29 Jul 2026 20:39:14 +0200 Subject: [PATCH 10/11] Fix five defects found reviewing the previous two commits A multi-role review of e733bad and 8e45d5f found five real defects, three of them reproduced independently by more than one reviewer. All five were introduced by those commits. Reweighting stopped short of its fixed point weighted_median_reference ran a hard-coded four passes. Raising the reference only ever raises weights, so the chosen index climbs one position per pass and a set spread over many decades never arrives: a ten-pool counterexample settled on 411_155 against a true fixed point of 501_964, and random search found spreads off by 81x. A wrong reference mis-sites the whole F=5 band. The loop now runs until it converges, bounded by the pool count, which is sufficient because the pick never revisits a position. A stale materialised bucket shadowed newer legs fetch_last_close_before consulted ohlcv_1h first and returned on any hit. That table is structurally at least three hours behind the tip and is empty for the whole of a post-version-bump rebuild, so the seed was routinely far older than the real last print -- reproduced returning 10.0 where the last print was 50.0. Fixing "unbounded lookback" in e733bad had removed the upper bound along with the lower one. The lookup is now anchored to the hour containing the token's most recent activity. ...and could return a price from after the instant asked about The same query took any bucket with bucket_ts < timestamp_end, so a non-hour-aligned cutoff could land on a bucket straddling it and return a close set later than the requested time. The predecessor was hour-aligned precisely to prevent this. Skipping unpriceable buckets discarded their volume Dropping the row avoided the f64 sentinels but took volume_sats, volume_tokens and tx_count with it, so the materialised path reported zero volume for hours the raw path reported in full. The guard makes those hours ordinary rather than exotic -- one manipulation burst can fill a whole bucket, and muting every leg in it then erased the hour. Such buckets are now written flat at the carried close. This also stops get_max_bucket_ts regressing, which had let the background sweep re-seed already-scanned ranges under a different snapshot. The rebuild seeded from unconfirmed reserves rebuild_range folds only legs with tx.blockhash IS NOT NULL, but the snapshot had no such filter and effective_timestamp is populated for mempool rows. A broadcast-but-unmined trade could therefore set the reference for a whole batch, making materialised output a function of mempool contents at rebuild time and freezing it via INSERT OR IGNORE. fetch_reserve_snapshot now takes confirmed_only, set by the rebuild and clear on the live query paths. Tests. Six mutation tests confirm each new regression test fails against the specific defect it names. The review also found four of e733bad's tests vacuous under mutation -- passing with the dust floor deleted, with min-depth weighting disabled, and under the exact integer-division rounding they were named for. Those are rewritten to be falsifiable, and the tier-2 self-qualification gap now has a test documenting it. Corrections to earlier commit messages in this branch. e733bad claimed every fix carried a test "confirmed to fail against the old code"; only two of them were drop-in regression tests, the rest exercise APIs that did not exist before it. e733bad's "~1125 resets per token" should read once per batch in which the token traded, and contradicts d961a5f's "~700" for the same span. 8e45d5f overstated its reach: excluding pools withdrawn before the snapshot instant does not address the OLA case, where the withdrawal lands mid-window -- that still needs the unbuilt withdrawal-event synthesis. Known gap, unchanged and now under test: tier 2 compares a pool's credit against a share of the largest credit, so the deepest pool clears a share of itself and any single-pool token is unguarded. This predates these commits and awaits a decision on the rule. OHLCV_VERSION 4 -> 5, and the version-4 changelog entry corrected: it named two of the six behaviour changes that version actually carried. Tests: 244 passing Co-Authored-By: Claude Opus 5 (1M context) --- src/db/cauldron/candlestick/mod.rs | 95 ++++++++++++----- src/db/cauldron/candlestick/policy.rs | 23 ++-- src/db/cauldron/candlestick/policy/tests.rs | 111 +++++++++++++++++--- src/db/cauldron/candlestick/tests.rs | 75 ++++++++++++- src/db/cauldron/ohlcv.rs | 103 +++++++++++++++--- 5 files changed, 341 insertions(+), 66 deletions(-) diff --git a/src/db/cauldron/candlestick/mod.rs b/src/db/cauldron/candlestick/mod.rs index b21dbcb..d82f5bb 100644 --- a/src/db/cauldron/candlestick/mod.rs +++ b/src/db/cauldron/candlestick/mod.rs @@ -295,17 +295,34 @@ ORDER BY phe.effective_timestamp ASC, phe.sequence ASC; /// reserves; counting those would let a pool that no longer holds anything go on voting /// the reference. Pools withdrawn *after* `timestamp` are kept, since they were live then /// — the same rule `poolvisitor` applies. +/// +/// `confirmed_only` must match whatever the caller's leg query does. `rebuild_range` folds +/// only confirmed legs, so seeding it from reserves that include mempool rows would make +/// materialised output a function of mempool contents at rebuild time — and `INSERT OR +/// IGNORE` would then freeze whichever run happened first. The live query paths do not +/// filter their legs, so they pass `false`. pub(crate) async fn fetch_reserve_snapshot( pool: &SqlitePool, token_blob: &[u8], timestamp: i64, + confirmed_only: bool, ) -> Result> { - let sql = r#" + // Interpolated, not bound: a constant fragment keeps the query planner able to use + // idx_pool_history_entry_pool_timestamp instead of evaluating a flag per row. + let confirmed_join = if confirmed_only { + "JOIN tx AS prior_tx ON prior_tx.txid = prior.txid AND prior_tx.blockhash IS NOT NULL" + } else { + "" + }; + + let sql = format!( + r#" SELECT phe.pool, phe.sequence, phe.sats, phe.token_amount FROM pool AS p JOIN pool_history_entry AS phe ON phe.utxo = ( SELECT prior.utxo FROM pool_history_entry AS prior + {confirmed_join} WHERE prior.pool = p.creation_utxo AND prior.effective_timestamp < ? ORDER BY prior.effective_timestamp DESC, prior.sequence DESC @@ -318,8 +335,9 @@ WHERE p.token_id = ? JOIN tx AS t ON us.txid = t.txid WHERE us.spent_utxo_hash = p.withdrawn_in_utxo ) >= ?); -"#; - let rows = sqlx::query(sql) +"# + ); + let rows = sqlx::query(&sql) .bind(timestamp) .bind(token_blob) .bind(timestamp) @@ -356,24 +374,11 @@ async fn fetch_last_close_before( token_blob: &[u8], timestamp_end: i64, ) -> Result> { - // Materialised buckets answer this with one index seek on (token_id, bucket_ts). - let materialised: Option = sqlx::query_scalar( - "SELECT close FROM ohlcv_1h - WHERE token_id = ? AND bucket_ts < ? - ORDER BY bucket_ts DESC LIMIT 1", - ) - .bind(token_blob) - .bind(timestamp_end) - .fetch_optional(pool) - .await?; - - if materialised.is_some() { - return Ok(materialised); - } - - // Nothing materialised: replay the token's most recent active hour through the guard. - // Only that hour is needed, because the policy is seeded from reserves rather than - // rebuilt by folding all of history. + // Locate the token's most recent activity first. Trusting `ohlcv_1h` before knowing + // this would let a stale bucket shadow newer legs: the table is structurally at least + // three hours behind the tip, and is empty for the whole of a post-version-bump + // rebuild, so "newest materialised bucket" is routinely far older than the real last + // print. let last_activity: Option = sqlx::query_scalar( "SELECT MAX(effective_timestamp) FROM pool_history_entry WHERE token_id = ? AND effective_timestamp < ?", @@ -389,7 +394,30 @@ async fn fetch_last_close_before( }; let scan_start = (last_activity / 3600) * 3600; - let snapshot = fetch_reserve_snapshot(pool, token_blob, scan_start).await?; + + // The last accepted print lives in the hour containing that activity. If that exact + // hour is materialised and closes at or before the cutoff, its close is the answer. + // + // The `scan_start + 3600 <= timestamp_end` test matters: a bucket straddling the cutoff + // closes on a print that happened *after* the instant being asked about, which an + // earlier revision leaked for any non-hour-aligned `timestamp_end`. + if scan_start + 3600 <= timestamp_end { + let exact: Option = sqlx::query_scalar( + "SELECT close FROM ohlcv_1h WHERE token_id = ? AND bucket_ts = ?", + ) + .bind(token_blob) + .bind(scan_start) + .fetch_optional(pool) + .await?; + + if exact.is_some() { + return Ok(exact); + } + } + + // Replay that hour through the guard. Only one hour is needed, because the policy is + // seeded from reserves rather than rebuilt by folding all of history. + let snapshot = fetch_reserve_snapshot(pool, token_blob, scan_start, false).await?; let mut policy = Policy::seeded(GuardParams::default(), &snapshot); let legs = fetch_raw_legs(pool, token_blob, scan_start, timestamp_end).await?; @@ -401,8 +429,25 @@ async fn fetch_last_close_before( } policy.apply(leg, judge.accepted()); } + if last_accepted_price.is_some() { + return Ok(last_accepted_price); + } - Ok(last_accepted_price.or_else(|| policy.reference())) + // That hour was entirely muted or unpriceable. Reach further back through the + // materialised table — unbounded, but only now that it cannot shadow newer legs. + let earlier: Option = sqlx::query_scalar( + "SELECT close FROM ohlcv_1h + WHERE token_id = ? AND bucket_ts + 3600 <= ? + ORDER BY bucket_ts DESC LIMIT 1", + ) + .bind(token_blob) + .bind(timestamp_end) + .fetch_optional(pool) + .await?; + + // Last resort: the price implied by current reserves. Not a executed print, but a + // carried price beats a hole in the chart. + Ok(earlier.or_else(|| policy.reference())) } /// `ohlcv_materialized_end`: exclusive upper bound of what is in `ohlcv_1h`. @@ -455,7 +500,7 @@ pub async fn candlesticks( // before it came from `ohlcv_1h` and never passed through this policy, so a // policy built from the window's start would enter the tail believing the // token had no history and wave its first leg through unjudged. - let snapshot = fetch_reserve_snapshot(pool, &token_blob, ohlcv_end).await?; + let snapshot = fetch_reserve_snapshot(pool, &token_blob, ohlcv_end, false).await?; let mut policy = Policy::seeded(params, &snapshot); let raw_legs = fetch_raw_legs(pool, &token_blob, ohlcv_end, timestamp_end).await?; @@ -488,7 +533,7 @@ pub async fn candlesticks( current_start += step_size; } - let snapshot = fetch_reserve_snapshot(pool, &token_blob, timestamp_start).await?; + let snapshot = fetch_reserve_snapshot(pool, &token_blob, timestamp_start, false).await?; let mut policy = Policy::seeded(params, &snapshot); let all_legs = fetch_raw_legs(pool, &token_blob, timestamp_start, timestamp_end).await?; diff --git a/src/db/cauldron/candlestick/policy.rs b/src/db/cauldron/candlestick/policy.rs index 41dc86f..94ca044 100644 --- a/src/db/cauldron/candlestick/policy.rs +++ b/src/db/cauldron/candlestick/policy.rs @@ -9,10 +9,6 @@ use std::collections::HashMap; /// Pools holding fewer tokens than this are dust and do not vote on the reference. const MIN_TOKEN_RESERVE: u64 = 10; -/// Upper bound on reweighting passes in [`weighted_median_reference`]. Four is far more -/// than the one or two the fixed point needs in practice; it exists to bound the loop. -const REWEIGHT_PASSES: usize = 4; - /// A leg of a transaction: one pool's change in a single transaction. #[derive(Clone, Debug)] pub struct Leg { @@ -251,7 +247,9 @@ impl Policy { credit: 0, }); - if leg.sequence >= state.sequence { + // Strict `>`: the `i64::MIN` sentinel above is what lets a pool's first leg through, + // so there is no reason to also relax the monotonicity rule for later ones. + if leg.sequence > state.sequence { state.sequence = leg.sequence; state.sats = leg.sats; state.token_amount = leg.token_amount; @@ -336,7 +334,15 @@ fn weighted_median_reference(entries: &mut [(f64, u64, u64)]) -> Option { entries.sort_unstable_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(Ordering::Equal)); let mut reference = median_by(entries, |_| 1)?; - for _ in 0..REWEIGHT_PASSES { + + // Raising `reference` can only raise each `min(S, T * reference)` weight, and only for + // pools not yet capped by their sats side — which are exactly the high-spot tail. So + // the chosen index moves monotonically upwards and never revisits a position, giving a + // fixed point within `entries.len()` passes. A fixed pass count would instead return a + // truncated iterate: on a set of ten pools spread over many decades, stopping at four + // passes yields 411_155 where the fixed point is 501_964, and wider spreads diverge by + // more than an order of magnitude. + for _ in 0..entries.len() { let Some(next) = median_by(entries, |e| min_depth(e.1, e.2, reference)) else { break; }; @@ -360,7 +366,10 @@ fn median_by( ) -> Option { let total: u128 = sorted.iter().map(|e| weight(e) as u128).sum(); if total == 0 { - return sorted.get(sorted.len() / 2).map(|e| e.0); + // Defensive only: `spot_ratio` rejects `sats == 0`, so every live weight is >= 1. + // Index chosen to match what the loop below returns for equal weights, rather than + // `len / 2`, which is the upper median for an even count. + return sorted.get(sorted.len().saturating_sub(1) / 2).map(|e| e.0); } let mut cumulative: u128 = 0; diff --git a/src/db/cauldron/candlestick/policy/tests.rs b/src/db/cauldron/candlestick/policy/tests.rs index e60d5cd..bd8b98f 100644 --- a/src/db/cauldron/candlestick/policy/tests.rs +++ b/src/db/cauldron/candlestick/policy/tests.rs @@ -200,32 +200,49 @@ fn test_seeding_withholds_credit_from_off_market_pools() { ); } -/// Min-depth weighting is what stops a pool holding worthless tokens from voting the -/// reference away from where the real liquidity sits. +/// Min-depth weighting is what stops shallow pools from voting the reference away from +/// where the real liquidity sits. +/// +/// The dust pools deliberately OUTNUMBER the real one, so an unweighted median lands on +/// them (10_000) and only the weighting pulls the answer back to 100. A test where all +/// pools quote the same price would pass no matter what the weighting did. #[test] -fn test_dust_pool_does_not_move_the_reference() { +fn test_dust_pools_do_not_outvote_one_deep_pool() { let policy = Policy::seeded( params(), &[ reserves(1, 1, 100_000_000, 1_000_000), // spot 100, deep - reserves(2, 2, 200_000_000, 2_000_000), // spot 100, deep - reserves(9, 3, 1_000, 10), // spot 100, but ~nothing behind it + reserves(8, 2, 100_000, 10), // spot 10_000, shallow + reserves(9, 3, 100_000, 10), // spot 10_000, shallow ], ); - assert_eq!(policy.reference(), Some(100.0)); + assert_eq!( + policy.reference(), + Some(100.0), + "two shallow pools must not outvote one holding a thousand times the depth" + ); } +/// The dust floor is a guard against degenerate arithmetic, not a weighting mechanism — +/// min-depth already suppresses thin pools. What it must do is keep a pool that cannot +/// quote a ratio from producing an infinity. #[test] -fn test_pools_below_the_dust_floor_are_ignored_entirely() { +fn test_pools_that_cannot_quote_a_ratio_are_excluded() { let policy = Policy::seeded( params(), &[ reserves(1, 1, 1_000_000, 10_000), // spot 100 - reserves(9, 2, 1_000_000, 1), // spot 1_000_000, below MIN_TOKEN_RESERVE + reserves(8, 2, 1_000_000, 0), // no tokens: ratio would divide by zero + reserves(9, 3, 0, 10_000), // no sats: ratio would be 0.0 ], ); - assert_eq!(policy.reference(), Some(100.0)); + + let reference = policy.reference().expect("the one quotable pool sets the reference"); + assert!( + reference.is_finite() && reference == 100.0, + "got {reference}" + ); } #[test] @@ -234,15 +251,19 @@ fn test_policy_with_no_usable_pools_has_no_reference() { assert_eq!(policy.reference(), None); } -/// `cumulative >= total / 2` truncates, which tips two-pool sets towards the lower ratio. -/// The comparison is written as `2 * cumulative >= total` to avoid that. +/// `cumulative >= total / 2` truncates, which tips the pick towards the lower ratio. +/// +/// Weights 2 and 3 are the minimal case that separates the two forms: the old test is +/// `2 >= 5/2 == 2`, true, so it stops on the first entry; the correct `2*2 >= 5` is false, +/// so it advances to the second. Driving `median_by` directly is what makes this +/// falsifiable — going through `weighted_median_reference` cannot pin the weights. #[test] fn test_weighted_median_is_not_biased_by_integer_division() { - let mut entries = vec![(100.0, 3, 1), (200.0, 3, 1)]; - let picked = weighted_median_reference(&mut entries).unwrap(); - assert!( - picked == 100.0 || picked == 200.0, - "expected one of the two inputs, got {picked}" + let sorted = [(100.0, 2u64, 0u64), (200.0, 3u64, 0u64)]; + assert_eq!( + median_by(&sorted, |e| e.1), + Some(200.0), + "weight 3 holds the majority, so the median is 200, not 100" ); // An odd, clearly-weighted set must land on the true weighted median. @@ -250,6 +271,40 @@ fn test_weighted_median_is_not_biased_by_integer_division() { assert_eq!(weighted_median_reference(&mut skewed), Some(100.0)); } +/// Regression: the reweighting loop ran a fixed four passes and returned whatever iterate +/// it had reached. Because raising the reference only ever raises weights, the pick climbs +/// one position per pass, so a set spread over many decades stops short — this one settled +/// on 411_155 against a true fixed point of 501_964, and wider spreads diverged by more +/// than an order of magnitude. +#[test] +fn test_reference_reaches_its_fixed_point_not_a_truncated_iterate() { + let pools: [(u64, u64); 10] = [ + (962, 249_939_293_128_076), + (3_198, 71_909_139_611_364), + (455_109_248_120, 243_581_757_429_499), + (279, 9_934), + (432_371, 2_608), + (208_541_212_143, 728_829_157), + (925_517_097_739, 737_748_546), + (34_778_861_631_604, 885_976_135), + (89_743_615_643_767, 218_272_037), + (499_655_065_278_927, 995_401_018), + ]; + let mut entries: Vec<(f64, u64, u64)> = pools + .iter() + .map(|(sats, tokens)| (*sats as f64 / *tokens as f64, *sats, *tokens)) + .collect(); + + let reference = weighted_median_reference(&mut entries).unwrap(); + + // `entries` is left sorted, so one more pass is a direct fixed-point check. + let next = median_by(&entries, |e| min_depth(e.1, e.2, reference)).unwrap(); + assert_eq!( + next, reference, + "reweighting must converge; {reference} moves to {next} on the next pass" + ); +} + /// All-zero weights used to make the cumulative test pass on the first entry, returning /// the smallest ratio rather than the middle one. #[test] @@ -258,6 +313,30 @@ fn test_all_zero_weights_fall_back_to_the_positional_median() { assert_eq!(weighted_median_reference(&mut entries), Some(100.0)); } +/// Documents a KNOWN GAP in tier-2 rather than endorsing it: the threshold is a share of +/// `largest_credit`, and the pool holding that largest credit trivially clears a share of +/// itself. So the deepest pool — and every pool of a single-pool token — can print any +/// price it likes. Left as-is pending a decision on the tier-2 rule; this test exists so +/// the behaviour is visible and any change to it is deliberate. +#[test] +fn test_known_gap_deepest_pool_self_qualifies_for_tier_two() { + let mut policy = Policy::seeded(params(), &[reserves(1, 1, 10_000_000, 100_000)]); + assert_eq!(policy.reference(), Some(100.0)); + assert!(policy.credit_of(&[1; 32]) > 0); + + // The same pool now prints ten million times off its own spot. + let absurd = leg(1, 2, 1_000_000_000, -1, 11_000_000_000, 99_999); + let judged = policy.judge(&absurd); + + assert!( + matches!(judged.verdict, Verdict::CreditExempt { .. }), + "expected the known self-qualification gap, got {:?}", + judged.verdict + ); + assert!(judged.accepted()); + let _ = &mut policy; +} + #[test] fn test_min_depth_values_the_token_side_at_the_reference() { // Token side is the thinner one: 10 tokens * 100 = 1_000 < 50_000 sats. diff --git a/src/db/cauldron/candlestick/tests.rs b/src/db/cauldron/candlestick/tests.rs index 340eb8b..e325cc4 100644 --- a/src/db/cauldron/candlestick/tests.rs +++ b/src/db/cauldron/candlestick/tests.rs @@ -269,7 +269,7 @@ async fn test_reserve_snapshot_drops_pools_withdrawn_before_the_instant() { let pool_hash = OutPointHash::from_byte_array([0x41u8.wrapping_add(0x80); 32]); register_pool(&mut conn, &pool_hash, &token, None).await; - let live = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 5000) + let live = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 5000, false) .await .unwrap(); assert_eq!(live.len(), 1, "a live pool must be in the snapshot"); @@ -278,7 +278,7 @@ async fn test_reserve_snapshot_drops_pools_withdrawn_before_the_instant() { let withdrawal_tx = Txid::from_byte_array([0x42; 32]); register_pool(&mut conn, &pool_hash, &token, Some((withdrawal_tx, 2000))).await; - let after = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 5000) + let after = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 5000, false) .await .unwrap(); assert!( @@ -287,7 +287,7 @@ async fn test_reserve_snapshot_drops_pools_withdrawn_before_the_instant() { ); // But it was live at ts=1500, so a historical query must still see it. - let before = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 1500) + let before = super::fetch_reserve_snapshot(&db.cauldron_r, &token_blob, 1500, false) .await .unwrap(); assert_eq!( @@ -297,6 +297,75 @@ async fn test_reserve_snapshot_drops_pools_withdrawn_before_the_instant() { ); } +/// Regression: the seed consulted `ohlcv_1h` first and returned on any hit. Since that +/// table is structurally at least three hours behind the tip — and completely empty for +/// the whole of a post-version-bump rebuild — a stale bucket shadowed every newer leg. +#[tokio::test] +async fn test_seed_prefers_recent_legs_over_a_stale_materialised_bucket() { + let db = mock_db_pool(setup_db).await; + let token = TokenID::from_byte_array([0xC7; 32]); + let token_blob = token.to_blob(); + let mut conn = db.cauldron_w.acquire().await.unwrap(); + + sqlx::query( + "INSERT INTO ohlcv_1h + (token_id, bucket_ts, open, high, low, close, volume_sats, volume_tokens, tx_count) + VALUES (?, 0, 10.0, 10.0, 10.0, 10.0, 1, 1, 1)", + ) + .bind(&token_blob) + .execute(&mut *conn) + .await + .unwrap(); + + // A real print at 50.0 two hours later that the table has not caught up with. + insert_trade_at(&mut conn, &token, 0x51, 7200, 100_000, 2_000).await; + + let price = super::fetch_last_close_before(&db.cauldron_r, &token_blob, 10800) + .await + .unwrap() + .expect("a seed must exist"); + + assert!( + (price - 50.0).abs() < f64::EPSILON, + "stale bucket must not shadow the newer print: got {price}" + ); +} + +/// Regression: the materialised lookup took any bucket with `bucket_ts < timestamp_end`, +/// so for a non-hour-aligned cutoff it could return a bucket straddling that cutoff — +/// closing on a print from *after* the instant being asked about. +#[tokio::test] +async fn test_seed_does_not_leak_a_price_from_after_the_cutoff() { + let db = mock_db_pool(setup_db).await; + let token = TokenID::from_byte_array([0xC8; 32]); + let token_blob = token.to_blob(); + let mut conn = db.cauldron_w.acquire().await.unwrap(); + + // Bucket [7200, 10800) closes at 80.0 — a price set late in that hour. + sqlx::query( + "INSERT INTO ohlcv_1h + (token_id, bucket_ts, open, high, low, close, volume_sats, volume_tokens, tx_count) + VALUES (?, 7200, 50.0, 80.0, 50.0, 80.0, 1, 1, 1)", + ) + .bind(&token_blob) + .execute(&mut *conn) + .await + .unwrap(); + + // What actually happened by ts=8000 was a single print at 50.0. + insert_trade_at(&mut conn, &token, 0x52, 7300, 100_000, 2_000).await; + + let price = super::fetch_last_close_before(&db.cauldron_r, &token_blob, 8000) + .await + .unwrap() + .expect("a seed must exist"); + + assert!( + (price - 50.0).abs() < f64::EPSILON, + "must not return the 80.0 close of a bucket that ends after the cutoff: got {price}" + ); +} + /// Regression: the seed lookback was briefly capped at 24 hours, which silently dropped /// the carry-forward price for any token trading less often than daily and left a hole /// where the leading candles should be. diff --git a/src/db/cauldron/ohlcv.rs b/src/db/cauldron/ohlcv.rs index e45060a..01892fa 100644 --- a/src/db/cauldron/ohlcv.rs +++ b/src/db/cauldron/ohlcv.rs @@ -13,9 +13,16 @@ use sqlx::{Row, SqlitePool}; /// /// 2: price switched from the signed net ratio to the gross volume ratio. /// 3: pricing switched from per-transaction to per-leg with policy-based acceptance. -/// 4: policy reference seeded from reserves rather than folded from the batch start, and -/// buckets with no priceable leg are skipped instead of written with f64 sentinels. -pub const OHLCV_VERSION: u32 = 4; +/// 4: policy reference seeded from reserves rather than folded from the batch start; +/// f64 sentinels no longer written; credit earned on a pool's first accepted print; +/// zero-sats legs unpriceable; weighted-median rounding and zero-weight fixes; +/// pools withdrawn before the snapshot instant excluded from the reference. +/// 5: reweighting iterates to its fixed point instead of stopping at four passes, so the +/// reference moves on multi-decade pool spreads; buckets with no accepted price are +/// written flat at the carried close instead of dropped, restoring their volume; the +/// rebuild seeds from confirmed-only reserves, so output no longer depends on mempool +/// contents at rebuild time. +pub const OHLCV_VERSION: u32 = 5; const OHLCV_VERSION_KEY: &str = "ohlcv_version"; pub async fn create_table(pool: &SqlitePool) { @@ -146,6 +153,9 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; let mut current_token: Option> = None; let mut policy = Policy::new(params); + // Reserve-implied price for each token as the batch opened, used to price a leading + // bucket whose legs were all muted so its volume is still recorded. + let mut entry_price: HashMap, Option> = HashMap::new(); for row in &rows { let token_id: Vec = row.get(0); @@ -160,8 +170,9 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; // through the first leg of every day of history unjudged. if current_token.as_ref() != Some(&token_id) { current_token = Some(token_id.clone()); - let snapshot = fetch_reserve_snapshot(read_pool, &token_id, since_ts).await?; + let snapshot = fetch_reserve_snapshot(read_pool, &token_id, since_ts, true).await?; policy = Policy::seeded(params, &snapshot); + entry_price.insert(token_id.clone(), policy.reference()); } let leg = Leg { @@ -206,18 +217,45 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; } // Phase 3: insert all computed buckets in one transaction. + // + // Ordered by (token, bucket_ts) so a token's buckets can carry their close forward. + // A bucket whose legs were all muted or unpriceable has no price of its own, but it + // does have real volume, and the guard makes such buckets ordinary rather than + // exotic — a manipulation burst filling an hour produces exactly one. Writing it with + // the carried close keeps that volume, where writing the raw accumulators would put + // the f64::MIN/MAX sentinels into the table and skipping the row entirely would make + // the materialised path report zero volume for an hour the raw path reports in full. + let mut ordered: Vec<((Vec, i64), OhlcvBucket)> = buckets.into_iter().collect(); + ordered.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + let mut tx = write_pool.begin().await?; let mut inserted = 0u64; + let mut carry_token: Option> = None; + let mut carry: Option = None; - for ((token_id, bucket_ts), bucket) in buckets { - // A bucket whose legs were all muted or unpriceable has no OHLC to report: its - // `high`/`low` are still the f64::MIN/MAX sentinels and open/close are unset. - // Writing that row would put ±1.8e308 into the table and hand it straight to the - // chart. Skip it, exactly as the SQL this replaced did by inner-joining prices to - // volume; the raw path's carry-forward renders the gap from the previous close. - let (Some(open), Some(close)) = (bucket.first_accepted_price, bucket.last_accepted_price) - else { - continue; + for ((token_id, bucket_ts), bucket) in ordered { + if carry_token.as_ref() != Some(&token_id) { + carry_token = Some(token_id.clone()); + carry = entry_price.get(&token_id).copied().flatten(); + } + + let (open, close, high, low) = match bucket.last_accepted_price { + Some(last) => { + carry = Some(last); + ( + bucket.first_accepted_price.unwrap_or(last), + last, + bucket.high, + bucket.low, + ) + } + // No price anywhere in this bucket. Flat at the carried close if one exists; + // if the token has no known price yet at all, there is nothing to plot and the + // row is skipped, as the pre-policy SQL did by inner-joining prices to volume. + None => match carry { + Some(prev) => (prev, prev, prev, prev), + None => continue, + }, }; inserted += sqlx::query( @@ -228,8 +266,8 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; .bind(&token_id) .bind(bucket_ts) .bind(open) - .bind(bucket.high) - .bind(bucket.low) + .bind(high) + .bind(low) .bind(close) .bind(bucket.volume_sats) .bind(bucket.volume_tokens) @@ -542,6 +580,41 @@ mod tests { ); } + /// Once a token has a known price, an hour whose legs are all unpriceable or muted + /// still carries real volume. Dropping the row made the materialised path report zero + /// volume for an hour the raw path reports in full — and the guard makes such hours + /// ordinary, since one manipulation burst can fill an entire bucket. + #[tokio::test] + async fn test_rebuild_range_keeps_volume_for_a_bucket_with_no_accepted_price() { + let pool = test_pool().await; + setup_db(&pool).await; + let token = [0x03_u8; 32]; + + // Hour 1727960400: an ordinary priced trade at 1000/25 = 40. + insert_confirmed_trade(&pool, [0x01; 32], [0x02; 32], token, 1727960500, -1000, 25).await; + // Hour 1727964000: tokens move but no sats, so the leg has no price. + insert_confirmed_trade(&pool, [0x11; 32], [0x12; 32], token, 1727964100, 0, 25).await; + + let n = rebuild_range(&pool, &pool, 1727960400, 1727967600) + .await + .unwrap(); + assert_eq!(n, 2, "both hours must be materialised"); + + let (volume, open, close, high, low): (i64, f64, f64, f64, f64) = sqlx::query_as( + "SELECT volume_tokens, open, close, high, low FROM ohlcv_1h WHERE bucket_ts = 1727964000", + ) + .fetch_one(&pool) + .await + .unwrap(); + + assert_eq!(volume, 25, "volume must survive a bucket with no accepted price"); + assert_eq!( + (open, close, high, low), + (40.0, 40.0, 40.0, 40.0), + "the bucket must sit flat at the carried close, not at a sentinel" + ); + } + /// A multi-pool arbitrage transaction whose legs nearly cancel must materialise the /// price its legs executed at, not the signed-net ratio. #[tokio::test] From f0a1b2cf138ce11714ae629c1783516a161774fd Mon Sep 17 00:00:00 2001 From: jakobsn Date: Thu, 30 Jul 2026 11:42:10 +0200 Subject: [PATCH 11/11] Robust pricing? --- src/db/cauldron/candlestick/mod.rs | 17 ++- src/db/cauldron/candlestick/policy.rs | 120 ++++++++++++-------- src/db/cauldron/candlestick/policy/tests.rs | 102 ++++++++++++++--- src/db/cauldron/candlestick/tests.rs | 4 +- src/db/cauldron/ohlcv.rs | 25 ++-- 5 files changed, 185 insertions(+), 83 deletions(-) diff --git a/src/db/cauldron/candlestick/mod.rs b/src/db/cauldron/candlestick/mod.rs index d82f5bb..4001c59 100644 --- a/src/db/cauldron/candlestick/mod.rs +++ b/src/db/cauldron/candlestick/mod.rs @@ -134,7 +134,7 @@ fn aggregate_raw_trades( } } - policy.apply(leg, judge.accepted()); + policy.apply(leg); leg_index += 1; } @@ -402,13 +402,12 @@ async fn fetch_last_close_before( // closes on a print that happened *after* the instant being asked about, which an // earlier revision leaked for any non-hour-aligned `timestamp_end`. if scan_start + 3600 <= timestamp_end { - let exact: Option = sqlx::query_scalar( - "SELECT close FROM ohlcv_1h WHERE token_id = ? AND bucket_ts = ?", - ) - .bind(token_blob) - .bind(scan_start) - .fetch_optional(pool) - .await?; + let exact: Option = + sqlx::query_scalar("SELECT close FROM ohlcv_1h WHERE token_id = ? AND bucket_ts = ?") + .bind(token_blob) + .bind(scan_start) + .fetch_optional(pool) + .await?; if exact.is_some() { return Ok(exact); @@ -427,7 +426,7 @@ async fn fetch_last_close_before( if judge.accepted() { last_accepted_price = judge.price; } - policy.apply(leg, judge.accepted()); + policy.apply(leg); } if last_accepted_price.is_some() { return Ok(last_accepted_price); diff --git a/src/db/cauldron/candlestick/policy.rs b/src/db/cauldron/candlestick/policy.rs index 94ca044..a49966c 100644 --- a/src/db/cauldron/candlestick/policy.rs +++ b/src/db/cauldron/candlestick/policy.rs @@ -106,14 +106,16 @@ impl Default for GuardParams { } } -/// Per-pool state: reserves and qualification credit. +/// Per-pool state: reserves only. +/// +/// Qualification credit is *derived* from these on demand (see [`credit_from`]) rather than +/// stored, which is what keeps a policy's verdicts a function of the reserves it holds +/// rather than of the order it learned them in. #[derive(Clone, Debug)] struct PoolState { sequence: i64, sats: u64, token_amount: u64, - /// Min-depth at this pool's last accepted print. - credit: u64, } /// The guard: judges each leg against a reference price derived from pool reserves. @@ -140,10 +142,9 @@ impl Policy { /// A policy primed with pool reserves as of some instant. /// - /// Credit is seeded for pools already trading within `F` of the reference, capped by - /// present depth. Pools sitting far off-reference start at zero and must earn credit - /// through an accepted print, so a snapshot cannot hand qualification to a pool that - /// was walked away from the market before the window opened. + /// Nothing beyond reserves needs priming: credit is derived from them (see + /// [`credit_from`]), so a seeded policy and one that folded its way to the same + /// reserves are indistinguishable by construction. pub fn seeded(params: GuardParams, snapshot: &[PoolReserves]) -> Self { let mut policy = Self { pools: HashMap::with_capacity(snapshot.len()), @@ -159,24 +160,10 @@ impl Policy { sequence: r.sequence, sats: r.sats, token_amount: r.token_amount, - credit: 0, }, ); } policy.recompute_reference(); - - if let Some(reference) = policy.reference { - let f = policy.params.max_deviation_factor; - for state in policy.pools.values_mut() { - let Some(spot) = spot_ratio(state.sats, state.token_amount) else { - continue; - }; - if deviation(spot, reference) <= f { - state.credit = min_depth(state.sats, state.token_amount, reference); - } - } - } - policy } @@ -190,8 +177,7 @@ impl Policy { }; } - let price = - leg.sats_delta.unsigned_abs() as f64 / leg.token_delta.unsigned_abs() as f64; + let price = leg.sats_delta.unsigned_abs() as f64 / leg.token_delta.unsigned_abs() as f64; let Some(reference) = self.reference else { return JudgeResult { @@ -208,11 +194,24 @@ impl Policy { }; } - let credit = self.pools.get(&leg.pool).map_or(0, |s| s.credit); - let largest_credit = self.pools.values().map(|s| s.credit).max().unwrap_or(0); - // `ceil().max(1)` keeps a pool that has never had an accepted print (credit 0) - // from qualifying just because the largest credit is small. - let threshold = ((largest_credit as f64 * self.params.min_share_fraction).ceil() as u64).max(1); + // Both sides derived from present reserves, so this scan is the whole of tier 2's + // state. It is O(pools), but only reached by off-band legs — `WithinBand` returns + // above — so the common path stays a single comparison. + let f = self.params.max_deviation_factor; + let credit = self + .pools + .get(&leg.pool) + .map_or(0, |s| credit_from(s.sats, s.token_amount, reference, f)); + let largest_credit = self + .pools + .values() + .map(|s| credit_from(s.sats, s.token_amount, reference, f)) + .max() + .unwrap_or(0); + // `ceil().max(1)` keeps a pool with no usable depth (credit 0) from qualifying just + // because the largest credit is small. + let threshold = + ((largest_credit as f64 * self.params.min_share_fraction).ceil() as u64).max(1); let verdict = if credit >= threshold { Verdict::CreditExempt { @@ -236,15 +235,14 @@ impl Policy { /// Advance state past a leg. Must be called for every leg, accepted or not: a muted /// leg still moved real reserves. - pub fn apply(&mut self, leg: &Leg, accepted: bool) { - let reference = self.reference; - + /// + /// Takes no verdict, because none is needed — credit follows the reserves this records. + pub fn apply(&mut self, leg: &Leg) { // `i64::MIN` so a pool's very first leg clears the monotonicity guard below. let state = self.pools.entry(leg.pool).or_insert(PoolState { sequence: i64::MIN, sats: 0, token_amount: 0, - credit: 0, }); // Strict `>`: the `i64::MIN` sentinel above is what lets a pool's first leg through, @@ -253,15 +251,6 @@ impl Policy { state.sequence = leg.sequence; state.sats = leg.sats; state.token_amount = leg.token_amount; - - if accepted && leg.token_delta != 0 { - // Value the token side at the reference the leg was judged against; for a - // token's very first print there is none, so use the leg's own price. - let valuation = reference.unwrap_or_else(|| { - leg.sats_delta.unsigned_abs() as f64 / leg.token_delta.unsigned_abs() as f64 - }); - state.credit = min_depth(leg.sats, leg.token_amount, valuation); - } } self.recompute_reference(); @@ -275,7 +264,17 @@ impl Policy { /// Qualification credit held by a pool. Exposed for tests and diagnostics. #[allow(dead_code)] pub fn credit_of(&self, pool: &[u8; 32]) -> u64 { - self.pools.get(pool).map_or(0, |s| s.credit) + let Some(reference) = self.reference else { + return 0; + }; + self.pools.get(pool).map_or(0, |s| { + credit_from( + s.sats, + s.token_amount, + reference, + self.params.max_deviation_factor, + ) + }) } fn recompute_reference(&mut self) { @@ -290,6 +289,33 @@ impl Policy { } } +/// Qualification credit: a pool's min-depth, but only while its own spot sits within `F` of +/// the reference. +/// +/// Stateless by construction, and that is the point. An earlier revision stored "min-depth +/// at the pool's last accepted print", which made credit path-dependent — a reserve +/// snapshot cannot recover it, so [`Policy::seeded`] had to approximate, and a seeded policy +/// returned `CreditExempt` where a folded one returned `Muted` on the very same leg. That +/// broke the invariant the whole seeding design exists to hold, and because `ohlcv_1h` is +/// written with `INSERT OR IGNORE`, whichever path ran first would freeze its answer in +/// permanently. +/// +/// Little security was given up. The path-dependent rule was a speed bump rather than a +/// wall: an attacker needed one cheap in-band trade to convert a fresh pool into a credited +/// one. What remains is a pure depth gate — printing far off-reference requires a pool +/// holding a real share of the token's liquidity, which is the part that actually costs an +/// attacker money. A pool that has been walked off-market is still excluded outright, so a +/// snapshot cannot launder an attacker's pool into qualification. +fn credit_from(sats: u64, token_amount: u64, reference: f64, f: f64) -> u64 { + let Some(spot) = spot_ratio(sats, token_amount) else { + return 0; + }; + if deviation(spot, reference) > f { + return 0; + } + min_depth(sats, token_amount, reference) +} + /// A pool's spot price, or `None` if it is too small or one-sided to quote one. fn spot_ratio(sats: u64, token_amount: u64) -> Option { if token_amount < MIN_TOKEN_RESERVE || sats == 0 { @@ -300,7 +326,10 @@ fn spot_ratio(sats: u64, token_amount: u64) -> Option { /// How far apart two prices are, as a factor >= 1 in whichever direction. fn deviation(price: f64, reference: f64) -> f64 { - if !(price > 0.0) || !(reference > 0.0) || !price.is_finite() || !reference.is_finite() { + // Finiteness is tested first so the `<=` comparisons below are total: NaN fails + // `is_finite` and returns here, which is why plain `<=` is safe rather than needing + // `!(price > 0.0)` to catch it. + if !price.is_finite() || !reference.is_finite() || price <= 0.0 || reference <= 0.0 { return f64::INFINITY; } (price / reference).max(reference / price) @@ -360,10 +389,7 @@ fn weighted_median_reference(entries: &mut [(f64, u64, u64)]) -> Option { /// Compares `2 * cumulative >= total` rather than `cumulative >= total / 2` so integer /// division cannot bias the pick downwards. When every weight is zero it falls back to /// the positional median, so an all-dust set does not collapse to its smallest ratio. -fn median_by( - sorted: &[(f64, u64, u64)], - weight: impl Fn(&(f64, u64, u64)) -> u64, -) -> Option { +fn median_by(sorted: &[(f64, u64, u64)], weight: impl Fn(&(f64, u64, u64)) -> u64) -> Option { let total: u128 = sorted.iter().map(|e| weight(e) as u128).sum(); if total == 0 { // Defensive only: `spot_ratio` rejects `sats == 0`, so every live weight is >= 1. diff --git a/src/db/cauldron/candlestick/policy/tests.rs b/src/db/cauldron/candlestick/policy/tests.rs index bd8b98f..b5662d5 100644 --- a/src/db/cauldron/candlestick/policy/tests.rs +++ b/src/db/cauldron/candlestick/policy/tests.rs @@ -45,7 +45,7 @@ fn reserves(pool: u8, sequence: i64, sats: u64, token_amount: u64) -> PoolReserv /// Feed a leg through judge-then-apply, returning the verdict. fn step(policy: &mut Policy, l: &Leg) -> JudgeResult { let judged = policy.judge(l); - policy.apply(l, judged.accepted()); + policy.apply(l); judged } @@ -95,18 +95,26 @@ fn test_leg_far_from_the_reference_without_credit_is_muted() { step(&mut policy, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); // A fresh pool printing 100x off market, with no accepted print behind it. - let judged = step(&mut policy, &leg(9, 2, 10_000_000, -1_000, 10_001_000, 1_010)); - assert!(matches!(judged.verdict, Verdict::Muted { .. }), "{:?}", judged.verdict); + let judged = step( + &mut policy, + &leg(9, 2, 10_000_000, -1_000, 10_001_000, 1_010), + ); + assert!( + matches!(judged.verdict, Verdict::Muted { .. }), + "{:?}", + judged.verdict + ); assert!(!judged.accepted()); // Muted, but the price was still computed — the caller still counts its volume. assert!(judged.price.is_some()); } /// Regression: `apply` inserted new pools with `sequence: leg.sequence` and then gated the -/// credit update behind `leg.sequence > state.sequence`, which is false on that very first -/// insert. A pool therefore had to make two accepted prints before earning any credit. +/// reserve update behind `leg.sequence > state.sequence`, which is false on that very first +/// insert — so a pool's opening leg left it recorded with zero reserves, and therefore zero +/// credit. The `i64::MIN` sentinel is what fixes it. #[test] -fn test_pool_earns_credit_on_its_first_accepted_print() { +fn test_pool_holds_credit_after_its_very_first_leg() { let mut policy = Policy::new(params()); step(&mut policy, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); @@ -116,23 +124,68 @@ fn test_pool_earns_credit_on_its_first_accepted_print() { assert!( policy.credit_of(&[2; 32]) > 0, - "a pool must hold credit after its first accepted print, not its second" + "a pool's opening leg must record its reserves, not leave it at zero" ); } +/// A wild print leaves the pool that made it sitting off-market, and an off-market pool +/// holds no credit — so the pool cannot use one absurd leg to qualify the next. #[test] -fn test_muted_leg_does_not_earn_credit() { +fn test_pool_left_off_market_by_its_own_print_holds_no_credit() { let mut policy = Policy::new(params()); step(&mut policy, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); - step(&mut policy, &leg(9, 2, 10_000_000, -1_000, 10_001_000, 1_010)); + step( + &mut policy, + &leg(9, 2, 10_000_000, -1_000, 10_001_000, 1_010), + ); assert_eq!( policy.credit_of(&[9; 32]), 0, - "a muted print must not qualify the pool that made it" + "a pool 99x off the reference must not qualify for tier 2" ); } +/// Regression for the seeded/folded credit divergence — the reason credit is derived from +/// reserves rather than remembered. +/// +/// Pool 2's only leg is a one-token trade at 100x the market. It is muted, but it is far too +/// small to move the pool's own spot, so the pool is left deep and in-band. Under the old +/// "min-depth at the last accepted print" rule the folded policy held pool 2 at zero credit +/// while a policy seeded from the resulting reserves granted it full credit — and the two +/// then returned `Muted` and `CreditExempt` for the identical next leg. Since `ohlcv_1h` is +/// written with `INSERT OR IGNORE`, whichever path ran first would have frozen its answer in. +#[test] +fn test_seeded_and_folded_agree_on_credit_after_a_muted_print() { + let mut folded = Policy::new(params()); + step(&mut folded, &leg(1, 1, 100_000, -1_000, 1_000_000, 10_000)); + // Pool 2's opening leg: absurd price, negligible size, so it ends in-band. + step(&mut folded, &leg(2, 2, 10_000, -1, 2_010_000, 19_999)); + + let seeded = Policy::seeded( + params(), + &[ + reserves(1, 1, 1_000_000, 10_000), + reserves(2, 2, 2_010_000, 19_999), + ], + ); + + assert_eq!(folded.reference(), seeded.reference()); + assert_eq!( + folded.credit_of(&[2; 32]), + seeded.credit_of(&[2; 32]), + "credit must not depend on whether the policy folded history or was seeded from it" + ); + assert!( + folded.credit_of(&[2; 32]) > 0, + "the test is vacuous unless the pool actually ends up in-band and credited" + ); + + // The consequence that made this worth fixing: identical verdicts on the next leg. + let next = leg(2, 3, 500_000, -1, 2_510_000, 19_998); + assert_eq!(folded.judge(&next).verdict, seeded.judge(&next).verdict); +} + /// The property the whole seeded design exists for: the reference depends only on pool /// reserves, never on how many legs the caller happened to read to arrive at them. #[test] @@ -145,7 +198,10 @@ fn test_reference_is_a_function_of_reserves_not_of_history_read() { // Same end state, reached by seeding instead of folding. let seeded = Policy::seeded( params(), - &[reserves(1, 3, 1_105_000, 9_000), reserves(2, 2, 2_000_000, 20_000)], + &[ + reserves(1, 3, 1_105_000, 9_000), + reserves(2, 2, 2_000_000, 20_000), + ], ); assert_eq!( @@ -165,7 +221,10 @@ fn test_seeded_and_folded_policies_agree_on_a_shared_leg() { let seeded = Policy::seeded( params(), - &[reserves(1, 1, 1_000_000, 10_000), reserves(2, 2, 2_000_000, 20_000)], + &[ + reserves(1, 1, 1_000_000, 10_000), + reserves(2, 2, 2_000_000, 20_000), + ], ); let shared = leg(1, 3, 300_000, -1_000, 1_300_000, 9_000); @@ -184,9 +243,9 @@ fn test_seeding_withholds_credit_from_off_market_pools() { let policy = Policy::seeded( params(), &[ - reserves(1, 1, 1_000_000, 10_000), // spot 100, at market - reserves(2, 2, 2_000_000, 20_000), // spot 100, at market - reserves(9, 3, 10_000_000, 1_000), // spot 10_000, 100x off + reserves(1, 1, 1_000_000, 10_000), // spot 100, at market + reserves(2, 2, 2_000_000, 20_000), // spot 100, at market + reserves(9, 3, 10_000_000, 1_000), // spot 10_000, 100x off ], ); @@ -238,7 +297,9 @@ fn test_pools_that_cannot_quote_a_ratio_are_excluded() { ], ); - let reference = policy.reference().expect("the one quotable pool sets the reference"); + let reference = policy + .reference() + .expect("the one quotable pool sets the reference"); assert!( reference.is_finite() && reference == 100.0, "got {reference}" @@ -356,6 +417,15 @@ fn test_deviation_is_symmetric_and_rejects_degenerate_input() { assert_eq!(deviation(50.0, 100.0), 2.0); assert_eq!(deviation(0.0, 100.0), f64::INFINITY); assert_eq!(deviation(100.0, 0.0), f64::INFINITY); + // NaN and the infinities must land on INFINITY (i.e. "out of band"), never sneak + // through as a small deviation. `<=` alone is false for NaN, so the finiteness test + // has to run first — this is what pins that ordering. + assert_eq!(deviation(f64::NAN, 100.0), f64::INFINITY); + assert_eq!(deviation(100.0, f64::NAN), f64::INFINITY); + assert_eq!(deviation(f64::INFINITY, 100.0), f64::INFINITY); + assert_eq!(deviation(100.0, f64::INFINITY), f64::INFINITY); + assert_eq!(deviation(-100.0, 100.0), f64::INFINITY); + assert_eq!(deviation(100.0, -100.0), f64::INFINITY); } /// A leg replayed at or below a pool's known sequence must not rewind its reserves. diff --git a/src/db/cauldron/candlestick/tests.rs b/src/db/cauldron/candlestick/tests.rs index e325cc4..ffff7b8 100644 --- a/src/db/cauldron/candlestick/tests.rs +++ b/src/db/cauldron/candlestick/tests.rs @@ -69,7 +69,9 @@ async fn register_pool( .unwrap(); if let Some((spent, txid, ts)) = withdrawn_utxo { - insert_mempool_tx(&mut **conn, &txid, ts as u64).await.unwrap(); + insert_mempool_tx(&mut **conn, &txid, ts as u64) + .await + .unwrap(); sqlx::query("INSERT OR REPLACE INTO utxo_spending (spent_utxo_hash, txid) VALUES (?, ?)") .bind(spent.to_blob()) .bind(txid.to_blob()) diff --git a/src/db/cauldron/ohlcv.rs b/src/db/cauldron/ohlcv.rs index 01892fa..95b6f67 100644 --- a/src/db/cauldron/ohlcv.rs +++ b/src/db/cauldron/ohlcv.rs @@ -22,7 +22,10 @@ use sqlx::{Row, SqlitePool}; /// written flat at the carried close instead of dropped, restoring their volume; the /// rebuild seeds from confirmed-only reserves, so output no longer depends on mempool /// contents at rebuild time. -pub const OHLCV_VERSION: u32 = 5; +/// 6: qualification credit derived from present reserves instead of remembered from a +/// pool's last accepted print, so a seeded policy and a folded one can no longer +/// disagree on the same leg. Tier 2 is now a pure depth gate. +pub const OHLCV_VERSION: u32 = 6; const OHLCV_VERSION_KEY: &str = "ohlcv_version"; pub async fn create_table(pool: &SqlitePool) { @@ -213,7 +216,7 @@ ORDER BY phe.token_id ASC, phe.effective_timestamp ASC, phe.sequence ASC; } } - policy.apply(&leg, judge.accepted()); + policy.apply(&leg); } // Phase 3: insert all computed buckets in one transaction. @@ -607,7 +610,10 @@ mod tests { .await .unwrap(); - assert_eq!(volume, 25, "volume must survive a bucket with no accepted price"); + assert_eq!( + volume, 25, + "volume must survive a bucket with no accepted price" + ); assert_eq!( (open, close, high, low), (40.0, 40.0, 40.0, 40.0), @@ -650,13 +656,12 @@ mod tests { .await .unwrap(); - let (close, open, high, low): (f64, f64, f64, f64) = sqlx::query_as( - "SELECT close, open, high, low FROM ohlcv_1h WHERE token_id = ?", - ) - .bind(token.as_slice()) - .fetch_one(&pool) - .await - .unwrap(); + let (close, open, high, low): (f64, f64, f64, f64) = + sqlx::query_as("SELECT close, open, high, low FROM ohlcv_1h WHERE token_id = ?") + .bind(token.as_slice()) + .fetch_one(&pool) + .await + .unwrap(); // Per-leg pricing: each leg executes at its own price. // Leg 1 (buy): 446,491,239 / 1,334,527,069 = 0.334569