Merge branch '4-price-token-at-timestamp-does-not-count-previously-active-pools' into 'master'

Count pools that were not withdrawn at the given time.

Closes #4

See merge request riftenlabs/riftenlabs-indexer!80
This commit is contained in:
jakobsn 2026-04-16 12:30:14 +00:00
commit d7c3ae8cc0
2 changed files with 684 additions and 3 deletions

View file

@ -123,9 +123,13 @@ pub(crate) async fn db_visit_pool_entries<T: PoolVisitor>(
) -> Result<()> {
let (phe_timestamp_filter, time_params) = time_filters(&filters);
let withdraw_time_filter = filters
.timestamp_lt
.min(filters.timestamp_lte)
// Take the minimum of whichever time bounds are set.
// Neither set → i64::MAX (current-state query: exclude all withdrawn pools).
// Note: Option::min treats None < Some(x), so we can't use it directly here.
let withdraw_time_filter = [filters.timestamp_lt, filters.timestamp_lte]
.into_iter()
.flatten()
.min()
.unwrap_or(i64::MAX as u64);
let mut outer_filters = Vec::new();

View file

@ -467,6 +467,7 @@ mod tests {
utxo_funding::{self, insert_utxo_funding},
};
use crate::utiltest::mock_db_pool;
use sqlx::SqliteConnection;
use super::*;
use bitcoin_hashes::Hash;
@ -479,6 +480,7 @@ mod tests {
const TIME_1: u64 = 1727963300;
const TIME_2: u64 = 1727963350;
const TIME_3: u64 = 1727963400;
const TIME_4: u64 = 1727963450;
fn dummy_cauldron(
txid: &Txid,
@ -934,6 +936,408 @@ mod tests {
.contains("Invalid timestamp format"));
}
/// Minimal helper: inserts one pool that is already withdrawn.
/// Does NOT insert utxo_funding — poolvisitor does not join it, and FK is disabled.
/// `pool_utxo` doubles as both the pool creation UTXO and the pool ID.
async fn insert_pool_and_withdraw(
conn: &mut SqliteConnection,
pool_utxo: OutPointHash,
token: TokenID,
sats: u64,
tokens: i64,
ts_created: u64,
withdraw_txid: Txid,
ts_withdrawn: u64,
) {
let pkh = PubkeyHash::all_zeros();
let block = BlockHash::all_zeros();
// Fake creation txid — not referenced by poolvisitor; FK disabled so no tx row needed
let fake_txid = Txid::from_inner(*withdraw_txid.as_inner()); // unique: reuse withdraw bytes
let cauldron = dummy_cauldron(
&fake_txid,
&pool_utxo,
&token,
sats,
tokens,
&pkh,
&OutPointHash::all_zeros(),
);
let cauldron_w = ParsedContract {
pkh,
is_withdrawn: true,
spent_utxo_hash: pool_utxo,
new_utxo_hash: None,
new_utxo_txid: None,
new_utxo_n: None,
token_id: None,
sats: None,
token_amount: None,
};
insert_new_pool(conn, &cauldron).await.unwrap();
insert_pool_history_entry(
conn,
&pool_utxo,
&cauldron,
Some(ts_created),
Some(ts_created),
0,
0,
)
.await
.unwrap();
// Withdrawal tx row needed so the timestamp subquery in poolvisitor can find it
insert_block_tx(conn, &withdraw_txid, &block, ts_withdrawn as i64)
.await
.unwrap();
insert_utxo_spending(conn, &vec![cauldron_w.clone()], &withdraw_txid, true)
.await
.unwrap();
flag_as_withdrawn(conn, &pool_utxo, &cauldron_w)
.await
.unwrap();
}
async fn setup_single_withdrawn_pool(pool: &sqlx::SqlitePool, token: &TokenID) {
utxo_funding::create_table(pool).await;
utxo_spending::create_table(pool).await;
tx::create_table(pool).await;
pool::create_table(pool).await;
dummy_init_seq();
let mut conn = pool.acquire().await.unwrap();
let pkh = PubkeyHash::all_zeros();
let block = BlockHash::all_zeros();
let creation_txid = Txid::from_inner([0xa0; 32]);
let creation_utxo = OutPointHash::from_inner([0xb0; 32]);
let withdraw_txid = Txid::from_inner([0xa1; 32]);
let cauldron = dummy_cauldron(
&creation_txid,
&creation_utxo,
token,
100_000,
2_000,
&pkh,
&OutPointHash::all_zeros(),
);
let cauldron_withdraw = ParsedContract {
pkh,
is_withdrawn: true,
spent_utxo_hash: creation_utxo,
new_utxo_hash: None,
new_utxo_txid: None,
new_utxo_n: None,
token_id: None,
sats: None,
token_amount: None,
};
insert_new_pool(&mut *conn, &cauldron).await.unwrap();
insert_utxo_funding(&mut *conn, &vec![cauldron.clone()], &creation_txid, true)
.await
.unwrap();
insert_block_tx(&mut *conn, &creation_txid, &block, TIME_1 as i64)
.await
.unwrap();
update_pool_history(&mut *conn, vec![cauldron], Some(TIME_1), Some(TIME_1))
.await
.unwrap();
insert_block_tx(&mut *conn, &withdraw_txid, &block, TIME_3 as i64)
.await
.unwrap();
insert_utxo_spending(
&mut *conn,
&vec![cauldron_withdraw.clone()],
&withdraw_txid,
true,
)
.await
.unwrap();
flag_as_withdrawn(&mut *conn, &creation_utxo, &cauldron_withdraw)
.await
.unwrap();
}
// Issue #4: price_at should include pools that were active at the queried timestamp
// even if they have since been withdrawn.
/// Query at TIME_2 (between creation TIME_1 and withdrawal TIME_3) must return
/// the pool's price because the pool was active at that moment.
#[rocket::async_test]
async fn test_price_at_includes_withdrawn_pool_when_active_at_query_time() {
let token = TokenID::from_inner([0x11; 32]);
let token_hex = token.to_hex();
let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move {
setup_single_withdrawn_pool(&pool, &TokenID::from_inner([0x11; 32])).await;
})
.await;
let rocket = rocket::build()
.manage(mock_db)
.mount("/cauldron", routes![price_at]);
let client = Client::tracked(rocket)
.await
.expect("valid rocket instance");
let response = client
.get(format!("/cauldron/price/{token_hex}/at/{TIME_2}"))
.dispatch()
.await;
assert_eq!(
response.status(),
Status::Ok,
"expected price for pool active at TIME_2, got: {}",
response.into_string().await.unwrap()
);
let json: Value = serde_json::from_str(&response.into_string().await.unwrap()).unwrap();
let price = json["price"].as_f64().unwrap();
// 100_000 sats / 2_000 tokens = 50.0
assert!((price - 50.0).abs() < 0.01, "expected 50.0, got {price}");
}
/// Query at TIME_3, exactly the withdrawal timestamp.
/// withdrawal_ts >= withdraw_time_filter → TIME_3 >= TIME_3 → true, so pool is included.
#[rocket::async_test]
async fn test_price_at_includes_withdrawn_pool_at_exact_withdrawal_timestamp() {
let token = TokenID::from_inner([0x12; 32]);
let token_hex = token.to_hex();
let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move {
setup_single_withdrawn_pool(&pool, &TokenID::from_inner([0x12; 32])).await;
})
.await;
let rocket = rocket::build()
.manage(mock_db)
.mount("/cauldron", routes![price_at]);
let client = Client::tracked(rocket)
.await
.expect("valid rocket instance");
let response = client
.get(format!("/cauldron/price/{token_hex}/at/{TIME_3}"))
.dispatch()
.await;
assert_eq!(
response.status(),
Status::Ok,
"expected pool included at its withdrawal timestamp, got: {}",
response.into_string().await.unwrap()
);
let json: Value = serde_json::from_str(&response.into_string().await.unwrap()).unwrap();
assert!((json["price"].as_f64().unwrap() - 50.0).abs() < 0.01);
}
/// Query at TIME_4 (after withdrawal TIME_3): pool must be excluded → 404.
#[rocket::async_test]
async fn test_price_at_excludes_withdrawn_pool_after_withdrawal() {
let token = TokenID::from_inner([0x13; 32]);
let token_hex = token.to_hex();
let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move {
setup_single_withdrawn_pool(&pool, &TokenID::from_inner([0x13; 32])).await;
})
.await;
let rocket = rocket::build()
.manage(mock_db)
.mount("/cauldron", routes![price_at]);
let client = Client::tracked(rocket)
.await
.expect("valid rocket instance");
let response = client
.get(format!("/cauldron/price/{token_hex}/at/{TIME_4}"))
.dispatch()
.await;
assert_eq!(
response.status(),
Status::NotFound,
"expected 404 after pool withdrawal, got: {}",
response.into_string().await.unwrap()
);
}
/// Token has two pools; one is later withdrawn.
///
/// Pool A (token [0x14;32]): 100_000 sats / 2_000 tokens, always active
/// Pool B (token [0x14;32]): 80_000 sats / 2_000 tokens, withdrawn at TIME_3
///
/// Query at TIME_2 → both contribute: (100_000+80_000)/(2_000+2_000) = 45.0
/// Query at TIME_4 → only A contributes: 100_000/2_000 = 50.0
#[rocket::async_test]
async fn test_price_at_mixed_active_and_withdrawn_pools() {
let token = TokenID::from_inner([0x14; 32]);
let token_hex = token.to_hex();
let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move {
utxo_funding::create_table(&pool).await;
utxo_spending::create_table(&pool).await;
tx::create_table(&pool).await;
pool::create_table(&pool).await;
dummy_init_seq();
let mut conn = pool.acquire().await.unwrap();
let token = TokenID::from_inner([0x14; 32]);
let pkh = PubkeyHash::all_zeros();
let block = BlockHash::all_zeros();
// Pool A: always active
let txid_a = Txid::from_inner([0xc0; 32]);
let utxo_a = OutPointHash::from_inner([0xd0; 32]);
let cauldron_a = dummy_cauldron(
&txid_a,
&utxo_a,
&token,
100_000,
2_000,
&pkh,
&OutPointHash::all_zeros(),
);
insert_new_pool(&mut *conn, &cauldron_a).await.unwrap();
insert_utxo_funding(&mut *conn, &vec![cauldron_a.clone()], &txid_a, true)
.await
.unwrap();
insert_block_tx(&mut *conn, &txid_a, &block, TIME_1 as i64)
.await
.unwrap();
update_pool_history(&mut *conn, vec![cauldron_a], Some(TIME_1), Some(TIME_1))
.await
.unwrap();
// Pool B: active at TIME_1, withdrawn at TIME_3
let txid_b = Txid::from_inner([0xc1; 32]);
let utxo_b = OutPointHash::from_inner([0xd1; 32]);
let txid_b_withdraw = Txid::from_inner([0xc2; 32]);
let cauldron_b = dummy_cauldron(
&txid_b,
&utxo_b,
&token,
80_000,
2_000,
&pkh,
&OutPointHash::all_zeros(),
);
let cauldron_b_withdraw = ParsedContract {
pkh,
is_withdrawn: true,
spent_utxo_hash: utxo_b,
new_utxo_hash: None,
new_utxo_txid: None,
new_utxo_n: None,
token_id: None,
sats: None,
token_amount: None,
};
insert_new_pool(&mut *conn, &cauldron_b).await.unwrap();
insert_utxo_funding(&mut *conn, &vec![cauldron_b.clone()], &txid_b, true)
.await
.unwrap();
insert_block_tx(&mut *conn, &txid_b, &block, TIME_1 as i64)
.await
.unwrap();
update_pool_history(&mut *conn, vec![cauldron_b], Some(TIME_1), Some(TIME_1))
.await
.unwrap();
insert_block_tx(&mut *conn, &txid_b_withdraw, &block, TIME_3 as i64)
.await
.unwrap();
insert_utxo_spending(
&mut *conn,
&vec![cauldron_b_withdraw.clone()],
&txid_b_withdraw,
true,
)
.await
.unwrap();
flag_as_withdrawn(&mut *conn, &utxo_b, &cauldron_b_withdraw)
.await
.unwrap();
})
.await;
let rocket = rocket::build()
.manage(mock_db)
.mount("/cauldron", routes![price_at]);
let client = Client::tracked(rocket)
.await
.expect("valid rocket instance");
// Before withdrawal: both pools
let response = client
.get(format!("/cauldron/price/{token_hex}/at/{TIME_2}"))
.dispatch()
.await;
assert_eq!(response.status(), Status::Ok);
let json: Value = serde_json::from_str(&response.into_string().await.unwrap()).unwrap();
let price = json["price"].as_f64().unwrap();
// (100_000 + 80_000) / (2_000 + 2_000) = 45.0
assert!(
(price - 45.0).abs() < 0.01,
"expected 45.0 (both pools), got {price}"
);
// After withdrawal: only pool A
let response = client
.get(format!("/cauldron/price/{token_hex}/at/{TIME_4}"))
.dispatch()
.await;
assert_eq!(response.status(), Status::Ok);
let json: Value = serde_json::from_str(&response.into_string().await.unwrap()).unwrap();
let price = json["price"].as_f64().unwrap();
// 100_000 / 2_000 = 50.0
assert!(
(price - 50.0).abs() < 0.01,
"expected 50.0 (only active pool), got {price}"
);
}
/// All pools for a token are withdrawn. Historical query before withdrawal returns
/// price; query after returns 404.
#[rocket::async_test]
async fn test_price_at_all_pools_withdrawn() {
let token = TokenID::from_inner([0x15; 32]);
let token_hex = token.to_hex();
let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move {
setup_single_withdrawn_pool(&pool, &TokenID::from_inner([0x15; 32])).await;
})
.await;
let rocket = rocket::build()
.manage(mock_db)
.mount("/cauldron", routes![price_at]);
let client = Client::tracked(rocket)
.await
.expect("valid rocket instance");
// Before withdrawal → price available
let response = client
.get(format!("/cauldron/price/{token_hex}/at/{TIME_1}"))
.dispatch()
.await;
assert_eq!(
response.status(),
Status::Ok,
"expected price before all pools withdrew: {}",
response.into_string().await.unwrap()
);
// After withdrawal → 404; no active liquidity at query time
let response = client
.get(format!("/cauldron/price/{token_hex}/at/{TIME_4}"))
.dispatch()
.await;
assert_eq!(
response.status(),
Status::NotFound,
"expected 404 after all pools withdrew: {}",
response.into_string().await.unwrap()
);
}
#[rocket::async_test]
async fn test_price_with_high_tokens_and_low_sats() {
let mock_db = mock_db_pool(setup_mock_db).await;
@ -991,4 +1395,277 @@ mod tests {
let json: serde_json::Value = serde_json::from_str(&body).expect("Response should be JSON");
assert!(json["error"]["code"].as_str().is_some());
}
// ── Tests derived from real production data ────────────────────────────
//
// Querying indexer.riften.net (old code) vs the fixed SQL on the local DB
// revealed two concrete failure modes:
//
// 1. Token 422134…: ALL 25 pools eventually withdrawn.
// Old code → 404 for every historical timestamp.
// Fix → returns real prices.
//
// 2. Token b38a33f…: 921 withdrawn pools + 108 still active.
// Old code → slightly wrong TVL (misses withdrawn-but-once-active pools).
// Fix → accurate weighted price using all pools active at query time.
//
// The tests below recreate these scenarios with minimal synthetic data.
/// Mirrors token 422134: multiple successive pool generations, ALL eventually
/// withdrawn. Without the fix every historical query returns 404 because the
/// WHERE clause reduces to `withdrawn_in_utxo IS NULL`, which matches nothing.
///
/// Timeline (BASE = 2025-09-01):
/// T1 = BASE+0 Pool A created (sats=120_000, tokens=2_000)
/// T2 = BASE+100 Pool B created (sats= 80_000, tokens=2_000) ← A+B overlap
/// T3 = BASE+200 Pool A withdrawn ← B alone
/// T4 = BASE+300 Pool B withdrawn ← gap, 404
/// T5 = BASE+400 Pool C created (sats= 90_000, tokens=3_000) ← C alone
/// T6 = BASE+500 Pool C withdrawn ← all gone, 404
///
/// Expected prices (sats_total / tokens_total):
/// at T1: A only → 120k/2k = 60.0
/// at T2: A + B → 200k/4k = 50.0
/// at T3: A(boundary) + B → 200k/4k = 50.0 (withdrawal_ts == query_ts)
/// at T3+50: B only → 80k/2k = 40.0
/// at T4+50: gap → 404
/// at T5: C only → 90k/3k = 30.0
/// at T6: C(boundary) → 90k/3k = 30.0
/// at T6+50: all withdrawn → 404
#[rocket::async_test]
async fn test_price_at_multi_generation_all_eventually_withdrawn() {
const BASE: u64 = 1_756_684_800; // 2025-09-01
let token = TokenID::from_inner([0x30; 32]);
let token_hex = token.to_hex();
let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move {
utxo_spending::create_table(&pool).await;
tx::create_table(&pool).await;
pool::create_table(&pool).await;
dummy_init_seq();
let mut conn = pool.acquire().await.unwrap();
let token = TokenID::from_inner([0x30; 32]);
// Pool A: T1 → T3
insert_pool_and_withdraw(
&mut *conn,
OutPointHash::from_inner([0xa1; 32]),
token,
120_000,
2_000,
BASE,
Txid::from_inner([0xb1; 32]),
BASE + 200,
)
.await;
// Pool B: T2 → T4
insert_pool_and_withdraw(
&mut *conn,
OutPointHash::from_inner([0xa2; 32]),
token,
80_000,
2_000,
BASE + 100,
Txid::from_inner([0xb2; 32]),
BASE + 300,
)
.await;
// Pool C: T5 → T6
insert_pool_and_withdraw(
&mut *conn,
OutPointHash::from_inner([0xa3; 32]),
token,
90_000,
3_000,
BASE + 400,
Txid::from_inner([0xb3; 32]),
BASE + 500,
)
.await;
})
.await;
let rocket = rocket::build()
.manage(mock_db)
.mount("/cauldron", routes![price_at]);
let client = Client::tracked(rocket).await.expect("valid rocket");
macro_rules! price_at {
($ts:expr) => {{
let r = client
.get(format!("/cauldron/price/{token_hex}/at/{}", $ts))
.dispatch()
.await;
if r.status() == Status::Ok {
let j: Value = serde_json::from_str(&r.into_string().await.unwrap()).unwrap();
Some(j["price"].as_f64().unwrap())
} else {
None
}
}};
}
// T1: only A → 60.0
let p = price_at!(BASE);
assert!((p.unwrap() - 60.0).abs() < 0.01, "t1: {p:?}");
// T2: A + B → 50.0
let p = price_at!(BASE + 100);
assert!((p.unwrap() - 50.0).abs() < 0.01, "t2: {p:?}");
// T3 boundary: A (withdrawal_ts == query_ts → included) + B → 50.0
let p = price_at!(BASE + 200);
assert!((p.unwrap() - 50.0).abs() < 0.01, "t3 boundary: {p:?}");
// T3+50: A withdrawn before query, only B → 40.0
let p = price_at!(BASE + 250);
assert!((p.unwrap() - 40.0).abs() < 0.01, "t3+50: {p:?}");
// T4+50: gap, both A and B withdrawn before query, C not yet created → 404
assert_eq!(price_at!(BASE + 350), None, "t4+50 gap should be 404");
// T5: only C → 30.0
let p = price_at!(BASE + 400);
assert!((p.unwrap() - 30.0).abs() < 0.01, "t5: {p:?}");
// T6 boundary: C (withdrawal_ts == query_ts → included) → 30.0
let p = price_at!(BASE + 500);
assert!((p.unwrap() - 30.0).abs() < 0.01, "t6 boundary: {p:?}");
// T6+50: all pools withdrawn → 404
assert_eq!(
price_at!(BASE + 550),
None,
"t6+50 all withdrawn should be 404"
);
}
/// Mirrors token b38a33f: token always has some active pools but also many
/// withdrawn ones. Historical price must include pools active at query time,
/// not just currently-active ones.
///
/// Pool A (always active): sats=100_000, tokens=2_000 (price=50)
/// Pool B (withdrawn T2): sats= 60_000, tokens=3_000 (price=20)
/// Pool C (created T2, still active): sats=40_000, tokens=1_000 (price=40)
///
/// At T1 (A+B active, C not yet):
/// fix → (160k/5k) = 32.0 correct: B was active here
/// bug → (100k/2k) = 50.0 wrong: B excluded because withdrawn_in_utxo IS NOT NULL
///
/// After T2 (A+C, B gone):
/// fix → (140k/3k) = 46.67 same as bug (B correctly excluded from both)
/// bug → (140k/3k) = 46.67
#[rocket::async_test]
async fn test_price_at_historical_price_reflects_pools_active_at_query_time() {
const BASE: u64 = 1_756_684_800;
let token = TokenID::from_inner([0x31; 32]);
let token_hex = token.to_hex();
let mock_db = mock_db_pool(|pool: sqlx::SqlitePool| async move {
utxo_spending::create_table(&pool).await;
tx::create_table(&pool).await;
pool::create_table(&pool).await;
dummy_init_seq();
let token = TokenID::from_inner([0x31; 32]);
let pkh = PubkeyHash::all_zeros();
let mut conn = pool.acquire().await.unwrap();
// Pool A: always active — no withdrawal
let utxo_a = OutPointHash::from_inner([0xc1; 32]);
let txid_a = Txid::from_inner([0xd1; 32]);
let ca = dummy_cauldron(
&txid_a,
&utxo_a,
&token,
100_000,
2_000,
&pkh,
&OutPointHash::all_zeros(),
);
insert_new_pool(&mut *conn, &ca).await.unwrap();
insert_pool_history_entry(&mut *conn, &utxo_a, &ca, Some(BASE), Some(BASE), 0, 0)
.await
.unwrap();
// Pool B: active at T1, withdrawn at T2
insert_pool_and_withdraw(
&mut *conn,
OutPointHash::from_inner([0xc2; 32]),
token,
60_000,
3_000,
BASE,
Txid::from_inner([0xd2; 32]),
BASE + 200,
)
.await;
// Pool C: created at T2, never withdrawn — no withdrawal
let utxo_c = OutPointHash::from_inner([0xc3; 32]);
let txid_c = Txid::from_inner([0xd3; 32]);
let cc = dummy_cauldron(
&txid_c,
&utxo_c,
&token,
40_000,
1_000,
&pkh,
&OutPointHash::all_zeros(),
);
insert_new_pool(&mut *conn, &cc).await.unwrap();
insert_pool_history_entry(
&mut *conn,
&utxo_c,
&cc,
Some(BASE + 200),
Some(BASE + 200),
0,
0,
)
.await
.unwrap();
})
.await;
let rocket = rocket::build()
.manage(mock_db)
.mount("/cauldron", routes![price_at]);
let client = Client::tracked(rocket).await.expect("valid rocket");
macro_rules! price_at {
($ts:expr) => {{
let r = client
.get(format!("/cauldron/price/{token_hex}/at/{}", $ts))
.dispatch()
.await;
if r.status() == Status::Ok {
let j: Value = serde_json::from_str(&r.into_string().await.unwrap()).unwrap();
Some(j["price"].as_f64().unwrap())
} else {
None
}
}};
}
// T1: A+B active → (100k+60k)/(2k+3k) = 32.0
// Without fix: only A (B excluded as withdrawn) → 50.0 ← wrong
let p = price_at!(BASE).unwrap();
assert!(
(p - 32.0).abs() < 0.01,
"at T1 expected 32.0 (A+B), got {p} — old code returns 50.0 (A only)"
);
// T2 boundary: A + B(boundary, withdrawal_ts==T2) + C → (200k/6k) = 33.33
let p = price_at!(BASE + 200).unwrap();
assert!((p - 33.33).abs() < 0.1, "at T2 boundary: {p}");
// After T2: A+C, B correctly excluded by both old and new code → 46.67
let p = price_at!(BASE + 300).unwrap();
assert!(
(p - 46.67).abs() < 0.1,
"after T2 expected 46.67 (A+C), got {p}"
);
}
}