From f05d4c6c3456c333e15eb98cf84d025958fe6f75 Mon Sep 17 00:00:00 2001 From: jakobsn Date: Tue, 28 Jul 2026 14:20:37 +0200 Subject: [PATCH] Expose first_pool_ts in cached token list API and add it as a sort key Adds the already-backfilled cached_token_metrics.first_pool_ts column to the TokenListItemCached response (nullable, unix seconds) in all three query paths (list_cached, search_cached, list_cached_by_ids), and accepts by=first_pool_ts on the cached list endpoints so a frontend can fetch "newest tokens" in one call. NULLs (and the 0 "not backfilled" sentinel, via NULLIF) always sort last regardless of order, so tokens without a timestamp never pollute newest results. Includes a matching expression index and sort/serialization tests for both the list and search paths. Co-Authored-By: Claude Fable 5 --- src/db/cauldron/tokenlist/db_utils.rs | 12 ++++ src/db/cauldron/tokenlist/list_cached.rs | 7 +++ src/db/cauldron/tokenlist/test.rs | 64 +++++++++++++++++++++ src/db/search/mod.rs | 7 ++- src/db/search/test.rs | 71 ++++++++++++++++++++++++ src/rpc/tokens.rs | 13 ++++- 6 files changed, 171 insertions(+), 3 deletions(-) diff --git a/src/db/cauldron/tokenlist/db_utils.rs b/src/db/cauldron/tokenlist/db_utils.rs index a3cff88..77b5364 100644 --- a/src/db/cauldron/tokenlist/db_utils.rs +++ b/src/db/cauldron/tokenlist/db_utils.rs @@ -32,6 +32,8 @@ pub enum CachedSort { Change7dUsdAsc, Apy30dDesc, Apy30dAsc, + FirstPoolTsDesc, + FirstPoolTsAsc, } pub fn order_clause(sort: CachedSort) -> &'static str { @@ -90,6 +92,14 @@ pub fn order_clause(sort: CachedSort) -> &'static str { } CachedSort::Apy30dDesc => "ORDER BY apy_30d_bp IS NULL, apy_30d_bp DESC, token_id ASC", CachedSort::Apy30dAsc => "ORDER BY apy_30d_bp IS NULL, apy_30d_bp ASC, token_id ASC", + // NULLIF(...,0): 0 is the "not backfilled yet" sentinel — sort it with the NULLs, + // last, so un-backfilled tokens never pollute "newest" results. + CachedSort::FirstPoolTsDesc => { + "ORDER BY NULLIF(first_pool_ts,0) IS NULL, NULLIF(first_pool_ts,0) DESC, token_id ASC" + } + CachedSort::FirstPoolTsAsc => { + "ORDER BY NULLIF(first_pool_ts,0) IS NULL, NULLIF(first_pool_ts,0) ASC, token_id ASC" + } } } @@ -131,6 +141,8 @@ ON cached_token_metrics(score, trade_volume, token_id); .execute(pool).await?; sqlx::query("CREATE INDEX IF NOT EXISTS idx_ctm_score_rank ON cached_token_metrics(score_rank, token_id);") .execute(pool).await?; + sqlx::query("CREATE INDEX IF NOT EXISTS idx_ctm_first_pool_ts_ord ON cached_token_metrics((NULLIF(first_pool_ts,0) IS NULL), NULLIF(first_pool_ts,0), token_id);") + .execute(pool).await?; sqlx::query("ANALYZE;").execute(pool).await?; sqlx::query("PRAGMA optimize;").execute(pool).await?; Ok(()) diff --git a/src/db/cauldron/tokenlist/list_cached.rs b/src/db/cauldron/tokenlist/list_cached.rs index d6e4a94..059fd07 100644 --- a/src/db/cauldron/tokenlist/list_cached.rs +++ b/src/db/cauldron/tokenlist/list_cached.rs @@ -35,6 +35,7 @@ pub struct TokenListItemCached { pub change_24h_usd_bp: Option, pub change_7d_usd_bp: Option, pub apy_30d_bp: Option, + pub first_pool_ts: Option, } const TOKEN_METRICS_COLUMNS: &str = r#" @@ -56,6 +57,7 @@ const TOKEN_METRICS_COLUMNS: &str = r#" change_24h_usd_bp, change_7d_usd_bp, apy_30d_bp, + first_pool_ts, score_rank, bcmr_json, bcmr_well_known_json @@ -92,6 +94,10 @@ fn parse_row(row: &sqlx::sqlite::SqliteRow) -> Result { let change_24h_usd_bp: Option = row.get("change_24h_usd_bp"); let change_7d_usd_bp: Option = row.get("change_7d_usd_bp"); let apy_30d_bp: Option = row.get("apy_30d_bp"); + // 0 is the "not backfilled yet" sentinel — treat it as unknown. + let first_pool_ts: Option = row + .get::, _>("first_pool_ts") + .filter(|&ts| ts != 0); let score_rank: i64 = row.get("score_rank"); let bcmr_json_str: Option = row.get("bcmr_json"); let bcmr_wk_json_str: Option = row.get("bcmr_well_known_json"); @@ -120,6 +126,7 @@ fn parse_row(row: &sqlx::sqlite::SqliteRow) -> Result { change_24h_usd_bp, change_7d_usd_bp, apy_30d_bp, + first_pool_ts, }) } diff --git a/src/db/cauldron/tokenlist/test.rs b/src/db/cauldron/tokenlist/test.rs index 09de3c0..4995d74 100644 --- a/src/db/cauldron/tokenlist/test.rs +++ b/src/db/cauldron/tokenlist/test.rs @@ -471,6 +471,70 @@ mod tests { assert!(v.is_empty()); } + #[tokio::test] + async fn test_db_list_tokens_cached_first_pool_ts_sort_nulls_last() { + let mock = mock_db_pool(|pool| async move { setup_basic_schemas(pool).await }).await; + let pool = &mock.cauldron_w; + + let t_old = TokenID::from_byte_array([0x61; 32]).to_string(); + let t_new = TokenID::from_byte_array([0x62; 32]).to_string(); + let t_null = TokenID::from_byte_array([0x63; 32]).to_string(); + let t_zero = TokenID::from_byte_array([0x64; 32]).to_string(); + + seed_cached_row( + pool, &t_old, 1, 1, 10, 10, "Old", "OLD", 1.0, 1.0, 1.0, 0, 0, 1.0, 1.0, 1.0, 0, 0, 0, + ) + .await; + seed_cached_row( + pool, &t_new, 1, 1, 20, 20, "New", "NEW", 1.0, 1.0, 1.0, 0, 0, 1.0, 1.0, 1.0, 0, 0, 0, + ) + .await; + seed_cached_row( + pool, &t_null, 1, 1, 30, 30, "Null", "NUL", 1.0, 1.0, 1.0, 0, 0, 1.0, 1.0, 1.0, 0, 0, + 0, + ) + .await; + seed_cached_row( + pool, &t_zero, 1, 1, 40, 40, "Zero", "ZER", 1.0, 1.0, 1.0, 0, 0, 1.0, 1.0, 1.0, 0, 0, + 0, + ) + .await; + + // Old at t=1000, New at t=2000; Null stays NULL, Zero has the 0 sentinel. + for (token_hex, ts) in [(&t_old, 1_000i64), (&t_new, 2_000i64), (&t_zero, 0i64)] { + sqlx::query("UPDATE cached_token_metrics SET first_pool_ts = ? WHERE token_id = ?") + .bind(ts) + .bind(hex::decode(token_hex).expect("valid hex")) + .execute(pool) + .await + .unwrap(); + } + + // desc → newest first; NULL and 0 sentinel sort last + let v = db_list_tokens_cached(pool, 10, 0, CachedSort::FirstPoolTsDesc) + .await + .unwrap(); + assert_eq!(v.len(), 4); + assert_eq!(v[0].display_name.as_deref(), Some("New")); + assert_eq!(v[0].first_pool_ts, Some(2_000)); + assert_eq!(v[1].display_name.as_deref(), Some("Old")); + assert_eq!(v[1].first_pool_ts, Some(1_000)); + assert_eq!(v[2].display_name.as_deref(), Some("Null")); + assert_eq!(v[2].first_pool_ts, None); + assert_eq!(v[3].display_name.as_deref(), Some("Zero")); + assert_eq!(v[3].first_pool_ts, None); + + // asc → oldest first; NULL and 0 sentinel still last + let v = db_list_tokens_cached(pool, 10, 0, CachedSort::FirstPoolTsAsc) + .await + .unwrap(); + let names: Vec> = v.iter().map(|x| x.display_name.as_deref()).collect(); + assert_eq!( + names, + vec![Some("Old"), Some("New"), Some("Null"), Some("Zero")] + ); + } + // ---------- APY calculator guard (no data → 0) ---------- #[tokio::test] diff --git a/src/db/search/mod.rs b/src/db/search/mod.rs index ceac0fc..271f9d1 100644 --- a/src/db/search/mod.rs +++ b/src/db/search/mod.rs @@ -235,7 +235,7 @@ pub async fn db_search_tokens_cached( price_now, price_24h, price_7d, change_24h_bp, change_7d_bp, display_name, display_symbol, price_now_usd, price_24h_usd, price_7d_usd, change_24h_usd_bp, change_7d_usd_bp, - apy_30d_bp, score_rank, bcmr_json, bcmr_well_known_json + apy_30d_bp, first_pool_ts, score_rank, bcmr_json, bcmr_well_known_json FROM cached_token_metrics {where_sql} {order_sql} @@ -292,6 +292,10 @@ pub async fn db_search_tokens_cached( let change_24h_usd_bp: Option = row.get("change_24h_usd_bp"); let change_7d_usd_bp: Option = row.get("change_7d_usd_bp"); let apy_30d_bp: Option = row.get("apy_30d_bp"); + // 0 is the "not backfilled yet" sentinel — treat it as unknown. + let first_pool_ts: Option = row + .get::, _>("first_pool_ts") + .filter(|&ts| ts != 0); let bcmr_json_str: Option = row.get("bcmr_json"); let bcmr_wk_json_str: Option = row.get("bcmr_well_known_json"); let bcmr: Option = bcmr_json_str.and_then(|s| serde_json::from_str(&s).ok()); @@ -322,6 +326,7 @@ pub async fn db_search_tokens_cached( change_24h_usd_bp, change_7d_usd_bp, apy_30d_bp, + first_pool_ts, }); } diff --git a/src/db/search/test.rs b/src/db/search/test.rs index f97b016..ee105ac 100644 --- a/src/db/search/test.rs +++ b/src/db/search/test.rs @@ -395,6 +395,77 @@ mod tests { assert_eq!(names, vec![Some("HighUSD"), Some("MidUSD"), Some("LowUSD")]); } + #[tokio::test] + async fn test_db_search_tokens_cached_first_pool_ts_sort_nulls_last() { + let mock = mock_db_pool(|pool: SqlitePool| async move { + cauldron_prepare_tables(&pool).await; + bcmr_prepare_tables(&pool).await; + crc20_prepare_tables(&pool).await; + setup_cached_tables(&pool).await; + }) + .await; + + let t_old = TokenID::from_byte_array([0x66; 32]).to_string(); + let t_new = TokenID::from_byte_array([0x77; 32]).to_string(); + let t_null = TokenID::from_byte_array([0x88; 32]).to_string(); + + for (token_hex, name, sym) in [ + (&t_old, "Old", "OLD"), + (&t_new, "New", "NEW"), + (&t_null, "Null", "NUL"), + ] { + seed_cached_row_simple( + &mock.cauldron_w, + token_hex, + name, + sym, + 1_000, + 100, + 1, + 1, + 1.0, + 1.0, + 1.0, + 0, + 0, + 1.0, + 1.0, + 1.0, + 0, + 0, + 0, + ) + .await; + } + + // Old at t=1000, New at t=2000; Null has no first_pool_ts. + for (token_hex, ts) in [(&t_old, 1_000i64), (&t_new, 2_000i64)] { + sqlx::query("UPDATE cached_token_metrics SET first_pool_ts = ? WHERE token_id = ?") + .bind(ts) + .bind(hex_to_blob(token_hex)) + .execute(&mock.cauldron_w) + .await + .unwrap(); + } + + // first_pool_ts desc → newest first, NULL last + let v = db_search_tokens_cached(&mock.cauldron_r, "", CachedSort::FirstPoolTsDesc, 10, 0) + .await + .unwrap(); + let names: Vec> = v.iter().map(|x| x.display_name.as_deref()).collect(); + assert_eq!(names, vec![Some("New"), Some("Old"), Some("Null")]); + assert_eq!(v[0].first_pool_ts, Some(2_000)); + assert_eq!(v[1].first_pool_ts, Some(1_000)); + assert_eq!(v[2].first_pool_ts, None); + + // first_pool_ts asc → oldest first, NULL still last + let v = db_search_tokens_cached(&mock.cauldron_r, "", CachedSort::FirstPoolTsAsc, 10, 0) + .await + .unwrap(); + let names: Vec> = v.iter().map(|x| x.display_name.as_deref()).collect(); + assert_eq!(names, vec![Some("Old"), Some("New"), Some("Null")]); + } + #[tokio::test] // Ensure that we are NOT looking for token_id when using incomplete hex. async fn test_search_token_by_inexact_hex_id() { diff --git a/src/rpc/tokens.rs b/src/rpc/tokens.rs index 24558af..cb49e3a 100644 --- a/src/rpc/tokens.rs +++ b/src/rpc/tokens.rs @@ -75,7 +75,7 @@ pub async fn search_by_volume(search_query: &str, db: &State) -> CachedApiRe /// - q: Search query string /// - limit: Maximum number of results (default: 250) /// - offset: Pagination offset (default: 0) -/// - by: Sort field (`name`, `symbol`, `tvl`, `volume`) +/// - by: Sort field (`name`, `symbol`, `tvl`, `volume`, `first_pool_ts`) /// - order: Sort direction (`asc`, `desc`) /// /// **Response:** A direct JSON array. @@ -191,6 +191,15 @@ fn parse_cached_sort(by: Option, order: Option) -> CachedSort { } } + // First pool creation timestamp (desc = newest tokens first) + Some("first_pool_ts") => { + if desc { + CachedSort::FirstPoolTsDesc + } else { + CachedSort::FirstPoolTsAsc + } + } + // default _ => { if desc { @@ -209,7 +218,7 @@ fn parse_cached_sort(by: Option, order: Option) -> CachedSort { /// /// - limit: Maximum number of results (default: 250) /// - offset: Pagination offset (default: 0) -/// - by: Sort field (`name`, `symbol`, `tvl`, `volume`, `score`) +/// - by: Sort field (`name`, `symbol`, `tvl`, `volume`, `score`, `first_pool_ts`) /// - order: Sort direction (`asc`, `desc`) /// /// **Response:** A direct JSON array.