BitcoinCash/riftenlabs-indexer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Delete zero-tvl tokens from cached list. Allow zero-tvl search on exact tokenId

Browse source
This commit is contained in:
jakobsn 2025-09-16 09:10:54 +00:00
parent 817b94efda
commit 6f25872ef7
3 changed files with 223 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
src/db/cauldron/tokenlist/metrics_cache.rs
View file

@ -26,7 +26,7 @@ use log::{error, info, warn};
use std::collections::HashMap;
// ---------- 2 min: TVL + price_now (and price_now_usd) ----------
fn update_tvl_and_price_now(
pub fn update_tvl_and_price_now(
cauldron_conn: &Connection,
bcmr_conn: &Connection,
crc20_conn: &Connection,
@ -43,6 +43,15 @@ fn update_tvl_and_price_now(
let tx = cauldron_conn.unchecked_transaction()?;
{
// Mark all cached rows stale for this tick so we can drop zero-liquidity rows deterministically
tx.execute_batch(
r#"
UPDATE cached_token_metrics
SET tvl_sats = 0,
tvl_tokens = 0;
"#,
)?;
let mut upsert = tx.prepare(
r#"
INSERT INTO cached_token_metrics
@ -66,7 +75,8 @@ fn update_tvl_and_price_now(
)?;
for (token_id, (tvl_sats, tvl_tokens)) in tvl_by_token {
if tvl_sats == 0 {
// Skip on no liquidity
if tvl_sats == 0 && tvl_tokens == 0 {
continue;
}
@ -87,11 +97,41 @@ fn update_tvl_and_price_now(
price_now_usd,
])?;
}
// Remove rows that ended up with *no* liquidity (both sats AND tokens zero)
tx.execute(
"DELETE FROM cached_token_metrics WHERE tvl_sats = 0 AND tvl_tokens = 0",
[],
)?;
}
tx.commit()?;
Ok(())
}
pub fn recompute_score_ranks(conn: &Connection) -> anyhow::Result<()> {
conn.execute_batch(
r#"
WITH ranked AS (
SELECT token_id,
ROW_NUMBER() OVER (
ORDER BY
score DESC,
trade_volume DESC,
tvl_sats DESC,
token_id ASC
) AS rnk
FROM cached_token_metrics
WHERE tvl_sats > 0
)
UPDATE cached_token_metrics
SET score_rank = (SELECT rnk
FROM ranked
WHERE ranked.token_id = cached_token_metrics.token_id)
WHERE token_id IN (SELECT token_id FROM ranked);
"#,
)?;
Ok(())
}
// ---------- 5 min: price changes (24h/7d), score, volume + ranking ----------
pub fn update_changes_score_volume_and_ranking(
cauldron_conn: &Connection,
@ -334,17 +374,26 @@ pub fn update_changes_score_volume_and_ranking(
}
// ranking
tx.execute_batch(r#"
WITH ranked AS (
SELECT token_id,
ROW_NUMBER() OVER (ORDER BY score DESC, trade_volume DESC, token_id ASC) AS rnk
FROM cached_token_metrics
WHERE tvl_sats > 0
)
UPDATE cached_token_metrics
SET score_rank = (SELECT rnk FROM ranked WHERE ranked.token_id = cached_token_metrics.token_id)
WHERE token_id IN (SELECT token_id FROM ranked);
"#)?;
// ranking
tx.execute_batch(
r#"
WITH ranked AS (
SELECT token_id,
ROW_NUMBER() OVER (
ORDER BY
score DESC,
trade_volume DESC,
tvl_sats DESC, -- <-- same tie-breaker here
token_id ASC
) AS rnk
FROM cached_token_metrics
WHERE tvl_sats > 0
)
UPDATE cached_token_metrics
SET score_rank = (SELECT rnk FROM ranked WHERE ranked.token_id = cached_token_metrics.token_id)
WHERE token_id IN (SELECT token_id FROM ranked);
"#,
)?;
tx.commit()?;
Ok(())
}
@ -449,6 +498,13 @@ pub fn spawn_token_metrics_updater(db: DB) {
error!("❌ fast(2.5m) failed after {:.3?}: {e:?}", t0.elapsed())
}
}
// Compact ranks immediately on fast-only ticks (avoid gap after deletions)
if tick % core_every != 0 {
if let Err(e) = recompute_score_ranks(cw) {
error!("rank recompute (fast-only tick) failed: {e:?}");
}
}
}
_ => {
if let Err(e) = &cw {

127
src/db/cauldron/tokenlist/test.rs
View file

@ -5,7 +5,10 @@
#[cfg(test)]
mod tests {
use crate::db::cauldron::tokenlist::metrics_cache::update_apy_only;
use crate::db::cauldron::tokenlist::metrics_cache::{
recompute_score_ranks, update_apy_only, update_changes_score_volume_and_ranking,
update_tvl_and_price_now,
};
use crate::timeutil::time_now;
use crate::bcmr::parsedbcmr::{FileMeta, ParsedBCMR as BCMRRow, Token as BCMRToken, Uris};
@ -635,7 +638,6 @@ mod tests {
#[test]
fn core_updater_keeps_deltas_null_for_young_token() {
use crate::db::cauldron::tokenlist::metrics_cache::update_changes_score_volume_and_ranking;
let mock = mock_db_pool(|conn| setup_basic_schemas(conn));
let cw = mock.cauldron_w.get().unwrap();
let bcmr = mock.bcmr_r.get().unwrap();
@ -676,4 +678,125 @@ mod tests {
assert!(c24u.is_none());
assert!(c7du.is_none());
}
#[test]
fn fast_updater_removes_rows_with_zero_liquidity() {
let mock = mock_db_pool(|conn| setup_basic_schemas(conn));
let cw = mock.cauldron_w.get().unwrap();
let bcmr = mock.bcmr_r.get().unwrap();
let crc = mock.crc20_r.get().unwrap();
let orc = mock.oracle_r.get().unwrap();
// Seed a cached row with some tvl so it exists initially
cw.execute_batch(
r#"
INSERT INTO cached_token_metrics
(token_id, trade_volume, tvl_sats, tvl_tokens, score,
display_name, display_symbol,
price_now, price_now_usd, updated_at)
VALUES
('tok_to_remove', 0, 123, 456, 0, 'Tmp', 'TMP', 1.0, 0.5, strftime('%s','now'));
"#,
)
.unwrap();
// Sanity: row exists
let count_before: i64 = cw
.query_row(
"SELECT COUNT(*) FROM cached_token_metrics WHERE token_id='tok_to_remove'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(count_before, 1);
// Run fast updater when there are *no* pools producing TVL → row should be deleted
update_tvl_and_price_now(&cw, &bcmr, &crc, &orc).unwrap();
let count_after: i64 = cw
.query_row(
"SELECT COUNT(*) FROM cached_token_metrics WHERE token_id='tok_to_remove'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(
count_after, 0,
"token with zero TVL must be removed from cache"
);
}
#[test]
fn fast_only_tick_compacts_ranks_after_deletion() {
let mock = mock_db_pool(|conn| setup_basic_schemas(conn));
let cw = mock.cauldron_w.get().unwrap();
let bcmr = mock.bcmr_r.get().unwrap();
let crc = mock.crc20_r.get().unwrap();
let orc = mock.oracle_r.get().unwrap();
// Two tokens in cache with initial TVL and scores so they get ranks 1 & 2
let tok_keep = TokenID::from_inner([0xAA; 32]).to_hex();
let tok_drop = TokenID::from_inner([0xAB; 32]).to_hex();
cw.execute_batch(&format!(
r#"
INSERT INTO cached_token_metrics
(token_id, trade_volume, tvl_sats, tvl_tokens, score,
price_now, price_now_usd, updated_at)
VALUES
('{tok_keep}', 100, 10, 10, 1000, 1.0, 0.5, strftime('%s','now')),
('{tok_drop}', 50, 10, 10, 900, 1.0, 0.5, strftime('%s','now'));
-- initial ranks like core would do
WITH ranked AS (
SELECT token_id,
ROW_NUMBER() OVER (ORDER BY score DESC, trade_volume DESC, token_id ASC) AS rnk
FROM cached_token_metrics WHERE tvl_sats > 0
)
UPDATE cached_token_metrics
SET score_rank = (SELECT rnk FROM ranked WHERE ranked.token_id = cached_token_metrics.token_id)
WHERE token_id IN (SELECT token_id FROM ranked);
"#
)).unwrap();
// Create *real* TVL only for tok_keep so the fast updater will keep it and drop tok_drop.
// We do this by seeding minimal pool history for tok_keep so TvlByTokenVisitor sees liquidity.
let now = time_now();
let t0 = now - 60; // 1 minute ago
let t1 = now - 30; // 30 seconds ago
// sats/tokens arbitrary but non-zero
super::tests::seed_minimal_token_history(
&cw,
TokenID::from_inner([0xAA; 32]),
t0,
t1,
1_000,
500, // initial
2_000,
1_000, // second event (also drives some volume on old pool)
);
// FAST updater: zeroes all tvl in cache, re-upserts only tokens with real TVL (tok_keep),
// then deletes rows with zero liquidity (tok_drop).
update_tvl_and_price_now(&cw, &bcmr, &crc, &orc).unwrap();
// Simulate a fast-only tick: compact ranks *here* (core won't run this tick).
recompute_score_ranks(&cw).unwrap();
// Verify: only tok_keep remains and it has contiguous rank 1
let rows: Vec<(String, i64)> = {
let mut v = Vec::new();
let mut stmt = cw
.prepare("SELECT token_id, score_rank FROM cached_token_metrics ORDER BY token_id")
.unwrap();
let mut r = stmt.query([]).unwrap();
while let Some(row) = r.next().unwrap() {
v.push((row.get(0).unwrap(), row.get(1).unwrap()));
}
v
};
assert_eq!(rows.len(), 1, "only the kept token should remain");
assert_eq!(rows[0].0, tok_keep);
assert_eq!(rows[0].1, 1, "ranks should be compacted to start at 1");
}
}

36
src/db/search/mod.rs
View file

@ -154,14 +154,22 @@ pub fn search_tokens_by_volume(
crc20_conn: &Connection,
search_query: &str,
) -> Result<Vec<TokenVolumeInfo>> {
let bcmr_data = search_bcmr(bcmr_conn, search_query)?;
let crc20_data = search_crc20(crc20_conn, search_query)?;
let combined_tokens = [bcmr_data, crc20_data].concat();
// Collect BCMR + CRC20 matches (by text or exact ID)
let mut combined_tokens = [
search_bcmr(bcmr_conn, search_query)?,
search_crc20(crc20_conn, search_query)?,
]
.concat();
// Dedupe by token_id (case-insensitive)
combined_tokens.sort_by(|a, b| a.0.cmp(&b.0));
combined_tokens.dedup_by(|a, b| a.0.eq_ignore_ascii_case(&b.0));
// Compute last-30d volume
let mut result = token_volume(cauldron_pool.clone(), combined_tokens)?;
// Sort by volume desc
result.sort_by(|a, b| b.3.cmp(&a.3));
Ok(result)
}
@ -186,9 +194,9 @@ pub fn db_search_tokens_cached(
let order_sql = order_clause(sort);
enum Filter {
None,
Id(String),
Pat(String),
None, // default list
Id(String), // exact 32-byte hex token_id
Pat(String), // name/symbol LIKE
}
let mut where_sql = String::new();
@ -197,9 +205,11 @@ pub fn db_search_tokens_cached(
if q.is_empty() {
Filter::None
} else if TokenID::from_hex(q).is_ok() {
// exact token_id lookup (allow zero TVL)
where_sql.push_str("WHERE token_id = :id");
Filter::Id(q.to_lowercase())
} else {
// name/symbol search (hide zero TVL)
where_sql.push_str(
"WHERE (display_name LIKE :pat COLLATE NOCASE OR display_symbol LIKE :pat COLLATE NOCASE)"
);
@ -207,6 +217,18 @@ pub fn db_search_tokens_cached(
}
};
// TVL gate:
// - default list (None): hide zero TVL
// - name/symbol LIKE (Pat): hide zero TVL
// - exact id (Id): DO NOT gate (allow zero TVL)
if matches!(filter, Filter::None | Filter::Pat(_)) {
if where_sql.is_empty() {
where_sql.push_str("WHERE tvl_sats > 0");
} else {
where_sql.push_str(" AND tvl_sats > 0");
}
}
let sql = format!(
"SELECT
token_id,