riftenlabs-indexer/src/bcmr/mod.rs
Dagur Valberg Johannsson f421327a5d
Switch to tokio and sqlx
This makes the application fully async; freeing web server threads to
handle new connections when waiting on SQL queries.

Additionally sqlx will allow easier move to a different database if
needed in the future.

Includes some SQL optimizations as well (slow queries more easliy identifiable
with sqlx).
2026-02-18 10:16:51 +01:00

679 lines
23 KiB
Rust

// Copyright (C) 2024-2026 Whiterun LLC
//
// This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later.
// A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html
use std::convert::TryInto;
use anyhow::Result;
#[cfg(test)]
use bitcoin_hashes::hex::FromHex;
use bitcoin_hashes::hex::ToHex;
#[cfg(test)]
use bitcoin_hashes::Hash;
use bitcoincash::{BlockHash, Script, TokenID, Transaction, Txid};
use log::debug;
use riftenlabs_defi::chainutil::{compute_outpoint_hash, read_push_from_script, OutPointHash};
use serde::{Deserialize, Serialize, Serializer};
use sqlx::{Row, SqlitePool};
use crate::{
db::bcmr::{insert_authheader, AuthChainEntry},
utiltoken::is_genesis_tx,
};
pub mod bcmrdownloader;
pub mod parsedbcmr;
pub mod utilurl;
pub mod wellknowndowloader;
pub const BCMR_PREFIX: &[u8] = &[
0x6a, // OP_RETURN
0x04, // PUSH 4
0x42, // B
0x43, // C
0x4d, // M
0x52, // R
];
// Custom function to serialize [u8; 32] as a hex string
fn as_hex<S>(bytes: &[u8; 32], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&hex::encode(bytes))
}
#[derive(Debug, Serialize, Deserialize)]
#[allow(clippy::upper_case_acronyms)]
pub struct BCMR {
#[serde(serialize_with = "as_hex")]
pub hash: [u8; 32],
pub uris: Vec<String>,
#[serde(skip)]
pub op_return: Vec<u8>,
}
async fn find_parent_auth_entries(
pool: &SqlitePool,
tx: &Transaction,
) -> Result<Vec<AuthChainEntry>> {
use crate::db::blob::{FromBlob, ToBlob};
let mut parents = Vec::new();
for vin in &tx.input {
if vin.previous_output.vout != 0 {
continue;
}
let prev = &vin.previous_output;
let prev_utxo: OutPointHash = compute_outpoint_hash(&prev.txid, prev.vout);
let rows = sqlx::query(
"SELECT token_id, txid, height, bcmr_data, utxo
FROM auth_chain_entry
WHERE utxo = ?",
)
.bind(prev_utxo.to_blob())
.fetch_all(pool)
.await?;
for row in rows {
let token_blob: Vec<u8> = row.get(0);
let txid_blob: Vec<u8> = row.get(1);
let height: i64 = row.get(2);
let bcmr_data_hex: Option<String> = row.get(3);
let utxo_blob: Vec<u8> = row.get(4);
let token_id = TokenID::from_blob(&token_blob)?;
let txid = Txid::from_blob(&txid_blob)?;
let utxo = OutPointHash::from_blob(&utxo_blob)?;
let bcmr_data = match bcmr_data_hex {
Some(hex_str) => Some(hex::decode(&hex_str)?),
None => None,
};
parents.push(AuthChainEntry {
utxo,
token_id,
txid,
height: height as usize,
bcmr_data,
});
}
}
Ok(parents)
}
pub fn parse_bcmr_from_opreturn(bcmr_op_return: &Script) -> Option<BCMR> {
let iter = bcmr_op_return[6..].iter();
let (iter, hash) = read_push_from_script(iter).ok()?;
let hash: [u8; 32] = hash.and_then(|h| h.try_into().ok())?;
let mut uris: Vec<String> = vec![];
let mut uri_element;
let mut uri_iter = iter;
loop {
(uri_iter, uri_element) = match read_push_from_script(uri_iter) {
Ok(r) => r,
Err(_) => break,
};
match uri_element {
Some(url) => {
if let Ok(uri) = String::from_utf8(url) {
uris.push(uri)
}
}
None => break,
}
}
Some(BCMR {
hash,
uris,
op_return: bcmr_op_return.to_bytes(),
})
}
pub fn parse_bcmr(tx: &Transaction) -> Option<BCMR> {
let bcmr_op_return = tx
.output
.iter()
.find(|o| o.script_pubkey.as_bytes().starts_with(BCMR_PREFIX))?;
parse_bcmr_from_opreturn(&bcmr_op_return.script_pubkey)
}
pub async fn index_bcmr(
pool: &SqlitePool,
blockhash: &BlockHash,
sorted: Vec<Transaction>, // TTOR sorted!
) -> Result<usize> {
let mut inserts = 0usize;
for tx in sorted.iter() {
// 1) Genesis path: start of auth chain (height 0)
if let Some(token_id) = is_genesis_tx(tx) {
let bcmr = parse_bcmr(tx);
let txid = tx.txid();
let utxo = compute_outpoint_hash(&txid, 0);
debug!(
"Found token genesis for {} in {}; has bcmr: {}",
token_id.to_hex(),
txid.to_hex(),
bcmr.is_some()
);
insert_authheader(
pool,
&utxo,
blockhash,
&txid,
&token_id,
0,
bcmr.map(|b| b.op_return),
)
.await?;
inserts += 1;
continue;
}
let parents = find_parent_auth_entries(pool, tx).await?;
if parents.is_empty() {
continue;
}
// 2) Update path: if any input spends a known auth utxo, extend that chain
let bcmr = parse_bcmr(tx);
let txid = tx.txid();
let new_utxo = compute_outpoint_hash(&txid, 0);
for parent in parents {
let new_height = parent.height + 1;
debug!(
"Found authheader update {} in {}; has bcmr: {}; new height: {}, utxo: {}",
parent.token_id.to_hex(),
txid.to_hex(),
bcmr.is_some(),
new_height,
new_utxo.to_hex()
);
insert_authheader(
pool,
&new_utxo,
blockhash,
&txid,
&parent.token_id,
new_height,
bcmr.as_ref().map(|b| b.op_return.clone()),
)
.await?;
inserts += 1;
}
}
Ok(inserts)
}
#[tokio::test]
async fn multi_parent_auth_update_creates_entries_for_all_tokens() {
use crate::db::bcmr::prepare_tables;
use bitcoincash::{OutPoint, PackedLockTime, Sequence, TxIn, TxOut};
let pool = SqlitePool::connect("sqlite::memory:").await.unwrap();
prepare_tables(&pool).await;
let bh = BlockHash::all_zeros();
// Two fake token IDs
let token_a =
TokenID::from_hex("0101010101010101010101010101010101010101010101010101010101010101")
.unwrap();
let token_b =
TokenID::from_hex("0202020202020202020202020202020202020202020202020202020202020202")
.unwrap();
// Two fake previous txids (current heads for A and B)
let prev_txid_a =
Txid::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap();
let prev_txid_b =
Txid::from_hex("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb").unwrap();
// Their auth-head UTXOs (vout 0)
let utxo_a = compute_outpoint_hash(&prev_txid_a, 0);
let utxo_b = compute_outpoint_hash(&prev_txid_b, 0);
// Insert both auth heads at height 0
insert_authheader(&pool, &utxo_a, &bh, &prev_txid_a, &token_a, 0, None)
.await
.unwrap();
insert_authheader(&pool, &utxo_b, &bh, &prev_txid_b, &token_b, 0, None)
.await
.unwrap();
// Build a single tx that spends BOTH auth heads
let input_a = TxIn {
previous_output: OutPoint {
txid: prev_txid_a,
vout: 0,
},
script_sig: Script::new(),
sequence: Sequence(0xFFFF_FFFF),
..Default::default()
};
let input_b = TxIn {
previous_output: OutPoint {
txid: prev_txid_b,
vout: 0,
},
script_sig: Script::new(),
sequence: Sequence(0xFFFF_FFFF),
..Default::default()
};
let output0 = TxOut {
value: 0,
script_pubkey: Script::new(),
..Default::default()
};
let update_tx = Transaction {
version: 2,
lock_time: PackedLockTime(0),
input: vec![input_a, input_b],
output: vec![output0],
};
let update_txid = update_tx.txid();
// Index this one tx as if it's in a block
index_bcmr(&pool, &bh, vec![update_tx]).await.unwrap();
// ---- ASSERTIONS ----
// Expect 2 auth_chain_entry rows for this tx (one per token)
{
use crate::db::blob::ToBlob;
let row: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM auth_chain_entry WHERE txid = ?")
.bind(update_txid.to_blob())
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
row.0, 2,
"Expected 2 auth_chain_entry rows for multi-parent auth update tx"
);
}
// Expect each token to have height 1 in this tx
{
use crate::db::blob::{blob_to_display_hex, ToBlob};
let rows = sqlx::query(
"SELECT token_id, height FROM auth_chain_entry WHERE txid = ? ORDER BY token_id",
)
.bind(update_txid.to_blob())
.fetch_all(&pool)
.await
.unwrap();
let mut seen: Vec<(String, i64)> = Vec::new();
for row in rows {
let token_blob: Vec<u8> = row.get(0);
let t = blob_to_display_hex::<TokenID>(&token_blob).unwrap();
let h: i64 = row.get(1);
seen.push((t, h));
}
assert_eq!(seen.len(), 2, "expected 2 rows for update tx");
seen.sort_by(|a, b| a.0.cmp(&b.0));
assert_eq!(seen[0].0, token_a.to_hex());
assert_eq!(seen[0].1, 1);
assert_eq!(seen[1].0, token_b.to_hex());
assert_eq!(seen[1].1, 1);
}
}
#[tokio::test]
async fn find_parent_auth_entries_reads_bcmr_data_from_text_column() {
// Regression test: bcmr_data is stored as hex-encoded TEXT, not BLOB.
// This test ensures find_parent_auth_entries correctly decodes it.
use crate::db::bcmr::prepare_tables;
use bitcoincash::{OutPoint, PackedLockTime, Sequence, TxIn, TxOut};
let pool = SqlitePool::connect("sqlite::memory:").await.unwrap();
prepare_tables(&pool).await;
let bh = BlockHash::all_zeros();
let token_id =
TokenID::from_hex("0101010101010101010101010101010101010101010101010101010101010101")
.unwrap();
let prev_txid =
Txid::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap();
let utxo = compute_outpoint_hash(&prev_txid, 0);
// Sample BCMR OP_RETURN data (the actual bytes that would be in the script)
let bcmr_op_return: Vec<u8> = vec![
0x6a, // OP_RETURN
0x04, 0x42, 0x43, 0x4d, 0x52, // PUSH "BCMR"
0x20, // PUSH 32 bytes (hash)
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e,
0x1f, 0x20,
];
// Insert auth entry WITH bcmr_data
insert_authheader(
&pool,
&utxo,
&bh,
&prev_txid,
&token_id,
0,
Some(bcmr_op_return.clone()),
)
.await
.unwrap();
// Build a tx that spends the auth head
let spending_tx = Transaction {
version: 2,
lock_time: PackedLockTime(0),
input: vec![TxIn {
previous_output: OutPoint {
txid: prev_txid,
vout: 0,
},
script_sig: Script::new(),
sequence: Sequence(0xFFFF_FFFF),
..Default::default()
}],
output: vec![TxOut {
value: 0,
script_pubkey: Script::new(),
..Default::default()
}],
};
// This should NOT panic - it exercises reading bcmr_data from TEXT column
let parents = find_parent_auth_entries(&pool, &spending_tx).await.unwrap();
assert_eq!(parents.len(), 1, "should find one parent entry");
assert_eq!(parents[0].token_id, token_id);
assert_eq!(parents[0].height, 0);
// Verify bcmr_data was correctly decoded from hex TEXT
let retrieved_bcmr = parents[0]
.bcmr_data
.as_ref()
.expect("bcmr_data should exist");
assert_eq!(
retrieved_bcmr, &bcmr_op_return,
"bcmr_data should match original bytes after hex decode"
);
}
#[cfg(test)]
mod tests {
use bitcoincash::{consensus::deserialize, Block};
use crate::utiltx::ttor_sorted_kahn;
use super::*;
use crate::db::blob::ToBlob;
use electrum_client_netagnostic::{Client, ElectrumApi, Param};
use std::collections::HashMap;
#[test]
fn test_parse_bcmr() {
let tx_hex = "0200000001925bd92e424c0f0bc290a794f491abf19e61b6dcdec7e70747fcb54682fc9bb700000000644181707613f45069c8df981b2ef0cbd05158d5623602fac3ae2e13f69d05fa356c278f2f3a4e34f33a79e0f8375abd35051150ec024b3625807cea185528a5de9e412103b4680dffa1e34b3bfdb024fd0a8498eb24f59ba5ead34acfb74f9d8549db6f3a0000000003e8030000000000003eef925bd92e424c0f0bc290a794f491abf19e61b6dcdec7e70747fcb54682fc9bb710fdf40176a914bd4c90f2c64743fc0d3ea6a14973a5d628260b7388ac0000000000000000456a0442434d5220c705cc90a56ac7ef9a15ef90ebbc8ba7e60e4c622e5464d52d8baf7887949fcc1d736f636b2e6361756c64726f6e2e71756573742f62636d722e6a736f6ed9210000000000001976a914bd4c90f2c64743fc0d3ea6a14973a5d628260b7388ac00000000";
let tx: Transaction = deserialize(&hex::decode(tx_hex).unwrap()).unwrap();
let result = parse_bcmr(&tx);
assert!(result.is_some());
let bcmr = result.unwrap();
assert_eq!(
hex::encode(bcmr.hash),
"c705cc90a56ac7ef9a15ef90ebbc8ba7e60e4c622e5464d52d8baf7887949fcc"
);
assert_eq!(bcmr.uris.len(), 1);
assert_eq!(bcmr.uris[0], "sock.cauldron.quest/bcmr.json");
}
#[test]
fn invalid_bcmr_hash() {
// txid 4e44c45cf1fb956fe4752c6dfa9637f29cf18e9251e20454dd65dbcc298617b6
let tx_hex = "0200000002020056e81cda3b55f6b912acf05131e4f50594dc76702e8022d5522582f74bd40000000064414048069ab85d72e0f195e6587a4ef30360d5304a7d24995e21f8bb3d9c9c52eb5d1408a3a4754fad8da6593dc65fd248e507fe1620004bcb87f3237fcea9679e4121028bb6fdf7137233695237a6fa538827afb4f7490bad82215098b40dc3d9c645fc00000000020056e81cda3b55f6b912acf05131e4f50594dc76702e8022d5522582f74bd4010000006441d501d321b457b2622388efc9290245e7021f082d2914bf1883184ea29f546a2b08d74537f848fba17d41e44ad6f9d6949b44670b379635b00fe69caa5236e47f4121028bb6fdf7137233695237a6fa538827afb4f7490bad82215098b40dc3d9c645fc0000000003e80300000000000042ef020056e81cda3b55f6b912acf05131e4f50594dc76702e8022d5522582f74bd431ff00b402869d7e0100a9147ca730ccf0ba8c552bae1287d01a9e7e5590f0fa870000000000000000d56a0442434d5240646331303939346363323530363731643830383737626639356131613136616164356431343033623365623263373136346366623231383532323363353162664c506e667473746f726167652e6c696e6b2f697066732f6261666b726569673463636d757a7173716d346f796262333337666e627566766b32786975616f7a36776c64726d746833656763736570637278343b6261666b726569673463636d757a7173716d346f796262333337666e627566766b32786975616f7a36776c64726d74683365676373657063727834bb0f0000000000001976a914d948abf743a76f472703b6a91437854dc1d7ced588ac00000000";
let tx: Transaction = deserialize(&hex::decode(tx_hex).unwrap()).unwrap();
let result = parse_bcmr(&tx);
// The hash value in this BCMR is 64 bytes. Not a valid sha256 hash.
assert!(result.is_none());
}
#[tokio::test]
#[ignore] // Integration test - requires network access. Run with `cargo test moria_issue --ignored`
async fn moria_issue() {
// Test that the indexer is able to follow the auth chain for MORIA token (part of issue #15).
// Map: block height -> expected txid of BCMR auth update transaction
let blocks_with_update: HashMap<u32, &str> = [
(
897958,
"06ef2e1f62b1efbf8a59476ae4f6d642a7eaf85a19d015909639211a7dec6a31",
),
(
897958,
"d9cc6381823202f7c6df06c90be9748e51912f57a5d1475ac3ad5f627093e507",
),
(
897957,
"6b76cfdadd3b70aa5439d1f2d4cbfc49d52b59ca45e3a3c353be7318daa105a2",
),
(
897957,
"17f6c81f4c6a95a14e542de8d9c77b292a16ee1d03c40f2fdcb9ab5043ee318b",
), // genesis
]
.iter()
.cloned()
.collect();
let client = Client::new("tcp://rostrum.cauldron.quest:50001").unwrap();
let pool = sqlx::SqlitePool::connect("sqlite::memory:").await.unwrap();
crate::db::bcmr::prepare_tables(&pool).await;
// Process blocks in ascending order (lowest to highest)
let mut sorted_blocks: Vec<_> = blocks_with_update.iter().collect();
sorted_blocks.sort_by(|a, b| a.0.cmp(b.0));
for (block_height, expected_hash) in sorted_blocks {
let block = client
.raw_call("blockchain.block.get", vec![Param::U32(*block_height)])
.unwrap();
let block: Block = deserialize(&hex::decode(block.as_str().unwrap()).unwrap()).unwrap();
let block_hash = block.block_hash();
let sorted_txs = ttor_sorted_kahn(block.txdata);
let expected_txid = *expected_hash;
index_bcmr(&pool, &block_hash, sorted_txs).await.unwrap();
let txid_blob = Txid::from_hex(expected_txid).unwrap().to_blob();
let row: (i64,) =
sqlx::query_as("SELECT COUNT(*) FROM auth_chain_entry WHERE txid = ?")
.bind(&txid_blob)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
row.0, 1,
"Expected 1 BCMR auth update for txid: {} in block {}",
expected_txid, block_height
);
}
}
#[tokio::test]
#[ignore] // Integration test - requires network access. Run with `cargo test furu_issue --ignored`
async fn furu_issue() {
// Test that the indexer is able to follow the auth chain for FURU token (part of issue #15).
// Map: block height -> expected txid of BCMR auth update transaction
let blocks_with_update: HashMap<u32, &str> = [
(
857450,
"c2cb85935d09e34f12b8b61e93efffc49c4f73a243748e53317ce6cd208de3dc",
),
(
857273,
"46cf4038b461ebbc8fb4268e74c8ec18e0beac0290ed91b7c2d2a0ddfe723c08",
),
(
819537,
"7c54400ebe360902acaaf7696fe0c0eee4f6dc5843425fbf335154bbf2445f5a",
),
(
817442,
"b5067f71ae01dd382c126c18f07f602586911f81cfaae442dfac228886af1b66",
),
(
817438,
"5e3915d6ab19c7389a531c7e8144e6d286f48bbc2db48588c27975047ff400aa",
), // genesis
]
.iter()
.cloned()
.collect();
let client = Client::new("tcp://rostrum.cauldron.quest:50001").unwrap();
let pool = sqlx::SqlitePool::connect("sqlite::memory:").await.unwrap();
crate::db::bcmr::prepare_tables(&pool).await;
// Process blocks in ascending order (lowest to highest)
let mut sorted_blocks: Vec<_> = blocks_with_update.iter().collect();
sorted_blocks.sort_by(|a, b| a.0.cmp(b.0));
for (block_height, expected_hash) in sorted_blocks {
let block = client
.raw_call("blockchain.block.get", vec![Param::U32(*block_height)])
.unwrap();
let block: Block = deserialize(&hex::decode(block.as_str().unwrap()).unwrap()).unwrap();
let block_hash = block.block_hash();
let sorted_txs = ttor_sorted_kahn(block.txdata);
let expected_txid = *expected_hash;
index_bcmr(&pool, &block_hash, sorted_txs).await.unwrap();
let txid_blob = Txid::from_hex(expected_txid).unwrap().to_blob();
let row: (i64,) =
sqlx::query_as("SELECT COUNT(*) FROM auth_chain_entry WHERE txid = ?")
.bind(&txid_blob)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
row.0, 1,
"Expected 1 BCMR auth update for txid: {} in block {}",
expected_txid, block_height
);
}
}
#[tokio::test]
#[ignore] // Integration test - requires network access. Run with `cargo test olando_issue --ignored`
async fn olando_issue() {
// Test that the indexer is able to follow the auth chain for OLANDO token (part of issue #15).
// Map: block height -> expected txid of BCMR auth update transaction
let blocks_with_update: HashMap<u32, &str> = [
(
919723,
"1987b8c8114c2cf8492467da319a8856e81f49345f6385e6992cd820871bef87",
),
(
917546,
"6c422e57f64a2c21d3e9da9161e483393da17ee1a0d8aea3ebf9e6c944ddf473",
),
(
911009,
"de926cb4c086ce4c0cd79379a53149c54a21f43ccab322bef8429d60cd53a6a1",
),
(
910728,
"536e199357a02fe9ea071cf7d0df4431911fd2ad170867659e796f053f7e6045",
),
(
908242,
"0370d0ae6f1f0c7b49b5c3f7e271a1ebf4daef9d91450f2316e878a0a78d6d08",
),
(
906927,
"68a0067153db9925560d7adb956295b41de958058ccd78d5c1adfdf4e0e70ed9",
),
(
906218,
"20e8074d404e5e80c6d8cbe30d51c754ab86ae0b58bc8ec17f12afdb6380195d",
),
(
887082,
"6a806f2e01e11a2a4000bb736ffb5570dd55412582088abe95fce700a17a3922",
),
(
887074,
"7c71bbe3e3f52c5bf9965f454e7f87b86f097ecfec0705360e9284f38c360d65",
), // genesis
]
.iter()
.cloned()
.collect();
let client = Client::new("tcp://rostrum.cauldron.quest:50001").unwrap();
let pool = sqlx::SqlitePool::connect("sqlite::memory:").await.unwrap();
crate::db::bcmr::prepare_tables(&pool).await;
// Process blocks in ascending order (lowest to highest)
let mut sorted_blocks: Vec<_> = blocks_with_update.iter().collect();
sorted_blocks.sort_by(|a, b| a.0.cmp(b.0));
for (block_height, expected_hash) in sorted_blocks {
let block = client
.raw_call("blockchain.block.get", vec![Param::U32(*block_height)])
.unwrap();
let block: Block = deserialize(&hex::decode(block.as_str().unwrap()).unwrap()).unwrap();
let block_hash = block.block_hash();
let sorted_txs = ttor_sorted_kahn(block.txdata);
let expected_txid = *expected_hash;
index_bcmr(&pool, &block_hash, sorted_txs).await.unwrap();
let txid_blob = Txid::from_hex(expected_txid).unwrap().to_blob();
let row: (i64,) =
sqlx::query_as("SELECT COUNT(*) FROM auth_chain_entry WHERE txid = ?")
.bind(&txid_blob)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
row.0, 1,
"Expected 1 BCMR auth update for txid: {} in block {}",
expected_txid, block_height
);
}
}
}