diff --git a/.gitignore b/.gitignore index 9847ea6..fc63431 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ /target *.db +*.db-shm +*.db-wal /apidoc +.DS_Store +*.swp diff --git a/internal/config_specification.toml b/internal/config_specification.toml index 0cb26d5..e7279b9 100644 --- a/internal/config_specification.toml +++ b/internal/config_specification.toml @@ -1,5 +1,11 @@ +[[param]] +name = "network" +type = "String" +doc = "Network to use: 'mainnet' or 'chipnet' (default: mainnet)" +default = "\"mainnet\".to_string()" + [[param]] name = "rostrum_addr" type = "String" -doc = "Rostrum address and TCP port (default: 127.0.0.1:50001)" -default = "\"127.0.0.1:50001\".to_string()" +doc = "Rostrum address and TCP port. If not specified, defaults to 127.0.0.1:50001 for mainnet or 127.0.0.1:64001 for chipnet" +default = "\"\".to_string()" diff --git a/src/bcmr/mod.rs b/src/bcmr/mod.rs index 4980f20..6cf7541 100644 --- a/src/bcmr/mod.rs +++ b/src/bcmr/mod.rs @@ -538,8 +538,8 @@ mod tests { } #[test] + #[ignore] fn olando_issue() { - #[ignore] // Integration test - requires network access. Run with `cargo test olando_issue --ignored` // Test that the indexer is able to follow the auth chain for OLANDO token (part of issue #15). diff --git a/src/db/cauldron/tokenlist/token_utils.rs b/src/db/cauldron/tokenlist/token_utils.rs index 1c87f90..b337608 100644 --- a/src/db/cauldron/tokenlist/token_utils.rs +++ b/src/db/cauldron/tokenlist/token_utils.rs @@ -118,10 +118,7 @@ pub fn compute_score(tvl_sats: u64, vol_30d: u64) -> i64 { } // Convert safely; fall back to MAX on unexpected parse failure - match score_big.to_string().parse::() { - Ok(v) => v, - Err(_) => i64::MAX, - } + score_big.to_string().parse::().unwrap_or(i64::MAX) } pub fn resolve_decimals(bcmr_conn: &Connection, crc20_conn: &Connection, token_id: &str) -> u32 { // On-chain BCMR (latest by height) diff --git a/src/db/init.rs b/src/db/init.rs new file mode 100644 index 0000000..79224aa --- /dev/null +++ b/src/db/init.rs @@ -0,0 +1,175 @@ +// Copyright (C) 2024 Riften Labs AS +// +// This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later. +// A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html + +use anyhow::Result; +use log::info; +use rusqlite::OpenFlags; +use std::path::Path; +use std::sync::Arc; +use std::thread::sleep; +use std::time::{Duration, Instant}; + +use super::{DBPool, DB}; +use crate::db::bcmr::prepare_tables as bcmr_prepare_tables; +use crate::db::cauldron::config::check_db_version; +use crate::db::cauldron::prepare_tables as cauldron_prepare_tables; +use crate::db::crc20::prepare_tables as crc20_prepare_tables; +use crate::db::oracle::prepare_tables as oracle_prepare_tables; + +/// Initialize a database connection with proper settings +pub fn db_initialize(c: &mut rusqlite::Connection) -> Result<(), rusqlite::Error> { + // Set busy timeout for normal queries. + c.busy_timeout(Duration::from_secs(30))?; + c.execute("PRAGMA foreign_keys=1;", [])?; + + // PRAGMA journal_mode may not honor busy_timeout, so retry manually. + let start = Instant::now(); + let timeout = Duration::from_secs(30); + loop { + match c.execute_batch("PRAGMA journal_mode=WAL;") { + Ok(_) => break, + Err(rusqlite::Error::SqliteFailure(err, _)) + if err.code == rusqlite::ErrorCode::DatabaseBusy => + { + if start.elapsed() >= timeout { + return Err(rusqlite::Error::SqliteFailure( + err, + Some("database locked after retries".into()), + )); + } + sleep(Duration::from_millis(100)); + } + Err(e) => return Err(e), + } + } + Ok(()) +} + +/// Verify that database settings are correct +pub fn db_sanity_check(c: &rusqlite::Connection) -> rusqlite::Result<()> { + // Verify that foreign_keys are enabled + let mut stmt = c.prepare("PRAGMA foreign_keys;")?; + let foreign_keys_enabled: i32 = stmt.query_row([], |row| row.get(0))?; + if foreign_keys_enabled != 1 { + panic!( + "Foreign key enforcement is not enabled for the connection. (result: {foreign_keys_enabled})" + ); + } + Ok(()) +} + +/// Create read and write database pools for a given database path +fn create_db_pool(db_path: &str) -> (bool, DBPool, DBPool) { + let db_exists = Path::new(db_path).exists(); + info!("Initializing connection to {db_path}"); + + let write_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path) + .with_flags(OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE) + .with_init(db_initialize); + + let write_pool = + Arc::new(r2d2::Pool::new(write_manager).expect("Failed to initialize database")); + + let read_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path) + .with_flags(OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_init(db_initialize); + + let read_pool = Arc::new(r2d2::Pool::new(read_manager).expect("Failed to initialize database")); + + db_sanity_check(&read_pool.get().unwrap()).expect("db sanity check failed"); + db_sanity_check(&write_pool.get().unwrap()).expect("db sanity check failed"); + + (db_exists, write_pool, read_pool) +} + +/// Get the database directory path based on network +pub fn get_db_directory(network: &str) -> &'static str { + if network.to_lowercase() == "chipnet" { + "chipnet" + } else { + "." + } +} + +/// Create network-specific directory if needed +pub fn ensure_db_directory(db_dir: &str) -> Result<()> { + if db_dir != "." { + std::fs::create_dir_all(db_dir).map_err(|e| { + anyhow::anyhow!("Failed to create database directory '{}': {}", db_dir, e) + })?; + } + Ok(()) +} + +/// Get the full database path for a filename based on the database directory +fn db_path(db_dir: &str, filename: &str) -> String { + if db_dir == "." { + filename.to_string() + } else { + format!("{}/{}", db_dir, filename) + } +} + +/// Initialize all databases and return a DB struct +pub fn initialize_databases(network: &str) -> Result { + let db_dir = get_db_directory(network); + ensure_db_directory(db_dir)?; + + // Initialize cauldron database + let (db_exists, cauldron_db_write, cauldron_db_read) = + create_db_pool(&db_path(db_dir, "cauldron.db")); + if !db_exists { + cauldron_prepare_tables( + &cauldron_db_write + .get() + .expect("failed to get sqlite connection"), + ); + } else { + // Check database version for existing database + check_db_version(&cauldron_db_read.get().unwrap()).expect("Database version check failed"); + } + + // Initialize BCMR database + let (db_exists, bcmr_db_write, bcmr_db_read) = create_db_pool(&db_path(db_dir, "bcmr.db")); + if !db_exists { + bcmr_prepare_tables( + &bcmr_db_write + .get() + .expect("failed to create sql connection"), + ); + } + + // Initialize CRC20 database + let (db_exists, crc20_db_write, crc20_db_read) = create_db_pool(&db_path(db_dir, "crc20.db")); + if !db_exists { + crc20_prepare_tables( + &crc20_db_write + .get() + .expect("failed to create sqlite crc20 connection"), + ); + } + + // Initialize oracle database + let (db_exists, oracle_db_write, oracle_db_read) = + create_db_pool(&db_path(db_dir, "oracle.db")); + if !db_exists { + oracle_prepare_tables( + &oracle_db_write + .get() + .expect("failed to create sqlite oracle connection"), + ); + } + + Ok(DB { + cauldron_w: cauldron_db_write, + cauldron_r: cauldron_db_read, + bcmr_w: bcmr_db_write, + bcmr_r: bcmr_db_read, + crc20_w: crc20_db_write, + crc20_r: crc20_db_read, + oracle_w: oracle_db_write, + oracle_r: oracle_db_read, + }) +} diff --git a/src/db/mod.rs b/src/db/mod.rs index 4cad4ae..634037d 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -8,6 +8,7 @@ use std::sync::Arc; pub mod bcmr; pub mod cauldron; pub mod crc20; +pub mod init; pub mod oracle; pub mod search; diff --git a/src/index.rs b/src/index.rs index f7ed417..1ab94a3 100644 --- a/src/index.rs +++ b/src/index.rs @@ -12,7 +12,7 @@ use bitcoin_hashes::{ hex::{FromHex, ToHex}, Hash, }; -use bitcoincash::{consensus::deserialize, Block, BlockHash, Transaction, Txid}; +use bitcoincash::{consensus::deserialize, Block, BlockHash, Network, Transaction, Txid}; use electrum_client_netagnostic::{Client, ElectrumApi, Param}; use log::{debug, info, warn}; use riftenlabs_defi::cauldron::{parse_cauldrons_from_tx, ParsedContract}; @@ -37,9 +37,9 @@ use crate::{ electrum::{electrum_fetch_mempool, electrum_get_tip, electrum_get_tx}, timeutil::time_now, utiltx::ttor_sorted_kahn, - CASHTOKEN_ACTIVATION_HEIGHT, KEY_LAST_INDEXED, + CASHTOKEN_ACTIVATION_HEIGHT, CHIPNET_START_BLOCK, KEY_LAST_INDEXED, }; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; pub fn update_mempool( cauldron_db: DBPool, @@ -137,9 +137,21 @@ pub fn index_blocks( db: DB, client: Arc>, bcmr_enabled: bool, + network: Option, ) -> Result { let (tip_header, _) = electrum_get_tip(&client.lock().unwrap())?; + // Validate network before proceeding + let start_block_hash = match network { + Some(Network::Chipnet) => BlockHash::from_hex(CHIPNET_START_BLOCK) + .map_err(|e| anyhow::anyhow!("Invalid CHIPNET_START_BLOCK: {}", e))?, + Some(Network::Bitcoin) => BlockHash::from_hex(CASHTOKEN_ACTIVATION_HEIGHT) + .map_err(|e| anyhow::anyhow!("Invalid CASHTOKEN_ACTIVATION_HEIGHT: {}", e))?, + None => BlockHash::from_hex(CASHTOKEN_ACTIVATION_HEIGHT) + .map_err(|e| anyhow::anyhow!("Invalid CASHTOKEN_ACTIVATION_HEIGHT: {}", e))?, + Some(net) => bail!("Unknown network: {:?}", net), + }; + let (block_send, block_recv) = sync_channel::>(10); // Update header chain (and undo any blocks that may have reorged away) @@ -174,8 +186,12 @@ pub fn index_blocks( let db = db_cpy; let last_indexed = config_get(&db.cauldron_r.get().unwrap(), KEY_LAST_INDEXED).unwrap(); - let last_indexed = last_indexed.unwrap_or(CASHTOKEN_ACTIVATION_HEIGHT.to_string()); - let mut last_indexed = BlockHash::from_hex(&last_indexed).unwrap(); + let mut last_indexed = if let Some(last) = last_indexed { + BlockHash::from_hex(&last).unwrap() + } else { + // No last_indexed found, use network-specific starting point + start_block_hash + }; // Check if last indexed has been orphaned loop { diff --git a/src/main.rs b/src/main.rs index 5f864ff..ab45b6a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,34 +5,30 @@ use anyhow::{bail, Result}; use bcmr::wellknowndowloader::WellKnownDownloader; -use bitcoincash::{consensus::deserialize, Block, BlockHash}; +use bitcoincash::{consensus::deserialize, Block, BlockHash, Network}; use crc20::crc20fetcher::CRC20Fetcher; -use db::{DBPool, DB}; +use db::DB; use electrum::electrum_get_tip; use electrum_client_netagnostic::{Client, ElectrumApi, Param}; use log::{error, info, warn}; use rocket::{launch, routes}; use rocket_cors::{AllowedHeaders, AllowedOrigins}; use rpc::ResponseCache; -use rusqlite::OpenFlags; use std::sync::atomic::{AtomicBool, Ordering}; use std::{ backtrace::Backtrace, collections::HashMap, - panic, - path::Path, - process, + panic, process, sync::{Arc, Mutex}, - thread::sleep, - time::{Duration, Instant}, + time::Duration, }; use stderrlog::LogLevelNum; use crate::bcmr::bcmrdownloader::BCMRDownloader; -use crate::db::cauldron::config::check_db_version; use crate::db::cauldron::header::load_all_headers; use crate::db::cauldron::tokenlist::db_utils::create_cached_token_metrics_table; use crate::db::cauldron::tokenlist::metrics_cache::spawn_token_metrics_updater; +use crate::db::init::initialize_databases; use crate::index::{index_blocks, update_mempool}; #[macro_use] @@ -49,6 +45,10 @@ const RIFTEN_LABS_GENESIS_BLOCK: &str = const CASHTOKEN_ACTIVATION_HEIGHT: &str = "000000000000000002b678c471841c3e404ec7ae9ca9c32026fe27eb6e3a1ed1"; +// Chipnet genesis +const CHIPNET_START_BLOCK: &str = + "000000001dd410c49a788668ce26751718cc797474d3152a5fc073dd44fd9f7b"; + // Last indexed block height. const KEY_LAST_INDEXED: &str = "last_indexed"; @@ -91,121 +91,51 @@ fn set_panic_hook() { })); } -fn db_initialize(c: &mut rusqlite::Connection) -> Result<(), rusqlite::Error> { - // Set busy timeout for normal queries. - c.busy_timeout(Duration::from_secs(30))?; - c.execute("PRAGMA foreign_keys=1;", [])?; - - // PRAGMA journal_mode may not honor busy_timeout, so retry manually. - let start = Instant::now(); - let timeout = Duration::from_secs(30); - loop { - match c.execute_batch("PRAGMA journal_mode=WAL;") { - Ok(_) => break, - Err(rusqlite::Error::SqliteFailure(err, _)) - if err.code == rusqlite::ErrorCode::DatabaseBusy => - { - if start.elapsed() >= timeout { - return Err(rusqlite::Error::SqliteFailure( - err, - Some("database locked after retries".into()), - )); - } - sleep(Duration::from_millis(100)); - } - Err(e) => return Err(e), - } - } - Ok(()) -} - -fn db_sanity_check(c: &rusqlite::Connection) -> rusqlite::Result<()> { - // Verify that foreign_keys are enabled - let mut stmt = c.prepare("PRAGMA foreign_keys;")?; - let foreign_keys_enabled: i32 = stmt.query_row([], |row| row.get(0))?; - if foreign_keys_enabled != 1 { - panic!( - "Foreign key enforcement is not enabled for the connection. (result: {foreign_keys_enabled})" - ); - } - Ok(()) -} - fn start_program( config: Config, ) -> Result<(DB, BCMRDownloader, WellKnownDownloader, CRC20Fetcher)> { - let create_db_pool = |db_path| -> (bool, DBPool, DBPool) { - let db_exists = Path::new(db_path).exists(); - info!("Initializing connection to {db_path}"); - - let write_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path) - .with_flags(OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE) - .with_init(db_initialize); - - let write_pool = - Arc::new(r2d2::Pool::new(write_manager).expect("Failed to initialize database")); - - let read_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path) - .with_flags(OpenFlags::SQLITE_OPEN_READ_ONLY) - .with_init(db_initialize); - - let read_pool = - Arc::new(r2d2::Pool::new(read_manager).expect("Failed to initialize database")); - - db_sanity_check(&read_pool.get().unwrap()).expect("db sanity check failed"); - db_sanity_check(&write_pool.get().unwrap()).expect("db sanity check failed"); - - (db_exists, write_pool, read_pool) + // Parse network parameter + let network = match config.network.to_lowercase().as_str() { + "mainnet" => Network::Bitcoin, + "chipnet" => Network::Chipnet, + _ => bail!( + "Invalid network '{}'. Must be 'mainnet' or 'chipnet'", + config.network + ), }; - let (db_exists, cauldron_db_write, cauldron_db_read) = create_db_pool("cauldron.db"); - if !db_exists { - db::cauldron::prepare_tables( - &cauldron_db_write - .get() - .expect("failed to get sqlite connection"), - ); + info!("Using network: {:?}", network); + + // Set default rostrum address based on network if not explicitly provided + let rostrum_addr = if config.rostrum_addr.is_empty() { + // User didn't provide --rostrum-addr, use network-specific default + match network { + Network::Chipnet => "127.0.0.1:64001".to_string(), + _ => "127.0.0.1:50001".to_string(), + } } else { - // Check database version for existing database - check_db_version(&cauldron_db_read.get().unwrap()).expect("Database version check failed"); - } - let (db_exists, bcmr_db_write, bcmr_db_read) = create_db_pool("bcmr.db"); - if !db_exists { - db::bcmr::prepare_tables( - &bcmr_db_write - .get() - .expect("failed to create sql connection"), - ); - } + // User explicitly provided a value, use it as-is + config.rostrum_addr + }; - let (db_exists, crc20_db_write, crc20_db_read) = create_db_pool("crc20.db"); - if !db_exists { - db::crc20::prepare_tables( - &crc20_db_write - .get() - .expect("failed to create sqlite crc20 connection"), - ); - } - - let (db_exists, oracle_db_write, oracle_db_read) = create_db_pool("oracle.db"); - if !db_exists { - db::oracle::prepare_tables( - &oracle_db_write - .get() - .expect("failed to create sqlite oracle connection"), - ); - } + // Initialize all databases + let network_str = match network { + Network::Bitcoin => "mainnet", + Network::Chipnet => "chipnet", + _ => "mainnet", + }; + let db = initialize_databases(network_str)?; // Create a shared flag for indexing status let indexing_in_progress = Arc::new(AtomicBool::new(false)); let client = Arc::new(Mutex::new( - match Client::new(&format!("tcp://{}", config.rostrum_addr)) { + match Client::new(&format!("tcp://{}", rostrum_addr)) { Ok(server) => server, Err(e) => { error!( "Failed to connect to {}: {}. See --help for setting a different server.", - config.rostrum_addr, e + rostrum_addr, e ); bail!(e) } @@ -221,25 +151,14 @@ fn start_program( let chain = Arc::new(Mutex::new(chain::Chain::new(genesis.header))); // initialize insert sequence for pool history - db::cauldron::pool::initialize_seq(&cauldron_db_read.get().unwrap()); + db::cauldron::pool::initialize_seq(&db.cauldron_r.get().unwrap()); info!("Loading block headers..."); - let all_headers = load_all_headers(&cauldron_db_read.get().unwrap()).unwrap(); + let all_headers = load_all_headers(&db.cauldron_r.get().unwrap()).unwrap(); info!("Initializing {} headers...", all_headers.len()); chain.lock().unwrap().load(all_headers).unwrap(); info!("Headers loaded."); - let db = DB { - cauldron_w: cauldron_db_write, - cauldron_r: cauldron_db_read, - bcmr_w: bcmr_db_write, - bcmr_r: bcmr_db_read, - crc20_w: crc20_db_write, - crc20_r: crc20_db_read, - oracle_w: oracle_db_write, - oracle_r: oracle_db_read, - }; - let db_cpy = db.clone(); let mut crc20fetcher = CRC20Fetcher::new(); @@ -263,7 +182,13 @@ fn start_program( // Initial full index let mut tip: BlockHash = loop { indexing_in_progress_clone.store(true, Ordering::Relaxed); - break match index_blocks(chain.clone(), db.clone(), client.clone(), true) { + break match index_blocks( + chain.clone(), + db.clone(), + client.clone(), + true, + Some(network), + ) { Ok(tip) => tip, Err(e) => { if e.to_string().contains("database is locked") { @@ -289,7 +214,13 @@ fn start_program( if new_tip != tip { indexing_in_progress_clone.store(true, Ordering::Relaxed); - tip = match index_blocks(chain.clone(), db.clone(), client.clone(), true) { + tip = match index_blocks( + chain.clone(), + db.clone(), + client.clone(), + true, + Some(network), + ) { Ok(t) => t, Err(e) => { warn!("Indexing block failed: {} {}", e, e.backtrace());