Add Chipnet support

.gitignore

@@ -1,3 +1,7 @@
 /target
 *.db
+*.db-shm
+*.db-wal
 /apidoc
+.DS_Store
+*.swp

internal/config_specification.toml

@@ -1,5 +1,11 @@
+[[param]]
+name = "network"
+type = "String"
+doc = "Network to use: 'mainnet' or 'chipnet' (default: mainnet)"
+default = "\"mainnet\".to_string()"
+
 [[param]]
 name = "rostrum_addr"
 type = "String"
-doc = "Rostrum address and TCP port (default: 127.0.0.1:50001)"
-default = "\"127.0.0.1:50001\".to_string()"
+doc = "Rostrum address and TCP port. If not specified, defaults to 127.0.0.1:50001 for mainnet or 127.0.0.1:64001 for chipnet"
+default = "\"\".to_string()"

src/bcmr/mod.rs

@@ -538,8 +538,8 @@ mod tests {
     }
 
     #[test]
+    #[ignore]
     fn olando_issue() {
-        #[ignore]
         // Integration test - requires network access. Run with `cargo test olando_issue --ignored`
         // Test that the indexer is able to follow the auth chain for OLANDO token (part of issue #15).
 

src/db/cauldron/tokenlist/token_utils.rs

@@ -118,10 +118,7 @@ pub fn compute_score(tvl_sats: u64, vol_30d: u64) -> i64 {
     }
 
     // Convert safely; fall back to MAX on unexpected parse failure
-    match score_big.to_string().parse::<i64>() {
-        Ok(v) => v,
-        Err(_) => i64::MAX,
-    }
+    score_big.to_string().parse::<i64>().unwrap_or(i64::MAX)
 }
 pub fn resolve_decimals(bcmr_conn: &Connection, crc20_conn: &Connection, token_id: &str) -> u32 {
     // On-chain BCMR (latest by height)

src/db/init.rs

@@ -0,0 +1,175 @@
+// Copyright (C) 2024 Riften Labs AS
+//
+// This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later.
+// A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html
+
+use anyhow::Result;
+use log::info;
+use rusqlite::OpenFlags;
+use std::path::Path;
+use std::sync::Arc;
+use std::thread::sleep;
+use std::time::{Duration, Instant};
+
+use super::{DBPool, DB};
+use crate::db::bcmr::prepare_tables as bcmr_prepare_tables;
+use crate::db::cauldron::config::check_db_version;
+use crate::db::cauldron::prepare_tables as cauldron_prepare_tables;
+use crate::db::crc20::prepare_tables as crc20_prepare_tables;
+use crate::db::oracle::prepare_tables as oracle_prepare_tables;
+
+/// Initialize a database connection with proper settings
+pub fn db_initialize(c: &mut rusqlite::Connection) -> Result<(), rusqlite::Error> {
+    // Set busy timeout for normal queries.
+    c.busy_timeout(Duration::from_secs(30))?;
+    c.execute("PRAGMA foreign_keys=1;", [])?;
+
+    // PRAGMA journal_mode may not honor busy_timeout, so retry manually.
+    let start = Instant::now();
+    let timeout = Duration::from_secs(30);
+    loop {
+        match c.execute_batch("PRAGMA journal_mode=WAL;") {
+            Ok(_) => break,
+            Err(rusqlite::Error::SqliteFailure(err, _))
+                if err.code == rusqlite::ErrorCode::DatabaseBusy =>
+            {
+                if start.elapsed() >= timeout {
+                    return Err(rusqlite::Error::SqliteFailure(
+                        err,
+                        Some("database locked after retries".into()),
+                    ));
+                }
+                sleep(Duration::from_millis(100));
+            }
+            Err(e) => return Err(e),
+        }
+    }
+    Ok(())
+}
+
+/// Verify that database settings are correct
+pub fn db_sanity_check(c: &rusqlite::Connection) -> rusqlite::Result<()> {
+    // Verify that foreign_keys are enabled
+    let mut stmt = c.prepare("PRAGMA foreign_keys;")?;
+    let foreign_keys_enabled: i32 = stmt.query_row([], |row| row.get(0))?;
+    if foreign_keys_enabled != 1 {
+        panic!(
+            "Foreign key enforcement is not enabled for the connection. (result: {foreign_keys_enabled})"
+        );
+    }
+    Ok(())
+}
+
+/// Create read and write database pools for a given database path
+fn create_db_pool(db_path: &str) -> (bool, DBPool, DBPool) {
+    let db_exists = Path::new(db_path).exists();
+    info!("Initializing connection to {db_path}");
+
+    let write_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path)
+        .with_flags(OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE)
+        .with_init(db_initialize);
+
+    let write_pool =
+        Arc::new(r2d2::Pool::new(write_manager).expect("Failed to initialize database"));
+
+    let read_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path)
+        .with_flags(OpenFlags::SQLITE_OPEN_READ_ONLY)
+        .with_init(db_initialize);
+
+    let read_pool = Arc::new(r2d2::Pool::new(read_manager).expect("Failed to initialize database"));
+
+    db_sanity_check(&read_pool.get().unwrap()).expect("db sanity check failed");
+    db_sanity_check(&write_pool.get().unwrap()).expect("db sanity check failed");
+
+    (db_exists, write_pool, read_pool)
+}
+
+/// Get the database directory path based on network
+pub fn get_db_directory(network: &str) -> &'static str {
+    if network.to_lowercase() == "chipnet" {
+        "chipnet"
+    } else {
+        "."
+    }
+}
+
+/// Create network-specific directory if needed
+pub fn ensure_db_directory(db_dir: &str) -> Result<()> {
+    if db_dir != "." {
+        std::fs::create_dir_all(db_dir).map_err(|e| {
+            anyhow::anyhow!("Failed to create database directory '{}': {}", db_dir, e)
+        })?;
+    }
+    Ok(())
+}
+
+/// Get the full database path for a filename based on the database directory
+fn db_path(db_dir: &str, filename: &str) -> String {
+    if db_dir == "." {
+        filename.to_string()
+    } else {
+        format!("{}/{}", db_dir, filename)
+    }
+}
+
+/// Initialize all databases and return a DB struct
+pub fn initialize_databases(network: &str) -> Result<DB> {
+    let db_dir = get_db_directory(network);
+    ensure_db_directory(db_dir)?;
+
+    // Initialize cauldron database
+    let (db_exists, cauldron_db_write, cauldron_db_read) =
+        create_db_pool(&db_path(db_dir, "cauldron.db"));
+    if !db_exists {
+        cauldron_prepare_tables(
+            &cauldron_db_write
+                .get()
+                .expect("failed to get sqlite connection"),
+        );
+    } else {
+        // Check database version for existing database
+        check_db_version(&cauldron_db_read.get().unwrap()).expect("Database version check failed");
+    }
+
+    // Initialize BCMR database
+    let (db_exists, bcmr_db_write, bcmr_db_read) = create_db_pool(&db_path(db_dir, "bcmr.db"));
+    if !db_exists {
+        bcmr_prepare_tables(
+            &bcmr_db_write
+                .get()
+                .expect("failed to create sql connection"),
+        );
+    }
+
+    // Initialize CRC20 database
+    let (db_exists, crc20_db_write, crc20_db_read) = create_db_pool(&db_path(db_dir, "crc20.db"));
+    if !db_exists {
+        crc20_prepare_tables(
+            &crc20_db_write
+                .get()
+                .expect("failed to create sqlite crc20 connection"),
+        );
+    }
+
+    // Initialize oracle database
+    let (db_exists, oracle_db_write, oracle_db_read) =
+        create_db_pool(&db_path(db_dir, "oracle.db"));
+    if !db_exists {
+        oracle_prepare_tables(
+            &oracle_db_write
+                .get()
+                .expect("failed to create sqlite oracle connection"),
+        );
+    }
+
+    Ok(DB {
+        cauldron_w: cauldron_db_write,
+        cauldron_r: cauldron_db_read,
+        bcmr_w: bcmr_db_write,
+        bcmr_r: bcmr_db_read,
+        crc20_w: crc20_db_write,
+        crc20_r: crc20_db_read,
+        oracle_w: oracle_db_write,
+        oracle_r: oracle_db_read,
+    })
+}

src/db/mod.rs

@@ -8,6 +8,7 @@ use std::sync::Arc;
 pub mod bcmr;
 pub mod cauldron;
 pub mod crc20;
+pub mod init;
 pub mod oracle;
 pub mod search;
 

src/index.rs

@@ -12,7 +12,7 @@ use bitcoin_hashes::{
     hex::{FromHex, ToHex},
     Hash,
 };
-use bitcoincash::{consensus::deserialize, Block, BlockHash, Transaction, Txid};
+use bitcoincash::{consensus::deserialize, Block, BlockHash, Network, Transaction, Txid};
 use electrum_client_netagnostic::{Client, ElectrumApi, Param};
 use log::{debug, info, warn};
 use riftenlabs_defi::cauldron::{parse_cauldrons_from_tx, ParsedContract};
@@ -37,9 +37,9 @@ use crate::{
     electrum::{electrum_fetch_mempool, electrum_get_tip, electrum_get_tx},
     timeutil::time_now,
     utiltx::ttor_sorted_kahn,
-    CASHTOKEN_ACTIVATION_HEIGHT, KEY_LAST_INDEXED,
+    CASHTOKEN_ACTIVATION_HEIGHT, CHIPNET_START_BLOCK, KEY_LAST_INDEXED,
 };
-use anyhow::{Context, Result};
+use anyhow::{bail, Context, Result};
 
 pub fn update_mempool(
     cauldron_db: DBPool,
@@ -137,9 +137,21 @@ pub fn index_blocks(
     db: DB,
     client: Arc<Mutex<Client>>,
     bcmr_enabled: bool,
+    network: Option<Network>,
 ) -> Result<BlockHash> {
     let (tip_header, _) = electrum_get_tip(&client.lock().unwrap())?;
 
+    // Validate network before proceeding
+    let start_block_hash = match network {
+        Some(Network::Chipnet) => BlockHash::from_hex(CHIPNET_START_BLOCK)
+            .map_err(|e| anyhow::anyhow!("Invalid CHIPNET_START_BLOCK: {}", e))?,
+        Some(Network::Bitcoin) => BlockHash::from_hex(CASHTOKEN_ACTIVATION_HEIGHT)
+            .map_err(|e| anyhow::anyhow!("Invalid CASHTOKEN_ACTIVATION_HEIGHT: {}", e))?,
+        None => BlockHash::from_hex(CASHTOKEN_ACTIVATION_HEIGHT)
+            .map_err(|e| anyhow::anyhow!("Invalid CASHTOKEN_ACTIVATION_HEIGHT: {}", e))?,
+        Some(net) => bail!("Unknown network: {:?}", net),
+    };
+
     let (block_send, block_recv) = sync_channel::<Option<(u64, u64, Block)>>(10);
 
     // Update header chain (and undo any blocks that may have reorged away)
@@ -174,8 +186,12 @@ pub fn index_blocks(
         let db = db_cpy;
 
         let last_indexed = config_get(&db.cauldron_r.get().unwrap(), KEY_LAST_INDEXED).unwrap();
-        let last_indexed = last_indexed.unwrap_or(CASHTOKEN_ACTIVATION_HEIGHT.to_string());
-        let mut last_indexed = BlockHash::from_hex(&last_indexed).unwrap();
+        let mut last_indexed = if let Some(last) = last_indexed {
+            BlockHash::from_hex(&last).unwrap()
+        } else {
+            // No last_indexed found, use network-specific starting point
+            start_block_hash
+        };
 
         // Check if last indexed has been orphaned
         loop {

src/main.rs

@@ -5,34 +5,30 @@
 
 use anyhow::{bail, Result};
 use bcmr::wellknowndowloader::WellKnownDownloader;
-use bitcoincash::{consensus::deserialize, Block, BlockHash};
+use bitcoincash::{consensus::deserialize, Block, BlockHash, Network};
 use crc20::crc20fetcher::CRC20Fetcher;
-use db::{DBPool, DB};
+use db::DB;
 use electrum::electrum_get_tip;
 use electrum_client_netagnostic::{Client, ElectrumApi, Param};
 use log::{error, info, warn};
 use rocket::{launch, routes};
 use rocket_cors::{AllowedHeaders, AllowedOrigins};
 use rpc::ResponseCache;
-use rusqlite::OpenFlags;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::{
     backtrace::Backtrace,
     collections::HashMap,
-    panic,
-    path::Path,
-    process,
+    panic, process,
     sync::{Arc, Mutex},
-    thread::sleep,
-    time::{Duration, Instant},
+    time::Duration,
 };
 use stderrlog::LogLevelNum;
 
 use crate::bcmr::bcmrdownloader::BCMRDownloader;
-use crate::db::cauldron::config::check_db_version;
 use crate::db::cauldron::header::load_all_headers;
 use crate::db::cauldron::tokenlist::db_utils::create_cached_token_metrics_table;
 use crate::db::cauldron::tokenlist::metrics_cache::spawn_token_metrics_updater;
+use crate::db::init::initialize_databases;
 use crate::index::{index_blocks, update_mempool};
 
 #[macro_use]
@@ -49,6 +45,10 @@ const RIFTEN_LABS_GENESIS_BLOCK: &str =
 const CASHTOKEN_ACTIVATION_HEIGHT: &str =
     "000000000000000002b678c471841c3e404ec7ae9ca9c32026fe27eb6e3a1ed1";
 
+// Chipnet genesis
+const CHIPNET_START_BLOCK: &str =
+    "000000001dd410c49a788668ce26751718cc797474d3152a5fc073dd44fd9f7b";
+
 // Last indexed block height.
 const KEY_LAST_INDEXED: &str = "last_indexed";
 
@@ -91,121 +91,51 @@ fn set_panic_hook() {
     }));
 }
 
-fn db_initialize(c: &mut rusqlite::Connection) -> Result<(), rusqlite::Error> {
-    // Set busy timeout for normal queries.
-    c.busy_timeout(Duration::from_secs(30))?;
-    c.execute("PRAGMA foreign_keys=1;", [])?;
-
-    // PRAGMA journal_mode may not honor busy_timeout, so retry manually.
-    let start = Instant::now();
-    let timeout = Duration::from_secs(30);
-    loop {
-        match c.execute_batch("PRAGMA journal_mode=WAL;") {
-            Ok(_) => break,
-            Err(rusqlite::Error::SqliteFailure(err, _))
-                if err.code == rusqlite::ErrorCode::DatabaseBusy =>
-            {
-                if start.elapsed() >= timeout {
-                    return Err(rusqlite::Error::SqliteFailure(
-                        err,
-                        Some("database locked after retries".into()),
-                    ));
-                }
-                sleep(Duration::from_millis(100));
-            }
-            Err(e) => return Err(e),
-        }
-    }
-    Ok(())
-}
-
-fn db_sanity_check(c: &rusqlite::Connection) -> rusqlite::Result<()> {
-    // Verify that foreign_keys are enabled
-    let mut stmt = c.prepare("PRAGMA foreign_keys;")?;
-    let foreign_keys_enabled: i32 = stmt.query_row([], |row| row.get(0))?;
-    if foreign_keys_enabled != 1 {
-        panic!(
-            "Foreign key enforcement is not enabled for the connection. (result: {foreign_keys_enabled})"
-        );
-    }
-    Ok(())
-}
-
 fn start_program(
     config: Config,
 ) -> Result<(DB, BCMRDownloader, WellKnownDownloader, CRC20Fetcher)> {
-    let create_db_pool = |db_path| -> (bool, DBPool, DBPool) {
-        let db_exists = Path::new(db_path).exists();
-        info!("Initializing connection to {db_path}");
-
-        let write_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path)
-            .with_flags(OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE)
-            .with_init(db_initialize);
-
-        let write_pool =
-            Arc::new(r2d2::Pool::new(write_manager).expect("Failed to initialize database"));
-
-        let read_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path)
-            .with_flags(OpenFlags::SQLITE_OPEN_READ_ONLY)
-            .with_init(db_initialize);
-
-        let read_pool =
-            Arc::new(r2d2::Pool::new(read_manager).expect("Failed to initialize database"));
-
-        db_sanity_check(&read_pool.get().unwrap()).expect("db sanity check failed");
-        db_sanity_check(&write_pool.get().unwrap()).expect("db sanity check failed");
-
-        (db_exists, write_pool, read_pool)
+    // Parse network parameter
+    let network = match config.network.to_lowercase().as_str() {
+        "mainnet" => Network::Bitcoin,
+        "chipnet" => Network::Chipnet,
+        _ => bail!(
+            "Invalid network '{}'. Must be 'mainnet' or 'chipnet'",
+            config.network
+        ),
     };
 
-    let (db_exists, cauldron_db_write, cauldron_db_read) = create_db_pool("cauldron.db");
-    if !db_exists {
-        db::cauldron::prepare_tables(
-            &cauldron_db_write
-                .get()
-                .expect("failed to get sqlite connection"),
-        );
+    info!("Using network: {:?}", network);
+
+    // Set default rostrum address based on network if not explicitly provided
+    let rostrum_addr = if config.rostrum_addr.is_empty() {
+        // User didn't provide --rostrum-addr, use network-specific default
+        match network {
+            Network::Chipnet => "127.0.0.1:64001".to_string(),
+            _ => "127.0.0.1:50001".to_string(),
+        }
     } else {
-        // Check database version for existing database
-        check_db_version(&cauldron_db_read.get().unwrap()).expect("Database version check failed");
-    }
-    let (db_exists, bcmr_db_write, bcmr_db_read) = create_db_pool("bcmr.db");
-    if !db_exists {
-        db::bcmr::prepare_tables(
-            &bcmr_db_write
-                .get()
-                .expect("failed to create sql connection"),
-        );
-    }
+        // User explicitly provided a value, use it as-is
+        config.rostrum_addr
+    };
 
-    let (db_exists, crc20_db_write, crc20_db_read) = create_db_pool("crc20.db");
-    if !db_exists {
-        db::crc20::prepare_tables(
-            &crc20_db_write
-                .get()
-                .expect("failed to create sqlite crc20 connection"),
-        );
-    }
-
-    let (db_exists, oracle_db_write, oracle_db_read) = create_db_pool("oracle.db");
-    if !db_exists {
-        db::oracle::prepare_tables(
-            &oracle_db_write
-                .get()
-                .expect("failed to create sqlite oracle connection"),
-        );
-    }
+    // Initialize all databases
+    let network_str = match network {
+        Network::Bitcoin => "mainnet",
+        Network::Chipnet => "chipnet",
+        _ => "mainnet",
+    };
+    let db = initialize_databases(network_str)?;
 
     // Create a shared flag for indexing status
     let indexing_in_progress = Arc::new(AtomicBool::new(false));
 
     let client = Arc::new(Mutex::new(
-        match Client::new(&format!("tcp://{}", config.rostrum_addr)) {
+        match Client::new(&format!("tcp://{}", rostrum_addr)) {
             Ok(server) => server,
             Err(e) => {
                 error!(
                     "Failed to connect to {}: {}. See --help for setting a different server.",
-                    config.rostrum_addr, e
+                    rostrum_addr, e
                 );
                 bail!(e)
             }
@@ -221,25 +151,14 @@ fn start_program(
     let chain = Arc::new(Mutex::new(chain::Chain::new(genesis.header)));
 
     // initialize insert sequence for pool history
-    db::cauldron::pool::initialize_seq(&cauldron_db_read.get().unwrap());
+    db::cauldron::pool::initialize_seq(&db.cauldron_r.get().unwrap());
 
     info!("Loading block headers...");
-    let all_headers = load_all_headers(&cauldron_db_read.get().unwrap()).unwrap();
+    let all_headers = load_all_headers(&db.cauldron_r.get().unwrap()).unwrap();
     info!("Initializing {} headers...", all_headers.len());
     chain.lock().unwrap().load(all_headers).unwrap();
     info!("Headers loaded.");
 
-    let db = DB {
-        cauldron_w: cauldron_db_write,
-        cauldron_r: cauldron_db_read,
-        bcmr_w: bcmr_db_write,
-        bcmr_r: bcmr_db_read,
-        crc20_w: crc20_db_write,
-        crc20_r: crc20_db_read,
-        oracle_w: oracle_db_write,
-        oracle_r: oracle_db_read,
-    };
-
     let db_cpy = db.clone();
 
     let mut crc20fetcher = CRC20Fetcher::new();
@@ -263,7 +182,13 @@ fn start_program(
         // Initial full index
         let mut tip: BlockHash = loop {
             indexing_in_progress_clone.store(true, Ordering::Relaxed);
-            break match index_blocks(chain.clone(), db.clone(), client.clone(), true) {
+            break match index_blocks(
+                chain.clone(),
+                db.clone(),
+                client.clone(),
+                true,
+                Some(network),
+            ) {
                 Ok(tip) => tip,
                 Err(e) => {
                     if e.to_string().contains("database is locked") {
@@ -289,7 +214,13 @@ fn start_program(
 
             if new_tip != tip {
                 indexing_in_progress_clone.store(true, Ordering::Relaxed);
-                tip = match index_blocks(chain.clone(), db.clone(), client.clone(), true) {
+                tip = match index_blocks(
+                    chain.clone(),
+                    db.clone(),
+                    client.clone(),
+                    true,
+                    Some(network),
+                ) {
                     Ok(t) => t,
                     Err(e) => {
                         warn!("Indexing block failed: {} {}", e, e.backtrace());