Merge branch 'db-cfg' into 'master'

Make DB read pool size configurable

See merge request riftenlabs/riftenlabs-indexer!65

Cargo.lock

@@ -1737,7 +1737,7 @@ dependencies = [
 
 [[package]]
 name = "riftenlabs-indexer"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
  "anyhow",
  "bitcoin_hashes",

Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "riftenlabs-indexer"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
-authors = ["Riften Labs AS <hello@cauldron.quest>"]
+authors = ["Whiterun LLC <hello@riftenlabs.com>"]
 description = "Riften Labs DeFi contract indexer on Bitcoin Cash"
 license = "AGPL-3.0-or-later"
 homepage = "https://gitlab.com/riftenlabs/riftenlabs-indexer"

internal/config_specification.toml

@@ -15,3 +15,27 @@ name = "serve_during_ibd"
 type = "bool"
 doc = "Allow serving API requests during initial block download (for debugging)"
 default = "false"
+
+[[param]]
+name = "cauldron_read_slots"
+type = "u32"
+doc = "Number of read connection pool slots for the cauldron database (default: 8)"
+default = "32"
+
+[[param]]
+name = "bcmr_read_slots"
+type = "u32"
+doc = "Number of read connection pool slots for the BCMR database (default: 8)"
+default = "8"
+
+[[param]]
+name = "crc20_read_slots"
+type = "u32"
+doc = "Number of read connection pool slots for the CRC20 database (default: 8)"
+default = "8"
+
+[[param]]
+name = "oracle_read_slots"
+type = "u32"
+doc = "Number of read connection pool slots for the oracle database (default: 8)"
+default = "8"

src/db/init.rs

@@ -61,9 +61,9 @@ pub fn db_sanity_check(c: &rusqlite::Connection) -> rusqlite::Result<()> {
 }
 
 /// Create read and write database pools for a given database path
-fn create_db_pool(db_path: &str) -> (bool, DBPool, DBPool) {
+fn create_db_pool(db_path: &str, read_max_size: u32) -> (bool, DBPool, DBPool) {
     let db_exists = Path::new(db_path).exists();
-    info!("Initializing connection to {db_path}");
+    info!("Initializing connection to {db_path} (read_slots={read_max_size})");
 
     let write_manager = r2d2_sqlite::SqliteConnectionManager::file(db_path)
         .with_flags(OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE)
@@ -82,7 +82,7 @@ fn create_db_pool(db_path: &str) -> (bool, DBPool, DBPool) {
 
     let read_pool = Arc::new(
         r2d2::Pool::builder()
-            .max_size(8) // Read pools can have more for concurrent requests
+            .max_size(read_max_size)
             .build(read_manager)
             .expect("Failed to initialize database"),
     );
@@ -121,14 +121,22 @@ fn db_path(db_dir: &str, filename: &str) -> String {
     }
 }
 
+/// Per-database read pool sizes
+pub struct ReadSlots {
+    pub cauldron: u32,
+    pub bcmr: u32,
+    pub crc20: u32,
+    pub oracle: u32,
+}
+
 /// Initialize all databases and return a DB struct
-pub fn initialize_databases(network: &str) -> Result<DB> {
+pub fn initialize_databases(network: &str, read_slots: ReadSlots) -> Result<DB> {
     let db_dir = get_db_directory(network);
     ensure_db_directory(db_dir)?;
 
     // Initialize cauldron database
     let (db_exists, cauldron_db_write, cauldron_db_read) =
-        create_db_pool(&db_path(db_dir, "cauldron.db"));
+        create_db_pool(&db_path(db_dir, "cauldron.db"), read_slots.cauldron);
     if !db_exists {
         cauldron_prepare_tables(
             &cauldron_db_write
@@ -141,7 +149,8 @@ pub fn initialize_databases(network: &str) -> Result<DB> {
     }
 
     // Initialize BCMR database
-    let (db_exists, bcmr_db_write, bcmr_db_read) = create_db_pool(&db_path(db_dir, "bcmr.db"));
+    let (db_exists, bcmr_db_write, bcmr_db_read) =
+        create_db_pool(&db_path(db_dir, "bcmr.db"), read_slots.bcmr);
     if !db_exists {
         bcmr_prepare_tables(
             &bcmr_db_write
@@ -151,7 +160,8 @@ pub fn initialize_databases(network: &str) -> Result<DB> {
     }
 
     // Initialize CRC20 database
-    let (db_exists, crc20_db_write, crc20_db_read) = create_db_pool(&db_path(db_dir, "crc20.db"));
+    let (db_exists, crc20_db_write, crc20_db_read) =
+        create_db_pool(&db_path(db_dir, "crc20.db"), read_slots.crc20);
     if !db_exists {
         crc20_prepare_tables(
             &crc20_db_write
@@ -162,7 +172,7 @@ pub fn initialize_databases(network: &str) -> Result<DB> {
 
     // Initialize oracle database
     let (db_exists, oracle_db_write, oracle_db_read) =
-        create_db_pool(&db_path(db_dir, "oracle.db"));
+        create_db_pool(&db_path(db_dir, "oracle.db"), read_slots.oracle);
     if !db_exists {
         oracle_prepare_tables(
             &oracle_db_write

src/main.rs

@@ -41,7 +41,7 @@ use crate::bcmr::bcmrdownloader::BCMRDownloader;
 use crate::db::cauldron::header::load_all_headers;
 use crate::db::cauldron::tokenlist::db_utils::create_cached_token_metrics_table;
 use crate::db::cauldron::tokenlist::metrics_cache::spawn_token_metrics_updater;
-use crate::db::init::initialize_databases;
+use crate::db::init::{initialize_databases, ReadSlots};
 use crate::index::{index_blocks, update_mempool};
 
 #[macro_use]
@@ -159,7 +159,15 @@ fn start_program(
         Network::Chipnet => "chipnet",
         _ => "mainnet",
     };
-    let db = initialize_databases(network_str)?;
+    let db = initialize_databases(
+        network_str,
+        ReadSlots {
+            cauldron: config.cauldron_read_slots,
+            bcmr: config.bcmr_read_slots,
+            crc20: config.crc20_read_slots,
+            oracle: config.oracle_read_slots,
+        },
+    )?;
 
     // Create a shared flag for indexing status
     let indexing_in_progress = Arc::new(AtomicBool::new(false));