// Copyright (C) 2024 Riften Labs AS // // This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later. // A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html use std::{ sync::{atomic::AtomicBool, Arc}, thread::{self, JoinHandle}, time::Duration, }; use crate::bcmr::parsedbcmr::ParsedBCMR; use crate::db::bcmr::insert_bcmr_data; use crate::db::bcmr::update_bcmr_failure; use crate::{ bcmr::parse_bcmr_from_opreturn, db::{ bcmr::{get_entries_missing_bcmr_download, AuthChainEntry}, DBPool, }, }; use anyhow::*; use bitcoin_hashes::{hex::ToHex, sha256, Hash}; use bitcoincash::{Script, TokenID}; use log::{info, warn}; use rand::thread_rng; use serde_json::Value; use ureq::Agent; use serde_json::json; use rand::seq::SliceRandom; use rayon::prelude::*; use std::result::Result::Ok; // We are generous on timeout to allow for slow ipfs gateway const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(60); const IPFS_GATEWAY: &str = "https://ipfs.io/ipfs/"; const MAX_BCMR_SIZE: usize = 1024 * 1024 * 100; // 100 MB const MAX_PARALLEL_DOWNLOADS: usize = 5; pub struct BCMRDownloader { db: DBPool, keep_running: Arc, download_thread: Option>, } const LOOP_SLEEP_TIME: Duration = Duration::from_secs(10); fn get_download_candidates(db: &DBPool) -> Vec { let conn = match db.get() { Ok(c) => c, Err(e) => { warn!("bcmr: Failed to get a db connection: {}", e); return Vec::default(); } }; match get_entries_missing_bcmr_download(&conn) { Ok(c) => c, Err(e) => { warn!("bcmr: Failed to fetch bcmr download candidates: {}", e); Vec::default() } } } fn get_url(url: &str, max_download_size: usize) -> Result<(String, String)> { let agent = Agent::new(); let response = agent.get(url).timeout(DOWNLOAD_TIMEOUT).call()?; if response.status() != 200 { bail!("Failed to fetch {}: HTTP {}", url, response.status()); } let mut reader = response.into_reader(); let mut content = Vec::new(); let mut buffer = [0; 1024]; // Read in chunks of 1KB. while let Ok(count) = reader.read(&mut buffer) { if count == 0 { break; } if content.len() + count > max_download_size { bail!("Download exceeded the maximum allowed size"); } content.extend_from_slice(&buffer[..count]); } let content_str = String::from_utf8(content)?; let hash = sha256::Hash::hash(content_str.as_bytes()); Ok((content_str, hash.to_hex())) } fn fetch_bcmr( urls: &[String], expected_hash: &str, ) -> ( Option<(String, String)>, String, /* error */ bool, /* fatal error (don't try again) */ ) { let mut candidate: Option<(String, String)> = None; let mut errors: Vec = Vec::new(); for url in urls { let url = if let Some(stripped) = url.strip_prefix("ipfs://") { format!("{}{}", IPFS_GATEWAY, stripped) } else if !url.starts_with("https://") { format!("https://{}", url) } else { url.to_string() }; let (contents, actual_hash) = match get_url(&url, MAX_BCMR_SIZE) { Ok(c) => c, Err(e) => { errors.push(e.to_string()); continue; } }; if actual_hash == expected_hash { return (Some((contents, actual_hash)), String::default(), false); } else { candidate = Some((contents, actual_hash)) } } // No URL's with matching exepected hash if candidate.is_some() { (candidate, String::default(), false) } else if errors.is_empty() { (None, "No URLs to fetch BCMR from".to_string(), true) } else { ( None, format!("Errors fetching BCMR: {}", errors.join("; ")), false, ) } } pub fn parse_bcmr_json( bcmr: &Value, token_id: &TokenID, actual_hash: String, expected_hash: String, ) -> Result { let identities = bcmr .get("identities") .context("BCMR missing 'identities'")?; let token_history = identities .get(token_id.to_hex()) .context("'identities' does not contain token ID")?; let first_entry = token_history .as_object() .context("token history is not an object")? .values() .next() .context("no entries in token history")?; let token = first_entry .get("token") .context("'token' missing in token history entry")?; let symbol = token .get("symbol") .context("'symbol' missing in 'token' entry")?; let symbol = symbol.as_str().context("'symbol' field is not a string")?; // decimals is optional and defaults to 0 let decimals_default = json!(0); let decimals = token .get("decimals") .unwrap_or(&decimals_default) .to_owned(); let decimals = { let d_int = if let Some(d) = decimals.as_i64() { d } else if let Some(d_str) = decimals.as_str() { d_str .parse::() .context("Invalid number in 'decimal' field for 'token")? } else { bail!("BCMR contains a 'decimal' field for 'token', but it's not a number") }; if d_int < 0 { bail!("'decimal' field for 'token' cannot be negative") } d_int as usize }; let empty_string = json!(""); let name = first_entry .get("name") .unwrap_or(&empty_string) .as_str() .unwrap_or(""); let description = first_entry .get("description") .unwrap_or(&empty_string) .as_str() .unwrap_or(""); let (icon, web) = if let Some(uris) = first_entry.get("uris") { let icon = uris .get("icon") .unwrap_or(&empty_string) .as_str() .unwrap_or(""); let web = uris .get("web") .unwrap_or(&empty_string) .as_str() .unwrap_or(""); (icon, web) } else { ("", "") }; Ok(ParsedBCMR::new( &token_id.to_hex(), symbol, decimals, name, description, icon, web, expected_hash, actual_hash, )) } impl BCMRDownloader { pub fn new(db: DBPool) -> Self { Self { db, keep_running: Arc::new(AtomicBool::new(true)), download_thread: None, } } pub fn start(&mut self) -> Result<()> { let db_cpy = self.db.clone(); let keep_running_cpy = self.keep_running.clone(); self.download_thread = Some( thread::Builder::new() .name("bcmr downloader".to_string()) .spawn(move || loop { if !keep_running_cpy.load(std::sync::atomic::Ordering::Relaxed) { info!("Exiting bcmr download thread"); return; } let mut queue = get_download_candidates(&db_cpy); let mut rng = thread_rng(); queue.shuffle(&mut rng); if queue.is_empty() { thread::sleep(LOOP_SLEEP_TIME); continue; } info!("bcmr: {} tokens need BCMR download", queue.len()); let pool = rayon::ThreadPoolBuilder::new() .num_threads(MAX_PARALLEL_DOWNLOADS) .build() .unwrap(); pool.install(|| { queue.par_iter().for_each(|entry| { info!( "bcmr: Dowloading BCMR for token {}", entry.token_id.to_hex() ); let bcmr = parse_bcmr_from_opreturn(&Script::from( entry.bcmr_data.as_ref().expect("bcmr data missing").clone(), )) .expect("invalid bcmr entry in db"); let (json_str, error, is_fatal) = fetch_bcmr(&bcmr.uris, &bcmr.hash.to_hex()); let conn = match db_cpy.get() { Ok(c) => c, Err(e) => { warn!("bcmr: Failed to get BCMR db connection: {}", e); return; } }; let (json_str, actual_hash) = match json_str { Some(j) => j, None => { if let Err(e) = update_bcmr_failure( &conn, &entry.utxo, &format!("Failed to fetch BCMR: {}", error), is_fatal, ) { warn!("bcmr: Failed to set BCMR error: {}", e) } return; } }; let json: Value = match serde_json::from_str(&json_str) { Ok(b) => b, Err(e) => { if let Err(e) = update_bcmr_failure( &conn, &entry.utxo, &format!("BCMR invalid JSON error: {}", e), true, ) { warn!("bcmr: Failed to set BCMR error: {}", e) } return; } }; let bcmr_parsed = match parse_bcmr_json( &json, &entry.token_id, actual_hash, bcmr.hash.to_hex(), ) { Ok(b) => b, Err(err) => { if let Err(e) = update_bcmr_failure( &conn, &entry.utxo, &format!("BCMR contents error: {}", err), true, ) { warn!("bcmr: Failed to set BCMR error: {}", e) } return; } }; if let Err(err) = insert_bcmr_data(&conn, &entry.utxo, &bcmr_parsed) { warn!("bcmr: Failed to insert BCMR data {}", err) } }); }); thread::sleep(LOOP_SLEEP_TIME) }) .expect("failed to start bcmr download thread"), ); Ok(()) } } impl Drop for BCMRDownloader { fn drop(&mut self) { self.keep_running .store(false, std::sync::atomic::Ordering::SeqCst); if let Some(thread) = self.download_thread.take() { // Wake the thread in case it is sleeping thread.thread().unpark(); match thread.join() { Ok(_) => info!("bcmr download thread done"), Err(e) => warn!("Failed to join bcmr download thread: {:?}", e), } } } } #[cfg(test)] mod tests { use super::*; use bitcoin_hashes::hex::FromHex; #[test] fn test_bcmr_parse() { let bcmr = r#" { "$schema": "https://cashtokens.org/bcmr-v2.schema.json", "version": { "major": 0, "minor": 1, "patch": 0 }, "latestRevision": "2023-05-15T12:18:32.912Z", "registryIdentity": { "name": "Fallout Coin", "description": "One of the earliest coins ever created on BCH", "uris": { "icon": "https://c3-soft.com/tokens/icon.ico", "web": "https://c3-soft.com/tokens/", "registry": "https://c3-soft.com/tokens/registry.json" } }, "identities": { "83e12eea20b19a9a0906bb0521ff18520db69a4a8136293bafbfca0acb2c2313": { "2023-05-12T12:00:00.000Z": { "name": "Fallout Coin", "description": "Commemorative Coin for the computer game Fallout", "token": { "category": "83e12eea20b19a9a0906bb0521ff18520db69a4a8136293bafbfca0acb2c2313", "symbol": "FC", "decimals": 2 }, "uris": { "icon": "https://c3-soft.com/tokens/fallout.ico" } } } }, "license": "CC0-1.0" } "#; let dummy_hash = TokenID::all_zeros().to_hex(); let parsed = parse_bcmr_json( &serde_json::from_str(bcmr).unwrap(), &TokenID::from_hex("83e12eea20b19a9a0906bb0521ff18520db69a4a8136293bafbfca0acb2c2313") .unwrap(), dummy_hash.clone(), dummy_hash, ) .unwrap(); assert_eq!("FC", parsed.token.symbol); assert_eq!(2, parsed.token.decimals); assert_eq!("Fallout Coin", parsed.name); assert_eq!( "Commemorative Coin for the computer game Fallout", parsed.description ); assert_eq!( "https://c3-soft.com/tokens/fallout.ico", parsed.uris.icon.unwrap() ); assert_eq!(None, parsed.uris.web); } }