450 lines
14 KiB
Rust
450 lines
14 KiB
Rust
|
// Copyright (C) 2024 Riften Labs AS
|
||
|
|
//
|
||
|
|
// This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later.
|
||
|
|
// A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html
|
||
|
|
|
||
|
|
use std::{
|
||
|
|
sync::{atomic::AtomicBool, Arc},
|
||
|
|
thread::{self, JoinHandle},
|
||
|
|
time::Duration,
|
||
|
|
};
|
||
|
|
|
||
|
|
use crate::bcmr::parsedbcmr::ParsedBCMR;
|
||
|
|
use crate::db::bcmr::insert_bcmr_data;
|
||
|
|
use crate::db::bcmr::update_bcmr_failure;
|
||
|
|
use crate::{
|
||
|
|
bcmr::parse_bcmr_from_opreturn,
|
||
|
|
db::{
|
||
|
|
bcmr::{get_entries_missing_bcmr_download, AuthChainEntry},
|
||
|
|
DBPool,
|
||
|
|
},
|
||
|
|
};
|
||
|
|
use anyhow::*;
|
||
|
|
use bitcoin_hashes::{hex::ToHex, sha256, Hash};
|
||
|
|
use bitcoincash::{Script, TokenID};
|
||
|
|
use log::{info, warn};
|
||
|
|
use rand::thread_rng;
|
||
|
|
use serde_json::Value;
|
||
|
|
use ureq::Agent;
|
||
|
|
|
||
|
|
use serde_json::json;
|
||
|
|
|
||
|
|
use rand::seq::SliceRandom;
|
||
|
|
use rayon::prelude::*;
|
||
|
|
use std::result::Result::Ok;
|
||
|
|
|
||
|
|
// We are generous on timeout to allow for slow ipfs gateway
|
||
|
|
const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(60);
|
||
|
|
const IPFS_GATEWAY: &str = "https://ipfs.io/ipfs/";
|
||
|
|
const MAX_BCMR_SIZE: usize = 1024 * 1024 * 100; // 100 MB
|
||
|
|
|
||
|
|
const MAX_PARALLEL_DOWNLOADS: usize = 5;
|
||
|
|
|
||
|
|
pub struct BCMRDownloader {
|
||
|
|
db: DBPool,
|
||
|
|
keep_running: Arc<AtomicBool>,
|
||
|
|
download_thread: Option<JoinHandle<()>>,
|
||
|
|
}
|
||
|
|
|
||
|
|
const LOOP_SLEEP_TIME: Duration = Duration::from_secs(10);
|
||
|
|
|
||
|
|
fn get_download_candidates(db: &DBPool) -> Vec<AuthChainEntry> {
|
||
|
|
let conn = match db.get() {
|
||
|
|
Ok(c) => c,
|
||
|
|
Err(e) => {
|
||
|
|
warn!("bcmr: Failed to get a db connection: {}", e);
|
||
|
|
return Vec::default();
|
||
|
|
}
|
||
|
|
};
|
||
|
|
match get_entries_missing_bcmr_download(&conn) {
|
||
|
|
Ok(c) => c,
|
||
|
|
Err(e) => {
|
||
|
|
warn!("bcmr: Failed to fetch bcmr download candidates: {}", e);
|
||
|
|
Vec::default()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn get_url(url: &str, max_download_size: usize) -> Result<(String, String)> {
|
||
|
|
let agent = Agent::new();
|
||
|
|
let response = agent.get(url).timeout(DOWNLOAD_TIMEOUT).call()?;
|
||
|
|
|
||
|
|
if response.status() != 200 {
|
||
|
|
bail!("Failed to fetch {}: HTTP {}", url, response.status());
|
||
|
|
}
|
||
|
|
|
||
|
|
let mut reader = response.into_reader();
|
||
|
|
let mut content = Vec::new();
|
||
|
|
let mut buffer = [0; 1024]; // Read in chunks of 1KB.
|
||
|
|
|
||
|
|
while let Ok(count) = reader.read(&mut buffer) {
|
||
|
|
if count == 0 {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
if content.len() + count > max_download_size {
|
||
|
|
bail!("Download exceeded the maximum allowed size");
|
||
|
|
}
|
||
|
|
content.extend_from_slice(&buffer[..count]);
|
||
|
|
}
|
||
|
|
|
||
|
|
let content_str = String::from_utf8(content)?;
|
||
|
|
let hash = sha256::Hash::hash(content_str.as_bytes());
|
||
|
|
Ok((content_str, hash.to_hex()))
|
||
|
|
}
|
||
|
|
|
||
|
|
fn fetch_bcmr(
|
||
|
|
urls: &[String],
|
||
|
|
expected_hash: &str,
|
||
|
|
) -> (
|
||
|
|
Option<(String, String)>,
|
||
|
|
String, /* error */
|
||
|
|
bool, /* fatal error (don't try again) */
|
||
|
|
) {
|
||
|
|
let mut candidate: Option<(String, String)> = None;
|
||
|
|
let mut errors: Vec<String> = Vec::new();
|
||
|
|
|
||
|
|
for url in urls {
|
||
|
|
let url = if let Some(stripped) = url.strip_prefix("ipfs://") {
|
||
|
|
format!("{}{}", IPFS_GATEWAY, stripped)
|
||
|
|
} else if !url.starts_with("https://") {
|
||
|
|
format!("https://{}", url)
|
||
|
|
} else {
|
||
|
|
url.to_string()
|
||
|
|
};
|
||
|
|
|
||
|
|
let (contents, actual_hash) = match get_url(&url, MAX_BCMR_SIZE) {
|
||
|
|
Ok(c) => c,
|
||
|
|
Err(e) => {
|
||
|
|
errors.push(e.to_string());
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
if actual_hash == expected_hash {
|
||
|
|
return (Some((contents, actual_hash)), String::default(), false);
|
||
|
|
} else {
|
||
|
|
candidate = Some((contents, actual_hash))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// No URL's with matching exepected hash
|
||
|
|
if candidate.is_some() {
|
||
|
|
(candidate, String::default(), false)
|
||
|
|
} else if errors.is_empty() {
|
||
|
|
(None, "No URLs to fetch BCMR from".to_string(), true)
|
||
|
|
} else {
|
||
|
|
(
|
||
|
|
None,
|
||
|
|
format!("Errors fetching BCMR: {}", errors.join("; ")),
|
||
|
|
false,
|
||
|
|
)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn parse_bcmr_json(
|
||
|
|
bcmr: &Value,
|
||
|
|
token_id: &TokenID,
|
||
|
|
actual_hash: String,
|
||
|
|
expected_hash: String,
|
||
|
|
) -> Result<ParsedBCMR> {
|
||
|
|
let identities = bcmr
|
||
|
|
.get("identities")
|
||
|
|
.context("BCMR missing 'identities'")?;
|
||
|
|
let token_history = identities
|
||
|
|
.get(token_id.to_hex())
|
||
|
|
.context("'identities' does not contain token ID")?;
|
||
|
|
let first_entry = token_history
|
||
|
|
.as_object()
|
||
|
|
.context("token history is not an object")?
|
||
|
|
.values()
|
||
|
|
.next()
|
||
|
|
.context("no entries in token history")?;
|
||
|
|
|
||
|
|
let token = first_entry
|
||
|
|
.get("token")
|
||
|
|
.context("'token' missing in token history entry")?;
|
||
|
|
|
||
|
|
let symbol = token
|
||
|
|
.get("symbol")
|
||
|
|
.context("'symbol' missing in 'token' entry")?;
|
||
|
|
let symbol = symbol.as_str().context("'symbol' field is not a string")?;
|
||
|
|
|
||
|
|
// decimals is optional and defaults to 0
|
||
|
|
let decimals_default = json!(0);
|
||
|
|
let decimals = token
|
||
|
|
.get("decimals")
|
||
|
|
.unwrap_or(&decimals_default)
|
||
|
|
.to_owned();
|
||
|
|
|
||
|
|
let decimals = {
|
||
|
|
let d_int = if let Some(d) = decimals.as_i64() {
|
||
|
|
d
|
||
|
|
} else if let Some(d_str) = decimals.as_str() {
|
||
|
|
d_str
|
||
|
|
.parse::<i64>()
|
||
|
|
.context("Invalid number in 'decimal' field for 'token")?
|
||
|
|
} else {
|
||
|
|
bail!("BCMR contains a 'decimal' field for 'token', but it's not a number")
|
||
|
|
};
|
||
|
|
|
||
|
|
if d_int < 0 {
|
||
|
|
bail!("'decimal' field for 'token' cannot be negative")
|
||
|
|
}
|
||
|
|
d_int as usize
|
||
|
|
};
|
||
|
|
|
||
|
|
let empty_string = json!("");
|
||
|
|
let name = first_entry
|
||
|
|
.get("name")
|
||
|
|
.unwrap_or(&empty_string)
|
||
|
|
.as_str()
|
||
|
|
.unwrap_or("");
|
||
|
|
let description = first_entry
|
||
|
|
.get("description")
|
||
|
|
.unwrap_or(&empty_string)
|
||
|
|
.as_str()
|
||
|
|
.unwrap_or("");
|
||
|
|
|
||
|
|
let (icon, web) = if let Some(uris) = first_entry.get("uris") {
|
||
|
|
let icon = uris
|
||
|
|
.get("icon")
|
||
|
|
.unwrap_or(&empty_string)
|
||
|
|
.as_str()
|
||
|
|
.unwrap_or("");
|
||
|
|
let web = uris
|
||
|
|
.get("web")
|
||
|
|
.unwrap_or(&empty_string)
|
||
|
|
.as_str()
|
||
|
|
.unwrap_or("");
|
||
|
|
(icon, web)
|
||
|
|
} else {
|
||
|
|
("", "")
|
||
|
|
};
|
||
|
|
|
||
|
|
Ok(ParsedBCMR::new(
|
||
|
|
&token_id.to_hex(),
|
||
|
|
symbol,
|
||
|
|
decimals,
|
||
|
|
name,
|
||
|
|
description,
|
||
|
|
icon,
|
||
|
|
web,
|
||
|
|
expected_hash,
|
||
|
|
actual_hash,
|
||
|
|
))
|
||
|
|
}
|
||
|
|
|
||
|
|
impl BCMRDownloader {
|
||
|
|
pub fn new(db: DBPool) -> Self {
|
||
|
|
Self {
|
||
|
|
db,
|
||
|
|
keep_running: Arc::new(AtomicBool::new(true)),
|
||
|
|
download_thread: None,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn start(&mut self) -> Result<()> {
|
||
|
|
let db_cpy = self.db.clone();
|
||
|
|
let keep_running_cpy = self.keep_running.clone();
|
||
|
|
self.download_thread = Some(
|
||
|
|
thread::Builder::new()
|
||
|
|
.name("bcmr downloader".to_string())
|
||
|
|
.spawn(move || loop {
|
||
|
|
if !keep_running_cpy.load(std::sync::atomic::Ordering::Relaxed) {
|
||
|
|
info!("Exiting bcmr download thread");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
let mut queue = get_download_candidates(&db_cpy);
|
||
|
|
|
||
|
|
let mut rng = thread_rng();
|
||
|
|
queue.shuffle(&mut rng);
|
||
|
|
|
||
|
|
if queue.is_empty() {
|
||
|
|
thread::sleep(LOOP_SLEEP_TIME);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
info!("bcmr: {} tokens need BCMR download", queue.len());
|
||
|
|
|
||
|
|
let pool = rayon::ThreadPoolBuilder::new()
|
||
|
|
.num_threads(MAX_PARALLEL_DOWNLOADS)
|
||
|
|
.build()
|
||
|
|
.unwrap();
|
||
|
|
|
||
|
|
pool.install(|| {
|
||
|
|
queue.par_iter().for_each(|entry| {
|
||
|
|
info!(
|
||
|
|
"bcmr: Dowloading BCMR for token {}",
|
||
|
|
entry.token_id.to_hex()
|
||
|
|
);
|
||
|
|
|
||
|
|
let bcmr = parse_bcmr_from_opreturn(&Script::from(
|
||
|
|
entry.bcmr_data.as_ref().expect("bcmr data missing").clone(),
|
||
|
|
))
|
||
|
|
.expect("invalid bcmr entry in db");
|
||
|
|
|
||
|
|
let (json_str, error, is_fatal) =
|
||
|
|
fetch_bcmr(&bcmr.uris, &bcmr.hash.to_hex());
|
||
|
|
|
||
|
|
let conn = match db_cpy.get() {
|
||
|
|
Ok(c) => c,
|
||
|
|
Err(e) => {
|
||
|
|
warn!("bcmr: Failed to get BCMR db connection: {}", e);
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
let (json_str, actual_hash) = match json_str {
|
||
|
|
Some(j) => j,
|
||
|
|
None => {
|
||
|
|
if let Err(e) = update_bcmr_failure(
|
||
|
|
&conn,
|
||
|
|
&entry.utxo,
|
||
|
|
&format!("Failed to fetch BCMR: {}", error),
|
||
|
|
is_fatal,
|
||
|
|
) {
|
||
|
|
warn!("bcmr: Failed to set BCMR error: {}", e)
|
||
|
|
}
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
let json: Value = match serde_json::from_str(&json_str) {
|
||
|
|
Ok(b) => b,
|
||
|
|
Err(e) => {
|
||
|
|
if let Err(e) = update_bcmr_failure(
|
||
|
|
&conn,
|
||
|
|
&entry.utxo,
|
||
|
|
&format!("BCMR invalid JSON error: {}", e),
|
||
|
|
true,
|
||
|
|
) {
|
||
|
|
warn!("bcmr: Failed to set BCMR error: {}", e)
|
||
|
|
}
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
let bcmr_parsed = match parse_bcmr_json(
|
||
|
|
&json,
|
||
|
|
&entry.token_id,
|
||
|
|
actual_hash,
|
||
|
|
bcmr.hash.to_hex(),
|
||
|
|
) {
|
||
|
|
Ok(b) => b,
|
||
|
|
Err(err) => {
|
||
|
|
if let Err(e) = update_bcmr_failure(
|
||
|
|
&conn,
|
||
|
|
&entry.utxo,
|
||
|
|
&format!("BCMR contents error: {}", err),
|
||
|
|
true,
|
||
|
|
) {
|
||
|
|
warn!("bcmr: Failed to set BCMR error: {}", e)
|
||
|
|
}
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
if let Err(err) = insert_bcmr_data(&conn, &entry.utxo, &bcmr_parsed) {
|
||
|
|
warn!("bcmr: Failed to insert BCMR data {}", err)
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
thread::sleep(LOOP_SLEEP_TIME)
|
||
|
|
})
|
||
|
|
.expect("failed to start bcmr download thread"),
|
||
|
|
);
|
||
|
|
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
impl Drop for BCMRDownloader {
|
||
|
|
fn drop(&mut self) {
|
||
|
|
self.keep_running
|
||
|
|
.store(false, std::sync::atomic::Ordering::SeqCst);
|
||
|
|
if let Some(thread) = self.download_thread.take() {
|
||
|
|
// Wake the thread in case it is sleeping
|
||
|
|
thread.thread().unpark();
|
||
|
|
|
||
|
|
match thread.join() {
|
||
|
|
Ok(_) => info!("bcmr download thread done"),
|
||
|
|
Err(e) => warn!("Failed to join bcmr download thread: {:?}", e),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod tests {
|
||
|
|
|
||
|
|
use super::*;
|
||
|
|
use bitcoin_hashes::hex::FromHex;
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn test_bcmr_parse() {
|
||
|
|
let bcmr = r#"
|
||
|
|
{
|
||
|
|
"$schema": "https://cashtokens.org/bcmr-v2.schema.json",
|
||
|
|
"version": {
|
||
|
|
"major": 0,
|
||
|
|
"minor": 1,
|
||
|
|
"patch": 0
|
||
|
|
},
|
||
|
|
"latestRevision": "2023-05-15T12:18:32.912Z",
|
||
|
|
"registryIdentity": {
|
||
|
|
"name": "Fallout Coin",
|
||
|
|
"description": "One of the earliest coins ever created on BCH",
|
||
|
|
"uris": {
|
||
|
|
"icon": "https://c3-soft.com/tokens/icon.ico",
|
||
|
|
"web": "https://c3-soft.com/tokens/",
|
||
|
|
"registry": "https://c3-soft.com/tokens/registry.json"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"identities": {
|
||
|
|
"83e12eea20b19a9a0906bb0521ff18520db69a4a8136293bafbfca0acb2c2313": {
|
||
|
|
"2023-05-12T12:00:00.000Z": {
|
||
|
|
"name": "Fallout Coin",
|
||
|
|
"description": "Commemorative Coin for the computer game Fallout",
|
||
|
|
"token": {
|
||
|
|
"category": "83e12eea20b19a9a0906bb0521ff18520db69a4a8136293bafbfca0acb2c2313",
|
||
|
|
"symbol": "FC",
|
||
|
|
"decimals": 2
|
||
|
|
},
|
||
|
|
"uris": {
|
||
|
|
"icon": "https://c3-soft.com/tokens/fallout.ico"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"license": "CC0-1.0"
|
||
|
|
}
|
||
|
|
"#;
|
||
|
|
|
||
|
|
let dummy_hash = TokenID::all_zeros().to_hex();
|
||
|
|
|
||
|
|
let parsed = parse_bcmr_json(
|
||
|
|
&serde_json::from_str(bcmr).unwrap(),
|
||
|
|
&TokenID::from_hex("83e12eea20b19a9a0906bb0521ff18520db69a4a8136293bafbfca0acb2c2313")
|
||
|
|
.unwrap(),
|
||
|
|
dummy_hash.clone(),
|
||
|
|
dummy_hash,
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
|
||
|
|
assert_eq!("FC", parsed.token.symbol);
|
||
|
|
assert_eq!(2, parsed.token.decimals);
|
||
|
|
assert_eq!("Fallout Coin", parsed.name);
|
||
|
|
assert_eq!(
|
||
|
|
"Commemorative Coin for the computer game Fallout",
|
||
|
|
parsed.description
|
||
|
|
);
|
||
|
|
assert_eq!(
|
||
|
|
"https://c3-soft.com/tokens/fallout.ico",
|
||
|
|
parsed.uris.icon.unwrap()
|
||
|
|
);
|
||
|
|
assert_eq!(None, parsed.uris.web);
|
||
|
|
}
|
||
|
|
}
|