riftenlabs-indexer/src/bcmr/utilurl.rs

// Copyright (C) 2024 Riften Labs AS
//
// This software is licensed under the GNU Affero General Public License (AGPL), version 3.0 or later.
// A copy of the license can be found in the LICENSE file or at https://www.gnu.org/licenses/agpl-3.0.html

use std::time;

use anyhow::{bail, Result};
use bitcoin_hashes::{hex::ToHex, sha256, Hash};
use ureq::Agent;

pub fn get_url(
    url: &str,
    timeout: time::Duration,
    max_download_size: usize,
) -> Result<(String, String)> {
    let agent = Agent::new();
    let response = agent.get(url).timeout(timeout).call()?;

    if response.status() != 200 {
        bail!("Failed to fetch {}: HTTP {}", url, response.status());
    }

    let mut reader = response.into_reader();
    let mut content = Vec::new();
    let mut buffer = [0; 1024]; // Read in chunks of 1KB.

    while let Ok(count) = reader.read(&mut buffer) {
        if count == 0 {
            break;
        }
        if content.len() + count > max_download_size {
            bail!("Download exceeded the maximum allowed size");
        }
        content.extend_from_slice(&buffer[..count]);
    }

    let content_str = String::from_utf8(content)?;
    let hash = sha256::Hash::hash(content_str.as_bytes());
    Ok((content_str, hash.to_hex()))
}