775 lines
27 KiB
Rust
Executable File
775 lines
27 KiB
Rust
Executable File
//! Source tarball fetching with checksum verification
|
|
|
|
use crate::package::{PackageSpec, Source};
|
|
use anyhow::{Context, Result, bail};
|
|
use indicatif::{ProgressBar, ProgressStyle};
|
|
use sha2::{Digest, Sha256};
|
|
use std::collections::HashSet;
|
|
use std::fs::{self, File};
|
|
use std::io::{Read, Write};
|
|
use std::path::{Path, PathBuf};
|
|
use url::Url;
|
|
|
|
const MAX_MIRROR_RETRIES: usize = 8;
|
|
|
|
/// Fetch an archive source tarball, returning path to downloaded file.
|
|
pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> Result<PathBuf> {
|
|
let url = spec.expand_vars(&source.url);
|
|
let filename = derive_filename_from_url(&url);
|
|
let dest_path = cache_dir.join(&filename);
|
|
|
|
// Create cache directory if needed
|
|
fs::create_dir_all(cache_dir)
|
|
.with_context(|| format!("Failed to create cache dir: {}", cache_dir.display()))?;
|
|
|
|
// Check if already cached and verified
|
|
if dest_path.exists() && verify_checksum(&dest_path, &source.sha256)? {
|
|
crate::log_info!("Using cached source: {}", dest_path.display());
|
|
return Ok(dest_path);
|
|
}
|
|
|
|
crate::log_info!("Fetching: {}", url);
|
|
|
|
// Parse URL early so we can handle non-HTTP schemes (FTP support)
|
|
let parsed_url = Url::parse(&url).with_context(|| format!("Invalid URL: {}", url))?;
|
|
|
|
// If this is an FTP URL, fetch via suppaftp into the cache and continue
|
|
if parsed_url.scheme() == "ftp" {
|
|
// Connect and login (anonymous fallback)
|
|
let host = parsed_url.host_str().context("FTP URL missing host")?;
|
|
let port = parsed_url.port_or_known_default().unwrap_or(21);
|
|
let addr = format!("{}:{}", host, port);
|
|
let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str())
|
|
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
|
|
let user = if parsed_url.username().is_empty() {
|
|
"anonymous"
|
|
} else {
|
|
parsed_url.username()
|
|
};
|
|
let pass = parsed_url.password().unwrap_or("anonymous@");
|
|
ftp_stream
|
|
.login(user, pass)
|
|
.with_context(|| format!("FTP login failed for {}", host))?;
|
|
|
|
// Retrieve the path (try with and without leading slash)
|
|
let path = parsed_url.path();
|
|
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
|
|
let mut retrieved = false;
|
|
for p in candidates.iter().filter(|s| !s.is_empty()) {
|
|
match ftp_stream.retr(
|
|
p,
|
|
|reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> {
|
|
let mut file =
|
|
File::create(&dest_path).map_err(suppaftp::FtpError::ConnectionError)?;
|
|
let mut buffer = [0u8; 8192];
|
|
loop {
|
|
let bytes_read = reader
|
|
.read(&mut buffer)
|
|
.map_err(suppaftp::FtpError::ConnectionError)?;
|
|
if bytes_read == 0 {
|
|
break;
|
|
}
|
|
file.write_all(&buffer[..bytes_read])
|
|
.map_err(suppaftp::FtpError::ConnectionError)?;
|
|
}
|
|
Ok(())
|
|
},
|
|
) {
|
|
Ok(_) => {
|
|
retrieved = true;
|
|
break;
|
|
}
|
|
Err(_) => continue,
|
|
}
|
|
}
|
|
ftp_stream.quit().ok();
|
|
if !retrieved {
|
|
bail!("FTP error fetching {}", url);
|
|
}
|
|
}
|
|
|
|
// Download with progress bar
|
|
// Use a sensible default User-Agent so servers that reject empty/unknown agents (e.g. IANA)
|
|
// will accept requests. Include package name/version at compile time.
|
|
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
|
|
let client =
|
|
super::build_blocking_client(&ua, None).with_context(|| "Failed to build HTTP client")?;
|
|
|
|
let mut response = client
|
|
.get(&url)
|
|
.send()
|
|
.with_context(|| format!("Failed to fetch: {}", url))?;
|
|
// If the server returned a non-success status, read a short body preview and fail early.
|
|
// This prevents saving HTML error pages (which then fail checksum) and gives a clearer
|
|
// diagnostic to the user.
|
|
let status = response.status();
|
|
if !status.is_success() {
|
|
let mut preview_bytes = Vec::new();
|
|
// read up to 1 KiB for a preview (ignore errors while reading preview)
|
|
let _ = response.take(1024).read_to_end(&mut preview_bytes);
|
|
let preview = String::from_utf8_lossy(&preview_bytes);
|
|
bail!(
|
|
"HTTP error fetching {}: {}{}",
|
|
url,
|
|
status,
|
|
if preview.trim().is_empty() {
|
|
"".to_string()
|
|
} else {
|
|
format!(" — preview: {}", preview.trim())
|
|
}
|
|
);
|
|
}
|
|
let total_size = response.content_length().unwrap_or(0);
|
|
let pb = ProgressBar::new(total_size);
|
|
pb.set_style(
|
|
ProgressStyle::default_bar()
|
|
.template("{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})")
|
|
.unwrap()
|
|
.progress_chars("#>-"),
|
|
);
|
|
|
|
let mut file = File::create(&dest_path)
|
|
.with_context(|| format!("Failed to create: {}", dest_path.display()))?;
|
|
|
|
let mut buffer = [0u8; 8192];
|
|
let mut downloaded = 0u64;
|
|
|
|
loop {
|
|
let bytes_read = response.read(&mut buffer)?;
|
|
if bytes_read == 0 {
|
|
break;
|
|
}
|
|
file.write_all(&buffer[..bytes_read])?;
|
|
downloaded += bytes_read as u64;
|
|
pb.set_position(downloaded);
|
|
}
|
|
|
|
pb.finish_with_message("Download complete");
|
|
|
|
// Quick validation: ensure the downloaded file looks like the expected
|
|
// archive (detect obvious HTML error pages or wrong formats by magic).
|
|
// If validation returns an alternate URL (e.g. SourceForge mirror), follow
|
|
// and retry a few times.
|
|
let mut next_alt = validate_downloaded_archive(&dest_path, &filename, &url)?;
|
|
let mut seen_alts: HashSet<String> = HashSet::new();
|
|
let mut retries = 0usize;
|
|
while let Some(alt) = next_alt {
|
|
retries += 1;
|
|
if retries > MAX_MIRROR_RETRIES {
|
|
bail!(
|
|
"Exceeded mirror retry limit ({}) while fetching {}",
|
|
MAX_MIRROR_RETRIES,
|
|
url
|
|
);
|
|
}
|
|
if !seen_alts.insert(alt.clone()) {
|
|
bail!("Mirror retry loop detected for URL: {}", alt);
|
|
}
|
|
crate::log_info!("Retrying download from mirror: {}", alt);
|
|
fs::remove_file(&dest_path).ok();
|
|
|
|
// If mirror URL is FTP -> use suppaftp; otherwise use HTTP retry.
|
|
if let Ok(alt_url) = Url::parse(&alt) {
|
|
if alt_url.scheme() == "ftp" {
|
|
// FTP mirror retrieval
|
|
let host = alt_url.host_str().context("FTP mirror URL missing host")?;
|
|
let port = alt_url.port_or_known_default().unwrap_or(21);
|
|
let addr = format!("{}:{}", host, port);
|
|
let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str())
|
|
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
|
|
let user = if alt_url.username().is_empty() {
|
|
"anonymous"
|
|
} else {
|
|
alt_url.username()
|
|
};
|
|
let pass = alt_url.password().unwrap_or("anonymous@");
|
|
ftp_stream
|
|
.login(user, pass)
|
|
.with_context(|| format!("FTP login failed for {}", host))?;
|
|
|
|
let path = alt_url.path();
|
|
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
|
|
let mut retrieved = false;
|
|
for p in candidates.iter().filter(|s| !s.is_empty()) {
|
|
match ftp_stream.retr(
|
|
p,
|
|
|reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> {
|
|
let mut file = File::create(&dest_path)
|
|
.map_err(suppaftp::FtpError::ConnectionError)?;
|
|
let mut buffer = [0u8; 8192];
|
|
let mut downloaded = 0u64;
|
|
loop {
|
|
let bytes_read = reader
|
|
.read(&mut buffer)
|
|
.map_err(suppaftp::FtpError::ConnectionError)?;
|
|
if bytes_read == 0 {
|
|
break;
|
|
}
|
|
file.write_all(&buffer[..bytes_read])
|
|
.map_err(suppaftp::FtpError::ConnectionError)?;
|
|
downloaded += bytes_read as u64;
|
|
pb.set_position(downloaded);
|
|
}
|
|
Ok(())
|
|
},
|
|
) {
|
|
Ok(_) => {
|
|
retrieved = true;
|
|
break;
|
|
}
|
|
Err(_) => continue,
|
|
}
|
|
}
|
|
ftp_stream.quit().ok();
|
|
if !retrieved {
|
|
bail!("FTP mirror error fetching {}", alt);
|
|
}
|
|
} else {
|
|
// HTTP(S) mirror retry (recreate client for retry)
|
|
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
|
|
let client = super::build_blocking_client(&ua, None)
|
|
.with_context(|| "Failed to build HTTP client")?;
|
|
let mut response = client
|
|
.get(&alt)
|
|
.send()
|
|
.with_context(|| format!("Failed to fetch mirror URL: {}", alt))?;
|
|
|
|
let status = response.status();
|
|
if !status.is_success() {
|
|
let mut preview_bytes = Vec::new();
|
|
let _ = response.take(1024).read_to_end(&mut preview_bytes);
|
|
let preview = String::from_utf8_lossy(&preview_bytes);
|
|
bail!(
|
|
"HTTP error fetching {}: {}{}",
|
|
alt,
|
|
status,
|
|
if preview.trim().is_empty() {
|
|
"".to_string()
|
|
} else {
|
|
format!(" — preview: {}", preview.trim())
|
|
}
|
|
);
|
|
}
|
|
|
|
let mut file = File::create(&dest_path)
|
|
.with_context(|| format!("Failed to create: {}", dest_path.display()))?;
|
|
let mut buffer = [0u8; 8192];
|
|
let mut downloaded = 0u64;
|
|
loop {
|
|
let bytes_read = response.read(&mut buffer)?;
|
|
if bytes_read == 0 {
|
|
break;
|
|
}
|
|
file.write_all(&buffer[..bytes_read])?;
|
|
downloaded += bytes_read as u64;
|
|
pb.set_position(downloaded);
|
|
}
|
|
}
|
|
} else {
|
|
// Fallback: unknown/malformed alt URL -> bail
|
|
bail!("Malformed mirror URL: {}", alt);
|
|
}
|
|
|
|
pb.finish_with_message("Download complete (mirror)");
|
|
next_alt = validate_downloaded_archive(&dest_path, &filename, &alt)?;
|
|
}
|
|
|
|
// Verify checksum
|
|
if !verify_checksum(&dest_path, &source.sha256)? {
|
|
fs::remove_file(&dest_path)?;
|
|
bail!("Checksum verification failed for {}", filename);
|
|
}
|
|
|
|
crate::log_info!("Checksum verified: {}", filename);
|
|
Ok(dest_path)
|
|
}
|
|
|
|
/// Verify checksum of a file supporting optional algorithm prefix.
|
|
///
|
|
/// Supported formats:
|
|
/// - `sha256:<hex>` (or just `<hex>` — default)
|
|
/// - `sha512:<hex>`
|
|
/// - `md5:<hex>`
|
|
/// - `b2:<hex>` / `b2sum:<hex>`
|
|
/// - `skip` to bypass verification
|
|
fn verify_checksum(path: &Path, expected: &str) -> Result<bool> {
|
|
// Delegate to the shared checker in the parent `source` module.
|
|
// The helper also understands the same string forms and "skip".
|
|
super::verify_file_hash(path, expected)
|
|
}
|
|
|
|
/// Derive a stable filename from a URL.
|
|
///
|
|
/// Rules:
|
|
/// - Parse as URL and use the last path segment if it looks like a filename (contains a dot)
|
|
/// - Otherwise fall back to a stable hash-based name: source-{sha256(url)[..12]}.download
|
|
pub(crate) fn derive_filename_from_url(url: &str) -> String {
|
|
// try to parse the URL
|
|
if let Some(last) = Url::parse(url).ok().and_then(|parsed| {
|
|
parsed
|
|
.path_segments()?
|
|
.rfind(|s| !s.is_empty())
|
|
.filter(|l| l.contains('.'))
|
|
.map(|l| l.to_string())
|
|
}) {
|
|
return last;
|
|
}
|
|
|
|
// fallback: stable short-hash name
|
|
let mut hasher = Sha256::new();
|
|
hasher.update(url.as_bytes());
|
|
let h = hasher.finalize();
|
|
let hex = format!("{:x}", h);
|
|
format!("source-{}.download", &hex[..12])
|
|
}
|
|
|
|
/// Validate downloaded file's magic header to make sure it is the expected
|
|
/// archive format (avoids saving HTML pages or other unexpected content).
|
|
fn validate_downloaded_archive(
|
|
path: &std::path::Path,
|
|
filename: &str,
|
|
orig_url: &str,
|
|
) -> Result<Option<String>> {
|
|
use std::io::Read;
|
|
let mut f = std::fs::File::open(path)?;
|
|
let mut buf = [0u8; 4096];
|
|
let n = f.read(&mut buf)?;
|
|
let head = &buf[..n.min(4096)];
|
|
|
|
// Detect obvious HTML error pages (case-insensitive)
|
|
if is_html_content(head) {
|
|
if url_contains_sourceforge_host(orig_url) {
|
|
let body = std::fs::read(path)
|
|
.map(|bytes| String::from_utf8_lossy(&bytes).to_string())
|
|
.unwrap_or_else(|_| String::from_utf8_lossy(head).to_string());
|
|
if let Some(alt) = sourceforge_alt_url_from_html(&body, orig_url) {
|
|
return Ok(Some(alt));
|
|
}
|
|
}
|
|
|
|
anyhow::bail!(
|
|
"Downloaded file '{}' looks like HTML (not an archive). Preview: {}",
|
|
filename,
|
|
html_preview(head)
|
|
);
|
|
}
|
|
|
|
// Validate by extension (best-effort)
|
|
let lower = filename.to_ascii_lowercase();
|
|
let is_ok = classify_archive_magic(head, &lower, path);
|
|
|
|
if !is_ok {
|
|
anyhow::bail!(
|
|
"Downloaded file '{}' does not match expected archive magic; preview: {}",
|
|
filename,
|
|
html_preview(head)
|
|
);
|
|
}
|
|
|
|
Ok(None)
|
|
}
|
|
|
|
fn sourceforge_alt_url_from_html(body: &str, orig_url: &str) -> Option<String> {
|
|
let lower = body.to_ascii_lowercase();
|
|
for href in extract_hrefs(body, &lower) {
|
|
if href.contains("downloads.sourceforge.net")
|
|
&& let Some(url) = sourceforge_candidate_from_href(&href)
|
|
{
|
|
return Some(url);
|
|
}
|
|
}
|
|
for href in extract_hrefs(body, &lower) {
|
|
if href.contains("/download")
|
|
&& let Some(url) = sourceforge_candidate_from_href(&href)
|
|
{
|
|
return Some(url);
|
|
}
|
|
}
|
|
sourceforge_download_fallback(orig_url)
|
|
}
|
|
|
|
fn extract_hrefs(body: &str, lower: &str) -> Vec<String> {
|
|
let mut out = Vec::new();
|
|
let mut i = 0usize;
|
|
while let Some(rel) = lower[i..].find("href=") {
|
|
let start = i + rel + 5;
|
|
if start >= body.len() {
|
|
break;
|
|
}
|
|
let quote = body.as_bytes()[start] as char;
|
|
if quote != '"' && quote != '\'' {
|
|
i = start + 1;
|
|
continue;
|
|
}
|
|
let val_start = start + 1;
|
|
if val_start > body.len() {
|
|
break;
|
|
}
|
|
if let Some(end_rel) = body[val_start..].find(quote) {
|
|
out.push(body[val_start..val_start + end_rel].to_string());
|
|
i = val_start + end_rel + 1;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
fn normalize_downloads_sf_href(href: &str) -> String {
|
|
let href = href.replace("&", "&");
|
|
if href.starts_with("//") {
|
|
format!("https:{}", href)
|
|
} else if href.starts_with('/') {
|
|
format!("https://downloads.sourceforge.net{}", href)
|
|
} else {
|
|
href
|
|
}
|
|
}
|
|
|
|
fn normalize_sf_href(href: &str) -> String {
|
|
let href = href.replace("&", "&");
|
|
if href.starts_with("//") {
|
|
format!("https:{}", href)
|
|
} else if href.starts_with('/') {
|
|
format!("https://sourceforge.net{}", href)
|
|
} else {
|
|
href
|
|
}
|
|
}
|
|
|
|
fn sourceforge_candidate_from_href(href: &str) -> Option<String> {
|
|
let normalized = if href.contains("downloads.sourceforge.net") {
|
|
normalize_downloads_sf_href(href)
|
|
} else {
|
|
normalize_sf_href(href)
|
|
};
|
|
let parsed = Url::parse(&normalized).ok()?;
|
|
let host = parsed.host_str()?.to_ascii_lowercase();
|
|
|
|
if is_sourceforge_host(&host) || is_downloads_sourceforge_host(&host) {
|
|
return Some(normalized);
|
|
}
|
|
|
|
// Some HTML pages use social share links that embed a real SourceForge URL
|
|
// in a query parameter (e.g. x.com/share?url=...).
|
|
if is_social_share_host(&host) {
|
|
for (k, v) in parsed.query_pairs() {
|
|
if (k.eq_ignore_ascii_case("url") || k.eq_ignore_ascii_case("u"))
|
|
&& let Ok(inner) = Url::parse(v.as_ref())
|
|
&& let Some(inner_host) = inner.host_str()
|
|
{
|
|
let inner_host = inner_host.to_ascii_lowercase();
|
|
if is_sourceforge_host(&inner_host) || is_downloads_sourceforge_host(&inner_host) {
|
|
return Some(inner.to_string());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
fn sourceforge_download_fallback(orig_url: &str) -> Option<String> {
|
|
let parsed = Url::parse(orig_url).ok()?;
|
|
let host = parsed.host_str()?.to_ascii_lowercase();
|
|
|
|
if is_sourceforge_host(&host) || is_downloads_sourceforge_host(&host) {
|
|
return Some(format!("{}/download", orig_url.trim_end_matches('/')));
|
|
}
|
|
|
|
if is_social_share_host(&host) {
|
|
for (k, v) in parsed.query_pairs() {
|
|
if (k.eq_ignore_ascii_case("url") || k.eq_ignore_ascii_case("u"))
|
|
&& let Ok(inner) = Url::parse(v.as_ref())
|
|
&& let Some(inner_host) = inner.host_str()
|
|
{
|
|
let inner_host = inner_host.to_ascii_lowercase();
|
|
if is_sourceforge_host(&inner_host) || is_downloads_sourceforge_host(&inner_host) {
|
|
return Some(format!("{}/download", inner.as_str().trim_end_matches('/')));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
fn is_sourceforge_host(host: &str) -> bool {
|
|
host == "sourceforge.net" || host.ends_with(".sourceforge.net")
|
|
}
|
|
|
|
fn is_downloads_sourceforge_host(host: &str) -> bool {
|
|
host == "downloads.sourceforge.net" || host.ends_with(".downloads.sourceforge.net")
|
|
}
|
|
|
|
fn is_social_share_host(host: &str) -> bool {
|
|
matches!(
|
|
host,
|
|
"x.com"
|
|
| "www.x.com"
|
|
| "twitter.com"
|
|
| "www.twitter.com"
|
|
| "facebook.com"
|
|
| "www.facebook.com"
|
|
| "linkedin.com"
|
|
| "www.linkedin.com"
|
|
| "reddit.com"
|
|
| "www.reddit.com"
|
|
| "t.me"
|
|
| "telegram.me"
|
|
)
|
|
}
|
|
|
|
fn url_contains_sourceforge_host(url: &str) -> bool {
|
|
Url::parse(url)
|
|
.ok()
|
|
.and_then(|u| u.host_str().map(|h| h.to_ascii_lowercase()))
|
|
.map(|h| is_sourceforge_host(&h) || is_downloads_sourceforge_host(&h))
|
|
.unwrap_or_else(|| {
|
|
url.contains("://sourceforge.net/")
|
|
|| url.contains("://downloads.sourceforge.net/")
|
|
|| url.contains(".sourceforge.net/")
|
|
})
|
|
}
|
|
|
|
fn html_preview(head: &[u8]) -> String {
|
|
String::from_utf8_lossy(&head[..head.len().min(1024)])
|
|
.trim()
|
|
.to_string()
|
|
}
|
|
|
|
fn is_html_content(head: &[u8]) -> bool {
|
|
let head_str = String::from_utf8_lossy(head).to_ascii_lowercase();
|
|
head_str.starts_with("<!doctype html")
|
|
|| head_str.starts_with("<html")
|
|
|| head_str.contains("<html")
|
|
}
|
|
|
|
fn classify_archive_magic(head: &[u8], lower_filename: &str, path: &Path) -> bool {
|
|
if lower_filename.ends_with(".tar.xz")
|
|
|| lower_filename.ends_with(".txz")
|
|
|| lower_filename.ends_with(".xz")
|
|
{
|
|
head.starts_with(&[0xFD, b'7', b'z', b'X', b'Z', 0x00])
|
|
} else if lower_filename.ends_with(".tar.gz")
|
|
|| lower_filename.ends_with(".tgz")
|
|
|| lower_filename.ends_with(".gz")
|
|
{
|
|
head.starts_with(&[0x1F, 0x8B])
|
|
} else if lower_filename.ends_with(".tar.zst")
|
|
|| lower_filename.ends_with(".tzst")
|
|
|| lower_filename.ends_with(".zst")
|
|
{
|
|
head.starts_with(&[0x28, 0xB5, 0x2F, 0xFD])
|
|
} else if lower_filename.ends_with(".zip") {
|
|
head.starts_with(b"PK\x03\x04")
|
|
} else if lower_filename.ends_with(".tar.bz2") || lower_filename.ends_with(".tbz2") {
|
|
head.starts_with(&[0x42, 0x5A, 0x68])
|
|
} else if lower_filename.ends_with(".tar") {
|
|
if let Ok(mut f2) = std::fs::File::open(path) {
|
|
let mut hdr = [0u8; 262];
|
|
if f2.read_exact(&mut hdr).is_ok() {
|
|
&hdr[257..262] == b"ustar"
|
|
} else {
|
|
true
|
|
}
|
|
} else {
|
|
true
|
|
}
|
|
} else if lower_filename.ends_with(".deb") {
|
|
head.starts_with(b"!<arch>")
|
|
} else {
|
|
// rpm/unknown extensions: keep permissive as before.
|
|
true
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn filename_from_simple_url() {
|
|
assert_eq!(
|
|
derive_filename_from_url("https://example.com/foo-1.2.3.tar.gz"),
|
|
"foo-1.2.3.tar.gz"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn filename_from_url_with_query() {
|
|
assert_eq!(
|
|
derive_filename_from_url("https://example.com/foo.tar.gz?raw=1"),
|
|
"foo.tar.gz"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn filename_from_url_without_real_name() {
|
|
let name = derive_filename_from_url("https://github.com/org/repo/releases/download?id=123");
|
|
assert!(name.starts_with("source-") && name.ends_with(".download"));
|
|
}
|
|
|
|
#[test]
|
|
fn filename_from_ftp_url() {
|
|
assert_eq!(
|
|
derive_filename_from_url("ftp://example.com/foo-1.2.3.tar.gz"),
|
|
"foo-1.2.3.tar.gz"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn filename_from_non_url_string() {
|
|
let name = derive_filename_from_url("not-a-url-at-all");
|
|
assert!(name.starts_with("source-") && name.ends_with(".download"));
|
|
}
|
|
|
|
#[test]
|
|
fn sourceforge_html_no_link_falls_back_to_download_suffix() {
|
|
use std::io::Write;
|
|
let mut tmp = tempfile::NamedTempFile::new().unwrap();
|
|
write!(
|
|
tmp,
|
|
"<!doctype html><html><body>No direct link</body></html>"
|
|
)
|
|
.unwrap();
|
|
let alt = validate_downloaded_archive(
|
|
tmp.path(),
|
|
"zsh-5.9.tar.xz",
|
|
"https://sourceforge.net/projects/zsh/files/zsh/5.9/zsh-5.9.tar.xz",
|
|
)
|
|
.unwrap();
|
|
assert_eq!(
|
|
alt,
|
|
Some(
|
|
"https://sourceforge.net/projects/zsh/files/zsh/5.9/zsh-5.9.tar.xz/download"
|
|
.to_string()
|
|
)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn sourceforge_html_single_quoted_href_extracts_downloads_link() {
|
|
use std::io::Write;
|
|
let mut tmp = tempfile::NamedTempFile::new().unwrap();
|
|
write!(tmp, "<!doctype html><a href='//downloads.sourceforge.net/project/zsh/zsh/5.9/zsh-5.9.tar.xz?download'>download</a>").unwrap();
|
|
let alt = validate_downloaded_archive(
|
|
tmp.path(),
|
|
"zsh-5.9.tar.xz",
|
|
"https://sourceforge.net/projects/zsh/files/zsh/5.9/zsh-5.9.tar.xz",
|
|
)
|
|
.unwrap();
|
|
assert_eq!(
|
|
alt,
|
|
Some(
|
|
"https://downloads.sourceforge.net/project/zsh/zsh/5.9/zsh-5.9.tar.xz?download"
|
|
.to_string()
|
|
)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn sourceforge_html_large_page_extracts_downloads_link_beyond_4k() {
|
|
use std::io::Write;
|
|
let mut tmp = tempfile::NamedTempFile::new().unwrap();
|
|
write!(
|
|
tmp,
|
|
"<!doctype html><html><body>{}<a href='//downloads.sourceforge.net/project/tcl/tcl8.6.17-src.tar.gz?download'>download</a></body></html>",
|
|
"x".repeat(9000)
|
|
)
|
|
.unwrap();
|
|
let alt = validate_downloaded_archive(
|
|
tmp.path(),
|
|
"tcl8.6.17-src.tar.gz",
|
|
"https://sourceforge.net/projects/tcl/files/tcl8.6.17-src.tar.gz",
|
|
)
|
|
.unwrap();
|
|
assert_eq!(
|
|
alt,
|
|
Some(
|
|
"https://downloads.sourceforge.net/project/tcl/tcl8.6.17-src.tar.gz?download"
|
|
.to_string()
|
|
)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn sourceforge_html_ignores_social_share_links_and_unwraps_url_param() {
|
|
use std::io::Write;
|
|
let mut tmp = tempfile::NamedTempFile::new().unwrap();
|
|
write!(
|
|
tmp,
|
|
"<!doctype html><a href='https://x.com/share?url=https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz/download&text=share'>share</a>"
|
|
)
|
|
.unwrap();
|
|
let alt = validate_downloaded_archive(
|
|
tmp.path(),
|
|
"tcl8.6.17-src.tar.gz",
|
|
"https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz",
|
|
)
|
|
.unwrap();
|
|
assert_eq!(
|
|
alt,
|
|
Some(
|
|
"https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz/download"
|
|
.to_string()
|
|
)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn sourceforge_share_url_fallback_uses_embedded_sourceforge_url() {
|
|
let fallback = sourceforge_download_fallback(
|
|
"https://x.com/share?url=https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz",
|
|
);
|
|
assert_eq!(
|
|
fallback,
|
|
Some(
|
|
"https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz/download"
|
|
.to_string()
|
|
)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn verify_checksum_accepts_md5_sha512_b2sum_and_default_sha256() {
|
|
use sha2::Digest;
|
|
use sha2::Sha256;
|
|
use sha2::Sha512;
|
|
|
|
let tmp = tempfile::NamedTempFile::new().unwrap();
|
|
std::fs::write(tmp.path(), b"abc").unwrap();
|
|
|
|
// compute expected values using the same libraries
|
|
let sha256_hex = {
|
|
let mut h = Sha256::new();
|
|
h.update(b"abc");
|
|
format!("{:x}", h.finalize())
|
|
};
|
|
|
|
let sha512_hex = {
|
|
let mut h = Sha512::new();
|
|
h.update(b"abc");
|
|
format!("{:x}", h.finalize())
|
|
};
|
|
|
|
let md5_hex = format!("{:x}", md5::compute(b"abc"));
|
|
let b2_hex = b2sum_rust::Blake2bSum::new(64)
|
|
.read(tmp.path())
|
|
.to_ascii_lowercase();
|
|
|
|
// unprefixed should default to sha256
|
|
assert!(verify_checksum(tmp.path(), &sha256_hex).unwrap());
|
|
// explicit prefixes
|
|
assert!(verify_checksum(tmp.path(), &format!("sha256:{}", sha256_hex)).unwrap());
|
|
assert!(verify_checksum(tmp.path(), &format!("sha512:{}", sha512_hex)).unwrap());
|
|
assert!(verify_checksum(tmp.path(), &format!("md5:{}", md5_hex)).unwrap());
|
|
assert!(verify_checksum(tmp.path(), &format!("b2:{}", b2_hex)).unwrap());
|
|
assert!(verify_checksum(tmp.path(), &format!("b2sum:{}", b2_hex)).unwrap());
|
|
// empty algorithm before colon -> assume sha256
|
|
assert!(verify_checksum(tmp.path(), &format!(":{}", sha256_hex)).unwrap());
|
|
// negative: wrong value fails
|
|
assert!(!verify_checksum(tmp.path(), "md5:deadbeef").unwrap());
|
|
}
|
|
}
|