SFG545/depot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement build state tracking and interactive package specification creation

Browse Source 
- Added StateTracker to manage build steps and allow resuming interrupted builds.
- Updated post_extract function to utilize StateTracker for patch and command execution.
- Introduced makefile.rs to handle building and installing packages via Makefile.
- Created interactive.rs for an interactive package specification creator with SHA256 computation for source URLs.
- Enhanced checksum verification to support multiple algorithms (SHA256, SHA512, MD5).
- Added example configuration files in contrib for system-wide and user-level Depot configurations.
- Updated tests to cover new functionality and ensure correctness of state tracking and interactive creation.
This commit is contained in:
SFG545
2026-02-16 21:38:05 -06:00
parent 00ca2ebac6
commit 9a00608104
30 changed files with 3501 additions and 434 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/source/extractor.rs
+309 -38
View File
@@ -7,6 +7,8 @@ use std::fs::{self, File};
use std::io::{Cursor, Read, Write};
use std::os::unix::fs as unix_fs;
use std::path::{Path, PathBuf};
use tempfile::{tempdir, NamedTempFile};
use walkdir::WalkDir;
use zstd::stream::read::Decoder as ZstdDecoder;
/// Extract an archive source to the build directory.
@@ -36,34 +38,34 @@ pub fn extract_archive(
println!("Extracting: {}", filename);
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
extract_tar_gz(archive_path, build_dir)?;
extract_tar_gz(archive_path, &extract_path)?;
} else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") {
extract_tar_xz(archive_path, build_dir)?;
extract_tar_xz(archive_path, &extract_path)?;
} else if filename.ends_with(".tar.bz2") || filename.ends_with(".tbz2") {
extract_tar_bz2(archive_path, build_dir)?;
extract_tar_bz2(archive_path, &extract_path)?;
} else if filename.ends_with(".tar.zst") || filename.ends_with(".tzst") {
extract_tar_zst(archive_path, build_dir)?;
extract_tar_zst(archive_path, &extract_path)?;
} else if filename.ends_with(".zip") {
extract_zip(archive_path, build_dir)?;
extract_zip(archive_path, &extract_path)?;
} else if filename.ends_with(".tar") {
extract_tar(archive_path, build_dir)?;
extract_tar(archive_path, &extract_path)?;
} else if filename.ends_with(".deb") {
extract_deb(archive_path, build_dir)?;
extract_deb(archive_path, &extract_path)?;
} else if filename.ends_with(".rpm") {
extract_rpm(archive_path, build_dir)?;
extract_rpm(archive_path, &extract_path)?;
} else if filename.ends_with(".gz") {
extract_gz_file(archive_path, build_dir)?;
extract_gz_file(archive_path, &extract_path)?;
} else if filename.ends_with(".xz") {
extract_xz_file(archive_path, build_dir)?;
extract_xz_file(archive_path, &extract_path)?;
} else if filename.ends_with(".zst") {
extract_zst_file(archive_path, build_dir)?;
extract_zst_file(archive_path, &extract_path)?;
} else {
bail!("Unsupported archive format: {}", filename);
}
if !extract_path.exists() {
bail!(
"Expected extraction directory not found: {}",
"Extraction did not create expected path: {}",
extract_path.display()
);
}
@@ -73,48 +75,274 @@ pub fn extract_archive(
}
fn extract_tar_gz(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let decoder = GzDecoder::new(file);
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
archive.unpack(tmp.path())?;
// If the archive produced a single top-level directory, decide whether to
// strip that top directory (source tarballs like foo-1.2.3/) or preserve
// it (system-layout archives like usr/). Otherwise move all top-level
// entries into `dest` so `dest` always contains the source root.
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(&top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(&top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
}
fn extract_tar_xz(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let decoder = xz2::read::XzDecoder::new(file);
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(&top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(&top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
}
fn extract_tar_bz2(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let decoder = bzip2::read::BzDecoder::new(file);
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(&top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(&top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
}
fn extract_tar(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let mut archive = tar::Archive::new(file);
archive.unpack(dest)?;
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(&top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(&top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
}
fn extract_zip(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let mut archive = zip::ZipArchive::new(file)?;
archive.extract(dest)?;
archive.extract(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(&top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(&top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
}
fn extract_tar_zst(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let decoder = ZstdDecoder::new(file)?;
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(&top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(&top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
}
@@ -186,38 +414,32 @@ fn extract_deb(path: &Path, dest: &Path) -> Result<()> {
let file = File::open(path)?;
let mut ar = ar::Archive::new(file);
// Iterate members and look for data.tar, data.tar.gz, data.tar.xz, data.tar.zst
while let Some(entry_result) = ar.next_entry() {
let entry = entry_result?;
let mut entry = entry_result?;
let id = String::from_utf8_lossy(entry.header().identifier()).to_string();
let lower = id.to_ascii_lowercase();
if lower.starts_with("data.tar") {
// Determine compression
// write the inner member to a temporary file and reuse tar extraction logic
let mut tmpf = NamedTempFile::new()?;
std::io::copy(&mut entry, &mut tmpf)?;
let tmp_path = tmpf.path().to_path_buf();
if lower.ends_with(".gz") {
let decoder = GzDecoder::new(entry);
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
extract_tar_gz(&tmp_path, dest)?;
return Ok(());
} else if lower.ends_with(".xz") {
let decoder = xz2::read::XzDecoder::new(entry);
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
extract_tar_xz(&tmp_path, dest)?;
return Ok(());
} else if lower.ends_with(".zst") {
let decoder = ZstdDecoder::new(entry)?;
let mut archive = tar::Archive::new(decoder);
archive.unpack(dest)?;
extract_tar_zst(&tmp_path, dest)?;
return Ok(());
} else {
// plain tar
let mut archive = tar::Archive::new(entry);
archive.unpack(dest)?;
extract_tar(&tmp_path, dest)?;
return Ok(());
}
}
}
// No data member found
anyhow::bail!("No data.tar.* member found in deb: {}", path.display());
}
@@ -334,27 +556,34 @@ fn extract_rpm(path: &Path, dest: &Path) -> Result<()> {
let zst_sig = b"\x28\xb5\x2f\xfd";
let cpio_sig = b"070701";
// Extract into a temporary directory first, then move/strip into `dest`.
let tmp = tempdir()?;
if let Some(pos) = find_subslice(&data, gz_sig) {
let cursor = Cursor::new(&data[pos..]);
let decoder = GzDecoder::new(cursor);
extract_cpio_newc_from_reader(decoder, dest)?;
extract_cpio_newc_from_reader(decoder, tmp.path())?;
move_dir_contents(tmp.path(), dest)?;
return Ok(());
}
if let Some(pos) = find_subslice(&data, xz_sig) {
let cursor = Cursor::new(&data[pos..]);
let decoder = xz2::read::XzDecoder::new(cursor);
extract_cpio_newc_from_reader(decoder, dest)?;
extract_cpio_newc_from_reader(decoder, tmp.path())?;
move_dir_contents(tmp.path(), dest)?;
return Ok(());
}
if let Some(pos) = find_subslice(&data, zst_sig) {
let cursor = Cursor::new(&data[pos..]);
let decoder = ZstdDecoder::new(cursor)?;
extract_cpio_newc_from_reader(decoder, dest)?;
extract_cpio_newc_from_reader(decoder, tmp.path())?;
move_dir_contents(tmp.path(), dest)?;
return Ok(());
}
if let Some(pos) = find_subslice(&data, cpio_sig) {
let cursor = Cursor::new(&data[pos..]);
extract_cpio_newc_from_reader(cursor, dest)?;
extract_cpio_newc_from_reader(cursor, tmp.path())?;
move_dir_contents(tmp.path(), dest)?;
return Ok(());
}
@@ -368,6 +597,48 @@ fn find_subslice(hay: &[u8], needle: &[u8]) -> Option<usize> {
hay.windows(needle.len()).position(|w| w == needle)
}
/// Move contents of `src` into `dest`. If `src` contains exactly one directory
/// and `strip_single_top` is intended, callers can pass the inner dir instead.
fn move_dir_contents(src: &Path, dest: &Path) -> Result<()> {
fs::create_dir_all(dest)?;
for entry in fs::read_dir(src)? {
let entry = entry?;
let src_path = entry.path();
let dest_path = dest.join(entry.file_name());
if fs::rename(&src_path, &dest_path).is_err() {
// fallback to copy-and-remove across filesystems
if src_path.is_dir() {
copy_dir_recursive_local(&src_path, &dest_path)?;
fs::remove_dir_all(&src_path)?;
} else {
fs::copy(&src_path, &dest_path)?;
fs::remove_file(&src_path)?;
}
}
}
Ok(())
}
fn copy_dir_recursive_local(src: &Path, dst: &Path) -> Result<()> {
for entry in WalkDir::new(src) {
let entry = entry?;
let rel = entry.path().strip_prefix(src).unwrap();
let target = dst.join(rel);
if entry.file_type().is_dir() {
fs::create_dir_all(&target)?;
} else if entry.file_type().is_symlink() {
let target_link = fs::read_link(entry.path())?;
unix_fs::symlink(target_link, &target)?;
} else {
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
fs::copy(entry.path(), &target)?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
src/source/fetcher.rs
+368 -25
View File
@@ -27,13 +27,82 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
println!("Fetching: {}", url);
// Parse URL early so we can handle non-HTTP schemes (FTP support)
let parsed_url = Url::parse(&url).with_context(|| format!("Invalid URL: {}", url))?;
// If this is an FTP URL, fetch via the ftp crate into the cache and continue
if parsed_url.scheme() == "ftp" {
// Connect and login (anonymous fallback)
let host = parsed_url.host_str().context("FTP URL missing host")?;
let port = parsed_url.port_or_known_default().unwrap_or(21);
let addr = format!("{}:{}", host, port);
let mut ftp_stream = ftp::FtpStream::connect(addr.as_str())
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
let user = if parsed_url.username().is_empty() { "anonymous" } else { parsed_url.username() };
let pass = parsed_url.password().unwrap_or("anonymous@");
ftp_stream.login(user, pass).with_context(|| format!("FTP login failed for {}", host))?;
// Retrieve the path (try with and without leading slash)
let path = parsed_url.path();
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
let mut retrieved = false;
for p in candidates.iter().filter(|s| !s.is_empty()) {
match ftp_stream.retr(p, |reader: &mut dyn Read| -> std::result::Result<(), ftp::FtpError> {
let mut file = File::create(&dest_path).map_err(ftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
loop {
let bytes_read = reader.read(&mut buffer).map_err(ftp::FtpError::ConnectionError)?;
if bytes_read == 0 { break; }
file.write_all(&buffer[..bytes_read]).map_err(ftp::FtpError::ConnectionError)?;
}
Ok(())
}) {
Ok(_) => {
retrieved = true;
break;
}
Err(_) => continue,
}
}
ftp_stream.quit().ok();
if !retrieved {
bail!("FTP error fetching {}", url);
}
}
// Download with progress bar
let client = reqwest::blocking::Client::new();
// Use a sensible default User-Agent so servers that reject empty/unknown agents (e.g. IANA)
// will accept requests. Include package name/version at compile time.
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
let client = reqwest::blocking::Client::builder()
.user_agent(ua)
.build()
.with_context(|| "Failed to build HTTP client")?;
let mut response = client
.get(&url)
.send()
.with_context(|| format!("Failed to fetch: {}", url))?;
// If the server returned a non-success status, read a short body preview and fail early.
// This prevents saving HTML error pages (which then fail checksum) and gives a clearer
// diagnostic to the user.
let status = response.status();
if !status.is_success() {
let mut preview_bytes = Vec::new();
// read up to 1 KiB for a preview (ignore errors while reading preview)
let _ = response.take(1024).read_to_end(&mut preview_bytes);
let preview = String::from_utf8_lossy(&preview_bytes);
bail!(
"HTTP error fetching {}: {}{}",
url,
status,
if preview.trim().is_empty() {
"".to_string()
} else {
format!(" — preview: {}", preview.trim())
}
);
}
let total_size = response.content_length().unwrap_or(0);
let pb = ProgressBar::new(total_size);
pb.set_style(
@@ -61,6 +130,92 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
pb.finish_with_message("Download complete");
// Quick validation: ensure the downloaded file looks like the expected
// archive (detect obvious HTML error pages or wrong formats by magic).
// If validate_downloaded_archive returns an alternate URL (e.g. SourceForge
// mirror), retry the download with that URL once.
if let Some(alt) = validate_downloaded_archive(&dest_path, &filename, &url)? {
println!("Retrying download from mirror: {}", alt);
fs::remove_file(&dest_path).ok();
// If mirror URL is FTP -> use ftp crate; otherwise use HTTP retry.
if let Ok(alt_url) = Url::parse(&alt) {
if alt_url.scheme() == "ftp" {
// FTP mirror retrieval
let host = alt_url.host_str().context("FTP mirror URL missing host")?;
let port = alt_url.port_or_known_default().unwrap_or(21);
let addr = format!("{}:{}", host, port);
let mut ftp_stream = ftp::FtpStream::connect(addr.as_str())
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
let user = if alt_url.username().is_empty() { "anonymous" } else { alt_url.username() };
let pass = alt_url.password().unwrap_or("anonymous@");
ftp_stream.login(user, pass).with_context(|| format!("FTP login failed for {}", host))?;
let path = alt_url.path();
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
let mut retrieved = false;
for p in candidates.iter().filter(|s| !s.is_empty()) {
match ftp_stream.retr(p, |reader: &mut dyn Read| -> std::result::Result<(), ftp::FtpError> {
let mut file = File::create(&dest_path).map_err(ftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = reader.read(&mut buffer).map_err(ftp::FtpError::ConnectionError)?;
if bytes_read == 0 { break; }
file.write_all(&buffer[..bytes_read]).map_err(ftp::FtpError::ConnectionError)?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
Ok(())
}) {
Ok(_) => {
retrieved = true;
break;
}
Err(_) => continue,
}
}
ftp_stream.quit().ok();
if !retrieved {
bail!("FTP mirror error fetching {}", alt);
}
} else {
// HTTP(S) mirror retry (recreate client for retry)
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
let client = reqwest::blocking::Client::builder()
.user_agent(ua)
.build()
.with_context(|| "Failed to build HTTP client")?;
let mut response = client
.get(&alt)
.send()
.with_context(|| format!("Failed to fetch mirror URL: {}", alt))?;
let mut file = File::create(&dest_path)
.with_context(|| format!("Failed to create: {}", dest_path.display()))?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = response.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
}
} else {
// Fallback: unknown/malformed alt URL -> bail
bail!("Malformed mirror URL: {}", alt);
}
pb.finish_with_message("Download complete (mirror)");
// Re-validate the mirrored file
validate_downloaded_archive(&dest_path, &filename, &alt)?;
}
// Verify checksum
if !verify_checksum(&dest_path, &source.sha256)? {
fs::remove_file(&dest_path)?;
@@ -71,32 +226,20 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
Ok(dest_path)
}
/// Verify SHA256 checksum of a file
/// Verify checksum of a file supporting optional algorithm prefix.
///
/// Supported formats:
/// - `sha256:<hex>` (or just `<hex>` — default)
/// - `sha512:<hex>`
/// - `md5:<hex>`
/// - `skip` to bypass verification
fn verify_checksum(path: &Path, expected: &str) -> Result<bool> {
// Skip verification if requested
if expected.to_lowercase() == "skip" {
println!("Checksum verification skipped");
return Ok(true);
}
let mut file = File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 8192];
loop {
let bytes_read = file.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
hasher.update(&buffer[..bytes_read]);
}
let result = hasher.finalize();
let actual = format!("{:x}", result);
Ok(actual == expected.to_lowercase())
// Delegate to the shared checker in the parent `source` module.
// The helper also understands the same string forms and "skip".
super::verify_file_hash(path, expected)
}
/// Derive a stable filename from a URL.
///
/// Rules:
@@ -122,6 +265,126 @@ fn derive_filename_from_url(url: &str) -> String {
format!("source-{}.download", &hex[..12])
}
/// Validate downloaded file's magic header to make sure it is the expected
/// archive format (avoids saving HTML pages or other unexpected content).
fn validate_downloaded_archive(path: &std::path::Path, filename: &str, orig_url: &str) -> Result<Option<String>> {
use std::io::Read;
let mut f = std::fs::File::open(path)?;
let mut buf = [0u8; 4096];
let n = f.read(&mut buf)?;
let head = &buf[..n.min(4096)];
// Detect obvious HTML error pages (case-insensitive)
let head_str = String::from_utf8_lossy(head).to_ascii_lowercase();
if head_str.starts_with("<!doctype html") || head_str.starts_with("<html") || head_str.contains("<html") {
// If this came from SourceForge project URL, try to extract a direct
// downloads.sourceforge.net mirror link and return it for a retry.
let body = String::from_utf8_lossy(head).to_string();
if orig_url.contains("sourceforge.net") {
// helper: find nearest href attribute before `pos` and support
// both double- and single-quoted attributes.
let find_href_before = |body: &str, pos: usize| -> Option<String> {
if let Some(start) = body[..pos].rfind("href=\"") {
let rest = &body[start + 6..];
if let Some(end) = rest.find('"') {
return Some(rest[..end].to_string());
}
}
if let Some(start) = body[..pos].rfind("href='") {
let rest = &body[start + 6..];
if let Some(end) = rest.find('\'') {
return Some(rest[..end].to_string());
}
}
None
};
// look for downloads.sourceforge.net links in HTML
if let Some(pos) = body.find("downloads.sourceforge.net") {
if let Some(href) = find_href_before(&body, pos) {
let alt = if href.starts_with("//") {
format!("https:{}", href)
} else if href.starts_with("/") {
format!("https://downloads.sourceforge.net{}", href)
} else {
href
};
return Ok(Some(alt));
}
}
// Also try to find '/download' link anywhere in the body
if let Some(pos) = body.find("/download") {
if let Some(href) = find_href_before(&body, pos) {
let alt = if href.starts_with("//") {
format!("https:{}", href)
} else {
href
};
return Ok(Some(alt));
}
}
// Final fallback for SF project pages: append '/download' to the
// original URL (this normally triggers the mirror redirect).
return Ok(Some(format!("{}/download", orig_url.trim_end_matches('/'))));
}
let preview = String::from_utf8_lossy(&head[..head.len().min(1024)]);
anyhow::bail!(
"Downloaded file '{}' looks like HTML (not an archive). Preview: {}",
filename,
preview.trim()
);
}
// Validate by extension (best-effort)
let lower = filename.to_ascii_lowercase();
let is_ok = if lower.ends_with(".tar.xz") || lower.ends_with(".txz") || lower.ends_with(".xz") {
head.starts_with(&[0xFD, b'7', b'z', b'X', b'Z', 0x00])
} else if lower.ends_with(".tar.gz") || lower.ends_with(".tgz") || lower.ends_with(".gz") {
head.starts_with(&[0x1F, 0x8B])
} else if lower.ends_with(".tar.zst") || lower.ends_with(".tzst") || lower.ends_with(".zst") {
head.starts_with(&[0x28, 0xB5, 0x2F, 0xFD])
} else if lower.ends_with(".zip") {
head.starts_with(b"PK\x03\x04")
} else if lower.ends_with(".tar.bz2") || lower.ends_with(".tbz2") {
head.starts_with(&[0x42, 0x5A, 0x68])
} else if lower.ends_with(".tar") {
// check for ustar magic at offset 257
if let Ok(mut f2) = std::fs::File::open(path) {
let mut hdr = [0u8; 262];
if f2.read_exact(&mut hdr).is_ok() {
&hdr[257..262] == b"ustar"
} else {
true // can't validate; be permissive
}
} else {
true
}
} else if lower.ends_with(".deb") {
// ar archive starts with "!<arch>\n"
head.starts_with(b"!<arch>")
} else if lower.ends_with(".rpm") {
// rpm contains cpio magic later; best-effort: accept
true
} else {
// Unknown extension -> be permissive
true
};
if !is_ok {
let preview = String::from_utf8_lossy(&head[..head.len().min(1024)]);
anyhow::bail!(
"Downloaded file '{}' does not match expected archive magic; preview: {}",
filename,
preview.trim()
);
}
Ok(None)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -148,9 +411,89 @@ mod tests {
assert!(name.starts_with("source-") && name.ends_with(".download"));
}
#[test]
fn filename_from_ftp_url() {
assert_eq!(derive_filename_from_url("ftp://example.com/foo-1.2.3.tar.gz"), "foo-1.2.3.tar.gz");
}
#[test]
fn filename_from_non_url_string() {
let name = derive_filename_from_url("not-a-url-at-all");
assert!(name.starts_with("source-") && name.ends_with(".download"));
}
#[test]
fn sourceforge_html_no_link_falls_back_to_download_suffix() {
use std::io::Write;
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(tmp, "<!doctype html><html><body>No direct link</body></html>").unwrap();
let alt = validate_downloaded_archive(
tmp.path(),
"zsh-5.9.tar.xz",
"https://sourceforge.net/projects/zsh/files/zsh/5.9/zsh-5.9.tar.xz",
)
.unwrap();
assert_eq!(
alt,
Some(
"https://sourceforge.net/projects/zsh/files/zsh/5.9/zsh-5.9.tar.xz/download"
.to_string()
)
);
}
#[test]
fn sourceforge_html_single_quoted_href_extracts_downloads_link() {
use std::io::Write;
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(tmp, "<!doctype html><a href='//downloads.sourceforge.net/project/zsh/zsh/5.9/zsh-5.9.tar.xz?download'>download</a>").unwrap();
let alt = validate_downloaded_archive(
tmp.path(),
"zsh-5.9.tar.xz",
"https://sourceforge.net/projects/zsh/files/zsh/5.9/zsh-5.9.tar.xz",
)
.unwrap();
assert_eq!(
alt,
Some("https://downloads.sourceforge.net/project/zsh/zsh/5.9/zsh-5.9.tar.xz?download".to_string())
);
}
#[test]
fn verify_checksum_accepts_md5_sha512_and_default_sha256() {
use sha2::Digest;
use sha2::Sha256;
use sha2::Sha512;
let tmp = tempfile::NamedTempFile::new().unwrap();
std::fs::write(tmp.path(), b"abc").unwrap();
// compute expected values using the same libraries
let sha256_hex = {
let mut h = Sha256::new();
h.update(b"abc");
format!("{:x}", h.finalize())
};
let sha512_hex = {
let mut h = Sha512::new();
h.update(b"abc");
format!("{:x}", h.finalize())
};
let md5_hex = format!("{:x}", md5::compute(b"abc"));
// unprefixed should default to sha256
assert!(verify_checksum(tmp.path(), &sha256_hex).unwrap());
// explicit prefixes
assert!(verify_checksum(tmp.path(), &format!("sha256:{}", sha256_hex)).unwrap());
assert!(verify_checksum(tmp.path(), &format!("sha512:{}", sha512_hex)).unwrap());
assert!(verify_checksum(tmp.path(), &format!("md5:{}", md5_hex)).unwrap());
// empty algorithm before colon -> assume sha256
assert!(verify_checksum(tmp.path(), &format!(":{}", sha256_hex)).unwrap());
// negative: wrong value fails
assert!(!verify_checksum(tmp.path(), "md5:deadbeef").unwrap());
}
}
src/source/hooks.rs
+20 -5
View File
@@ -8,6 +8,8 @@ use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::process::Command;
use crate::builder::state::{BuildStep, StateTracker};
/// Apply patches and run `post_extract` commands in the extracted source tree.
pub fn post_extract(
spec: &PackageSpec,
@@ -15,8 +17,18 @@ pub fn post_extract(
src_dir: &Path,
cache_dir: &Path,
) -> Result<()> {
apply_patches(spec, source, src_dir, cache_dir)?;
run_post_extract_commands(spec, source, src_dir)?;
let mut state = StateTracker::new(src_dir)?;
if !state.is_done(BuildStep::PatchesApplied) {
apply_patches(spec, source, src_dir, cache_dir)?;
state.mark_done(BuildStep::PatchesApplied)?;
}
if !state.is_done(BuildStep::PostExtractDone) {
run_post_extract_commands(spec, source, src_dir)?;
state.mark_done(BuildStep::PostExtractDone)?;
}
Ok(())
}
@@ -80,7 +92,7 @@ fn run_post_extract_commands(spec: &PackageSpec, source: &Source, src_dir: &Path
// Use a shell for convenience; this is a package manager, so specs are trusted input.
let status = Command::new("sh")
.current_dir(src_dir)
.env("NYAPM_SPECDIR", &spec.spec_dir)
.env("DEPOT_SPECDIR", &spec.spec_dir)
.arg("-c")
.arg(&cmd)
.status()
@@ -109,8 +121,9 @@ pub fn run_post_compile_commands(spec: &PackageSpec, src_dir: &Path, destdir: &P
let status = Command::new("sh")
.current_dir(src_dir)
.env("NYAPM_SPECDIR", &spec.spec_dir)
.env("DEPOT_SPECDIR", &spec.spec_dir)
.env("DESTDIR", destdir)
.env("DEPOT_ROOTFS", &spec.build.flags.rootfs)
.env("CC", &spec.build.flags.cc)
.env("AR", &spec.build.flags.ar)
.arg("-c")
@@ -141,8 +154,9 @@ pub fn run_post_install_commands(spec: &PackageSpec, src_dir: &Path, destdir: &P
let status = Command::new("sh")
.current_dir(src_dir)
.env("NYAPM_SPECDIR", &spec.spec_dir)
.env("DEPOT_SPECDIR", &spec.spec_dir)
.env("DESTDIR", destdir)
.env("DEPOT_ROOTFS", &spec.build.flags.rootfs)
.env("CC", &spec.build.flags.cc)
.env("AR", &spec.build.flags.ar)
.arg("-c")
@@ -241,6 +255,7 @@ mod tests {
homepage: "h".into(),
license: "MIT".into(),
},
packages: Vec::new(),
alternatives: Alternatives::default(),
manual_sources: Vec::new(),
source: vec![Source {
src/source/mod.rs
+139 -19
View File
@@ -9,7 +9,6 @@ use crate::package::PackageSpec;
use anyhow::{Context, Result, bail};
use sha2::{Digest, Sha256};
use std::fs;
use std::io::Read;
use std::os::unix::fs as unix_fs;
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
@@ -36,15 +35,13 @@ pub fn copy_manual_sources(spec: &PackageSpec, build_dir: &Path) -> Result<()> {
);
}
// Verify SHA256 if provided (unless "skip")
// Verify checksum if provided (supports `sha256:...`, `sha512:...`, `md5:...`, or raw SHA256).
if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip") {
let actual_hash = compute_sha256(&src_path)?;
if actual_hash != *expected_hash {
if !verify_file_hash(&src_path, expected_hash)? {
bail!(
"SHA256 mismatch for {}: expected {}, got {}",
"Checksum mismatch for {}: expected {}",
manual.file,
expected_hash,
actual_hash
expected_hash
);
}
}
@@ -71,18 +68,76 @@ pub fn copy_manual_sources(spec: &PackageSpec, build_dir: &Path) -> Result<()> {
Ok(())
}
fn compute_sha256(path: &Path) -> Result<String> {
let mut file = fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 8192];
loop {
let n = file.read(&mut buffer)?;
if n == 0 {
break;
}
hasher.update(&buffer[..n]);
/// Verify a file against an `expected` checksum string.
///
/// Formats accepted: `sha256:HEX`, `sha512:HEX`, `md5:HEX`, or plain `HEX` (assumed sha256).
fn verify_file_hash(path: &Path, expected: &str) -> Result<bool> {
use anyhow::bail;
use std::io::Read;
let exp = expected.trim();
if exp.eq_ignore_ascii_case("skip") {
println!("Checksum verification skipped");
return Ok(true);
}
// parse `alg:hex` or default to sha256 when no algorithm given
let (alg, hex) = if let Some(pos) = exp.find(':') {
let a = exp[..pos].trim().to_ascii_lowercase();
let h = exp[pos + 1..].trim().to_ascii_lowercase();
let alg = if a.is_empty() { "sha256".to_string() } else { a };
(alg, h.to_string())
} else {
("sha256".to_string(), exp.to_ascii_lowercase())
};
match alg.as_str() {
"sha256" => {
let mut f = fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buf = [0u8; 8192];
loop {
let n = f.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
let actual = format!("{:x}", hasher.finalize());
Ok(actual == hex)
}
"sha512" => {
use sha2::Sha512;
let mut f = fs::File::open(path)?;
let mut hasher = Sha512::new();
let mut buf = [0u8; 8192];
loop {
let n = f.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
let actual = format!("{:x}", hasher.finalize());
Ok(actual == hex)
}
"md5" => {
let mut ctx = md5::Context::new();
let mut f = fs::File::open(path)?;
let mut buf = [0u8; 8192];
loop {
let n = f.read(&mut buf)?;
if n == 0 {
break;
}
ctx.consume(&buf[..n]);
}
let digest = ctx.finalize();
let actual = format!("{:x}", digest);
Ok(actual == hex)
}
_ => bail!("Unsupported checksum algorithm: {}", alg),
}
Ok(format!("{:x}", hasher.finalize()))
}
/// Fetch + extract all sources.
@@ -128,12 +183,33 @@ fn prepare_one(
if local_path.is_dir() {
let dst = build_dir.join(&extract_dir_name);
if dst.exists() {
// If it exists and has a state file, assume we are resuming
if dst.join(".depot_state").exists() {
println!(
"Resuming build in existing source directory: {}",
dst.display()
);
return Ok(dst);
}
fs::remove_dir_all(&dst)?;
}
copy_dir_recursive(&local_path, &dst)?;
hooks::post_extract(spec, source, &dst, cache_dir)?;
return Ok(dst);
} else if local_path.is_file() {
let src_dir = build_dir.join(&extract_dir_name);
if src_dir.exists() {
if src_dir.join(".depot_state").exists() {
println!(
"Resuming build in existing source directory: {}",
src_dir.display()
);
return Ok(src_dir);
}
// If no state file, or we want a fresh start, we'd delete it.
// For now, let's just delete if no state file.
fs::remove_dir_all(&src_dir)?;
}
let src_dir = extractor::extract_archive(&local_path, spec, source, build_dir)?;
hooks::post_extract(spec, source, &src_dir, cache_dir)?;
return Ok(src_dir);
@@ -146,6 +222,13 @@ fn prepare_one(
// (except when it clearly looks like an archive URL)
if let Some((base, rev)) = split_git_url(&url) {
let checkout_dir = build_dir.join(&extract_dir_name);
if checkout_dir.exists() && checkout_dir.join(".depot_state").exists() {
println!(
"Resuming build in existing git directory: {}",
checkout_dir.display()
);
return Ok(checkout_dir);
}
git::checkout(
&base,
&rev,
@@ -157,6 +240,15 @@ fn prepare_one(
return Ok(checkout_dir);
}
let src_dir = build_dir.join(&extract_dir_name);
if src_dir.exists() && src_dir.join(".depot_state").exists() {
println!(
"Resuming build in existing source directory: {}",
src_dir.display()
);
return Ok(src_dir);
}
let archive_path = fetcher::fetch_archive(spec, source, cache_dir)?;
let src_dir = extractor::extract_archive(&archive_path, spec, source, build_dir)?;
hooks::post_extract(spec, source, &src_dir, cache_dir)?;
@@ -218,7 +310,7 @@ fn split_git_url(url: &str) -> Option<(String, String)> {
#[cfg(test)]
mod tests {
use super::split_git_url;
use super::*;
#[test]
fn split_git_url_accepts_git_with_rev() {
@@ -246,4 +338,32 @@ mod tests {
assert_eq!(base, "https://example.com/repo.git");
assert_eq!(rev, "HEAD");
}
#[test]
fn verify_file_hash_accepts_multiple_algorithms() {
use sha2::{Digest, Sha256, Sha512};
let tmp = tempfile::NamedTempFile::new().unwrap();
std::fs::write(tmp.path(), b"abc").unwrap();
let sha256_hex = {
let mut h = Sha256::new();
h.update(b"abc");
format!("{:x}", h.finalize())
};
let sha512_hex = {
let mut h = Sha512::new();
h.update(b"abc");
format!("{:x}", h.finalize())
};
let md5_hex = format!("{:x}", md5::compute(b"abc"));
assert!(verify_file_hash(tmp.path(), &sha256_hex).unwrap());
assert!(verify_file_hash(tmp.path(), &format!("sha256:{}", sha256_hex)).unwrap());
assert!(verify_file_hash(tmp.path(), &format!("sha512:{}", sha512_hex)).unwrap());
assert!(verify_file_hash(tmp.path(), &format!("md5:{}", md5_hex)).unwrap());
assert!(verify_file_hash(tmp.path(), &format!(":{}", sha256_hex)).unwrap());
assert!(!verify_file_hash(tmp.path(), "md5:deadbeef").unwrap());
}
}