diff --git a/Cargo.lock b/Cargo.lock index 7e3d620..52c2e23 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -418,7 +418,7 @@ checksum = "807800ff3288b621186fe0a8f3392c4652068257302709c24efd918c3dffcdc2" [[package]] name = "depot" -version = "0.15.2" +version = "0.16.0" dependencies = [ "anyhow", "ar", @@ -433,6 +433,8 @@ dependencies = [ "git2", "indicatif", "inquire", + "lz4_flex", + "lzma-rust2", "md5", "minisign", "nix", @@ -1299,6 +1301,15 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lz4_flex" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-rust2" version = "0.16.2" @@ -2358,6 +2369,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typed-path" version = "0.12.3" diff --git a/Cargo.toml b/Cargo.toml index 8ce2d6f..34358c0 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "depot" -version = "0.15.2" +version = "0.16.0" edition = "2024" [lints.rust] @@ -43,6 +43,8 @@ sys-mount = { version = "3.1.0", default-features = false } time = { version = "0.3.47", features = ["formatting", "parsing"] } b2sum-rust = "0.3.0" serde_ignored = "0.1.14" +lz4_flex = "0.12.0" +lzma-rust2 = "0.16.2" [dev-dependencies] tempfile = "=3.27.0" diff --git a/meson.build b/meson.build index 5d2ad13..bbcf0c3 100644 --- a/meson.build +++ b/meson.build @@ -1,6 +1,6 @@ project( 'depot', - version: '0.15.2', + version: '0.16.0', meson_version: '>=0.60.0', ) diff --git a/src/builder/cmake.rs b/src/builder/cmake.rs index 754cddf..7d7890d 100755 --- a/src/builder/cmake.rs +++ b/src/builder/cmake.rs @@ -128,7 +128,7 @@ pub fn build( crate::log_info!("Skipping tests: disabled by build.flags.skip_tests"); } else { let test_targets = phase_targets(&flags.make_test_target, &flags.make_test_targets); - if !test_targets.is_empty() { + if !cmake_uses_default_ctest(flags) { let joined = test_targets.join(" "); crate::log_info!("Running cmake test target(s): {}...", joined); let mut test_cmd = Command::new("cmake"); @@ -149,7 +149,20 @@ pub fn build( anyhow::bail!("cmake test target(s) '{}' failed", joined); } } else { - crate::log_info!("Skipping tests: no build.flags.make_test_target(s) for cmake"); + crate::log_info!("Running ctest..."); + let mut test_cmd = Command::new("ctest"); + test_cmd.current_dir(&build_dir); + test_cmd.arg("--test-dir").arg(&build_dir); + test_cmd.arg("-j").arg(num_cpus().to_string()); + test_cmd.arg("--output-on-failure"); + crate::builder::prepare_tool_command(&mut test_cmd, &env_vars); + + let status = test_cmd + .status() + .with_context(|| format!("Failed to run ctest for {}", spec.package.name))?; + if !status.success() { + anyhow::bail!("ctest failed"); + } } } @@ -251,6 +264,10 @@ fn phase_targets(single: &str, many: &[String]) -> Vec { targets } +fn cmake_uses_default_ctest(flags: &crate::package::BuildFlags) -> bool { + phase_targets(&flags.make_test_target, &flags.make_test_targets).is_empty() +} + fn cmake_generator_for_make_exec(make_exec: &str) -> Option<&'static str> { let tool = Path::new(make_exec) .file_name() @@ -434,6 +451,23 @@ mod tests { assert!(phase_targets("", &[]).is_empty()); } + #[test] + fn test_cmake_uses_default_ctest_without_explicit_targets() { + assert!(cmake_uses_default_ctest(&BuildFlags::default())); + + let explicit_single = BuildFlags { + make_test_target: "test".into(), + ..BuildFlags::default() + }; + assert!(!cmake_uses_default_ctest(&explicit_single)); + + let explicit_many = BuildFlags { + make_test_targets: vec!["check".into()], + ..BuildFlags::default() + }; + assert!(!cmake_uses_default_ctest(&explicit_many)); + } + #[test] fn test_cmake_generator_for_make_exec_detects_ninja_and_make() { assert_eq!(cmake_generator_for_make_exec("ninja"), Some("Ninja")); diff --git a/src/builder/meson.rs b/src/builder/meson.rs index a34d59f..6c533e8 100755 --- a/src/builder/meson.rs +++ b/src/builder/meson.rs @@ -83,6 +83,36 @@ pub fn build( anyhow::bail!("ninja build failed"); } + if flags.skip_tests { + crate::log_info!("Skipping tests: disabled by build.flags.skip_tests"); + } else { + let test_suites = meson_test_suites(flags); + if test_suites.is_empty() { + crate::log_info!("Running meson test..."); + } else { + crate::log_info!("Running meson test suite(s): {}...", test_suites.join(" ")); + } + + let mut test_cmd = Command::new("meson"); + test_cmd.current_dir(&build_dir); + test_cmd.arg("test"); + test_cmd.arg("-C").arg(&build_dir); + test_cmd.arg("--num-processes").arg(num_cpus().to_string()); + test_cmd.arg("--print-errorlogs"); + for suite in &test_suites { + test_cmd.arg("--suite").arg(suite); + } + + crate::builder::prepare_tool_command(&mut test_cmd, &env_vars); + + let status = test_cmd + .status() + .with_context(|| format!("Failed to run meson test for {}", spec.package.name))?; + if !status.success() { + anyhow::bail!("meson test failed"); + } + } + crate::source::hooks::run_post_compile_commands(spec, &actual_src, destdir)?; state.mark_done(BuildStep::PostCompileDone)?; } else { @@ -146,6 +176,21 @@ fn resolve_build_dir(actual_src: &Path, flags: &crate::package::BuildFlags) -> P } } +fn meson_test_suites(flags: &crate::package::BuildFlags) -> Vec { + let mut suites = Vec::new(); + let single = flags.make_test_target.trim(); + if !single.is_empty() { + suites.push(single.to_string()); + } + for suite in &flags.make_test_targets { + let trimmed = suite.trim(); + if !trimmed.is_empty() { + suites.push(trimmed.to_string()); + } + } + suites +} + fn has_option(configure: &[String], long: &str) -> bool { let prefix = format!("{long}="); for arg in configure { @@ -407,6 +452,28 @@ mod tests { ); } + #[test] + fn test_meson_test_suites_uses_single_and_multiple_targets() { + let flags = BuildFlags { + make_test_target: "unit".to_string(), + make_test_targets: vec!["integration".to_string(), " smoke ".to_string()], + ..BuildFlags::default() + }; + assert_eq!( + meson_test_suites(&flags), + vec![ + "unit".to_string(), + "integration".to_string(), + "smoke".to_string() + ] + ); + } + + #[test] + fn test_meson_test_suites_empty_without_targets() { + assert!(meson_test_suites(&BuildFlags::default()).is_empty()); + } + #[test] fn test_resolve_actual_src_uses_source_subdir_under_source() -> Result<()> { let src = tempdir()?; diff --git a/src/commands.rs b/src/commands.rs index 3702ba3..f046a35 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -489,10 +489,13 @@ fn extract_package_archive_to_staging( Ok(tmp_dir) } -fn build_type_runs_automatic_tests(build_type: package::BuildType) -> bool { +fn build_type_runs_automatic_tests(spec: &package::PackageSpec) -> bool { matches!( - build_type, - package::BuildType::Autotools | package::BuildType::CMake | package::BuildType::Perl + spec.build.build_type, + package::BuildType::Autotools + | package::BuildType::CMake + | package::BuildType::Meson + | package::BuildType::Perl ) } @@ -501,7 +504,7 @@ fn maybe_disable_tests_for_missing_deps( db_path: &Path, ) -> Result<()> { if pkg_spec.build.flags.skip_tests - || !build_type_runs_automatic_tests(pkg_spec.build.build_type) + || !build_type_runs_automatic_tests(pkg_spec) || pkg_spec.dependencies.test.is_empty() { return Ok(()); @@ -4435,6 +4438,48 @@ pub fn run(cli: Cli) -> Result<()> { #[cfg(test)] mod tests { use super::*; + + fn test_package_spec( + build_type: package::BuildType, + make_test_target: Option<&str>, + make_test_targets: &[&str], + ) -> package::PackageSpec { + let mut flags = package::BuildFlags::default(); + if let Some(target) = make_test_target { + flags.make_test_target = target.to_string(); + } + flags.make_test_targets = make_test_targets + .iter() + .map(|target| (*target).to_string()) + .collect(); + + package::PackageSpec { + package: package::PackageInfo { + name: "pkg".into(), + version: "1.0".into(), + revision: 1, + description: "d".into(), + homepage: "h".into(), + license: vec!["MIT".into()], + }, + packages: Vec::new(), + alternatives: Default::default(), + manual_sources: Vec::new(), + source: vec![package::Source { + url: "https://example.test/pkg.tar.gz".into(), + sha256: "skip".into(), + extract_dir: "pkg".into(), + patches: Vec::new(), + post_extract: Vec::new(), + cherry_pick: Vec::new(), + }], + build: package::Build { build_type, flags }, + dependencies: package::Dependencies::default(), + package_alternatives: Default::default(), + package_dependencies: Default::default(), + spec_dir: PathBuf::from("."), + } + } use anyhow::Context; use std::io::Write; @@ -5450,6 +5495,30 @@ optional = [] assert_eq!(install_success_action(true), "updated"); } + #[test] + fn build_type_runs_automatic_tests_matches_builder_behavior() { + assert!(build_type_runs_automatic_tests(&test_package_spec( + package::BuildType::Autotools, + None, + &[] + ))); + assert!(build_type_runs_automatic_tests(&test_package_spec( + package::BuildType::Perl, + None, + &[] + ))); + assert!(build_type_runs_automatic_tests(&test_package_spec( + package::BuildType::Meson, + None, + &[] + ))); + assert!(build_type_runs_automatic_tests(&test_package_spec( + package::BuildType::CMake, + None, + &[] + ))); + } + #[test] fn rootfs_is_system_root_detects_live_rootfs() { assert!(rootfs_is_system_root(Path::new("/"))); diff --git a/src/deps.rs b/src/deps.rs index fadd9a2..9b083ea 100755 --- a/src/deps.rs +++ b/src/deps.rs @@ -142,7 +142,7 @@ fn is_dep_satisfied( fn build_type_runs_automatic_tests(spec: &PackageSpec) -> bool { matches!( spec.build.build_type, - BuildType::Autotools | BuildType::CMake | BuildType::Perl + BuildType::Autotools | BuildType::CMake | BuildType::Meson | BuildType::Perl ) } @@ -324,6 +324,47 @@ pub fn require_test_deps(spec: &PackageSpec, db_path: &Path) -> Result<()> { mod tests { use super::*; + fn test_spec_with_build( + build_type: BuildType, + configure_test_target: Option<&str>, + configure_test_targets: &[&str], + ) -> PackageSpec { + let mut flags = crate::package::BuildFlags::default(); + if let Some(target) = configure_test_target { + flags.make_test_target = target.to_string(); + } + flags.make_test_targets = configure_test_targets + .iter() + .map(|target| (*target).to_string()) + .collect(); + PackageSpec { + package: crate::package::PackageInfo { + name: "foo".into(), + version: "1.0".into(), + revision: 1, + description: "d".into(), + homepage: "h".into(), + license: vec!["MIT".into()], + }, + packages: Vec::new(), + alternatives: Default::default(), + manual_sources: Vec::new(), + source: vec![crate::package::Source { + url: "https://example.test/foo.tar.gz".into(), + sha256: "skip".into(), + extract_dir: "foo".into(), + patches: Vec::new(), + post_extract: Vec::new(), + cherry_pick: Vec::new(), + }], + build: crate::package::Build { build_type, flags }, + dependencies: crate::package::Dependencies::default(), + package_alternatives: Default::default(), + package_dependencies: Default::default(), + spec_dir: std::path::PathBuf::from("."), + } + } + #[test] fn test_parse_dep() { let cases = vec![ @@ -506,4 +547,28 @@ mod tests { let missing = check_runtime_deps(&spec, Path::new("/definitely/not/a/real/db")).unwrap(); assert_eq!(missing, vec!["python".to_string()]); } + + #[test] + fn test_build_type_runs_automatic_tests_matches_builder_behavior() { + assert!(build_type_runs_automatic_tests(&test_spec_with_build( + BuildType::Autotools, + None, + &[] + ))); + assert!(build_type_runs_automatic_tests(&test_spec_with_build( + BuildType::Perl, + None, + &[] + ))); + assert!(build_type_runs_automatic_tests(&test_spec_with_build( + BuildType::Meson, + None, + &[] + ))); + assert!(build_type_runs_automatic_tests(&test_spec_with_build( + BuildType::CMake, + None, + &[] + ))); + } } diff --git a/src/source/extractor.rs b/src/source/extractor.rs index e9852bf..7500ca1 100755 --- a/src/source/extractor.rs +++ b/src/source/extractor.rs @@ -4,14 +4,37 @@ use crate::package::{PackageSpec, Source}; use anyhow::{Context, Result, bail}; use filetime::FileTime; use flate2::read::GzDecoder; +use lz4_flex::frame::FrameDecoder as Lz4FrameDecoder; +use lzma_rust2::{LzipReader, LzmaReader}; use std::fs::{self, File}; use std::io::{Cursor, Read, Write}; use std::os::unix::fs as unix_fs; use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; use tempfile::{NamedTempFile, tempdir}; use walkdir::WalkDir; use zstd::stream::read::Decoder as ZstdDecoder; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ArchiveFormat { + TarGz, + TarXz, + TarBz2, + TarZst, + TarLz4, + TarLzma, + TarLzip, + TarCompress, + Zip, + Tar, + Deb, + Rpm, + Cpio, + GzFile, + XzFile, + ZstFile, +} + /// Extract an archive source to the build directory. pub fn extract_archive( archive_path: &Path, @@ -38,30 +61,24 @@ pub fn extract_archive( crate::log_info!("Extracting: {}", filename); - if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") { - extract_tar_gz(archive_path, &extract_path)?; - } else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") { - extract_tar_xz(archive_path, &extract_path)?; - } else if filename.ends_with(".tar.bz2") || filename.ends_with(".tbz2") { - extract_tar_bz2(archive_path, &extract_path)?; - } else if filename.ends_with(".tar.zst") || filename.ends_with(".tzst") { - extract_tar_zst(archive_path, &extract_path)?; - } else if filename.ends_with(".zip") { - extract_zip(archive_path, &extract_path)?; - } else if filename.ends_with(".tar") { - extract_tar(archive_path, &extract_path)?; - } else if filename.ends_with(".deb") { - extract_deb(archive_path, &extract_path)?; - } else if filename.ends_with(".rpm") { - extract_rpm(archive_path, &extract_path)?; - } else if filename.ends_with(".gz") { - extract_gz_file(archive_path, &extract_path)?; - } else if filename.ends_with(".xz") { - extract_xz_file(archive_path, &extract_path)?; - } else if filename.ends_with(".zst") { - extract_zst_file(archive_path, &extract_path)?; - } else { - bail!("Unsupported archive format: {}", filename); + match archive_format_for_filename(filename) { + Some(ArchiveFormat::TarGz) => extract_tar_gz(archive_path, &extract_path)?, + Some(ArchiveFormat::TarXz) => extract_tar_xz(archive_path, &extract_path)?, + Some(ArchiveFormat::TarBz2) => extract_tar_bz2(archive_path, &extract_path)?, + Some(ArchiveFormat::TarZst) => extract_tar_zst(archive_path, &extract_path)?, + Some(ArchiveFormat::TarLz4) => extract_tar_lz4(archive_path, &extract_path)?, + Some(ArchiveFormat::TarLzma) => extract_tar_lzma(archive_path, &extract_path)?, + Some(ArchiveFormat::TarLzip) => extract_tar_lzip(archive_path, &extract_path)?, + Some(ArchiveFormat::TarCompress) => extract_tar_compress(archive_path, &extract_path)?, + Some(ArchiveFormat::Zip) => extract_zip(archive_path, &extract_path)?, + Some(ArchiveFormat::Tar) => extract_tar(archive_path, &extract_path)?, + Some(ArchiveFormat::Deb) => extract_deb(archive_path, &extract_path)?, + Some(ArchiveFormat::Rpm) => extract_rpm(archive_path, &extract_path)?, + Some(ArchiveFormat::Cpio) => extract_cpio(archive_path, &extract_path)?, + Some(ArchiveFormat::GzFile) => extract_gz_file(archive_path, &extract_path)?, + Some(ArchiveFormat::XzFile) => extract_xz_file(archive_path, &extract_path)?, + Some(ArchiveFormat::ZstFile) => extract_zst_file(archive_path, &extract_path)?, + None => bail!("Unsupported archive format: {}", filename), } if !extract_path.exists() { @@ -75,187 +92,65 @@ pub fn extract_archive( Ok(extract_path) } +fn archive_format_for_filename(filename: &str) -> Option { + if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") { + Some(ArchiveFormat::TarGz) + } else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") { + Some(ArchiveFormat::TarXz) + } else if filename.ends_with(".tar.bz2") || filename.ends_with(".tbz2") { + Some(ArchiveFormat::TarBz2) + } else if filename.ends_with(".tar.zst") || filename.ends_with(".tzst") { + Some(ArchiveFormat::TarZst) + } else if filename.ends_with(".tar.lz4") { + Some(ArchiveFormat::TarLz4) + } else if filename.ends_with(".tar.lzma") { + Some(ArchiveFormat::TarLzma) + } else if filename.ends_with(".tar.lz") { + Some(ArchiveFormat::TarLzip) + } else if filename.ends_with(".tar.Z") { + Some(ArchiveFormat::TarCompress) + } else if filename.ends_with(".zip") { + Some(ArchiveFormat::Zip) + } else if filename.ends_with(".tar") { + Some(ArchiveFormat::Tar) + } else if filename.ends_with(".deb") { + Some(ArchiveFormat::Deb) + } else if filename.ends_with(".rpm") { + Some(ArchiveFormat::Rpm) + } else if filename.ends_with(".cpio") { + Some(ArchiveFormat::Cpio) + } else if filename.ends_with(".gz") { + Some(ArchiveFormat::GzFile) + } else if filename.ends_with(".xz") { + Some(ArchiveFormat::XzFile) + } else if filename.ends_with(".zst") { + Some(ArchiveFormat::ZstFile) + } else { + None + } +} + fn extract_tar_gz(path: &Path, dest: &Path) -> Result<()> { - let tmp = tempdir()?; let file = File::open(path)?; let decoder = GzDecoder::new(file); - let mut archive = tar::Archive::new(decoder); - archive.unpack(tmp.path())?; - - // If the archive produced a single top-level directory, decide whether to - // strip that top directory (source tarballs like foo-1.2.3/) or preserve - // it (system-layout archives like usr/). Otherwise move all top-level - // entries into `dest` so `dest` always contains the source root. - let top = fs::read_dir(tmp.path())? - .filter_map(|r| r.ok()) - .collect::>(); - if top.len() == 1 && top[0].path().is_dir() { - let top_name = top[0].file_name().to_string_lossy().to_string(); - let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or(""); - - // blacklist of system-layout directories we should NOT strip - let sys_blacklist = [ - "usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev", - "proc", "sys", "boot", "srv", "home", - ]; - - let looks_like_versioned = - |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit()); - let should_strip = (!sys_blacklist.contains(&top_name.as_str())) - && (top_name == expected_basename - || (!expected_basename.is_empty() && top_name.contains(expected_basename)) - || looks_like_versioned(&top_name)); - - if should_strip { - // strip the single top-level folder (move its contents into dest) - move_dir_contents(&top[0].path(), dest)?; - } else { - // preserve the top-level folder (move the directory itself under dest) - fs::create_dir_all(dest)?; - let dest_top = dest.join(top_name); - if fs::rename(top[0].path(), &dest_top).is_err() { - copy_dir_recursive_local(&top[0].path(), &dest_top)?; - fs::remove_dir_all(top[0].path())?; - } - } - } else { - move_dir_contents(tmp.path(), dest)?; - } - Ok(()) + extract_tar_reader(decoder, dest) } fn extract_tar_xz(path: &Path, dest: &Path) -> Result<()> { - let tmp = tempdir()?; let file = File::open(path)?; let decoder = xz2::read::XzDecoder::new(file); - let mut archive = tar::Archive::new(decoder); - archive.unpack(tmp.path())?; - - let top = fs::read_dir(tmp.path())? - .filter_map(|r| r.ok()) - .collect::>(); - if top.len() == 1 && top[0].path().is_dir() { - let top_name = top[0].file_name().to_string_lossy().to_string(); - let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or(""); - - // blacklist of system-layout directories we should NOT strip - let sys_blacklist = [ - "usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev", - "proc", "sys", "boot", "srv", "home", - ]; - - let looks_like_versioned = - |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit()); - let should_strip = (!sys_blacklist.contains(&top_name.as_str())) - && (top_name == expected_basename - || (!expected_basename.is_empty() && top_name.contains(expected_basename)) - || looks_like_versioned(&top_name)); - - if should_strip { - // strip the single top-level folder (move its contents into dest) - move_dir_contents(&top[0].path(), dest)?; - } else { - // preserve the top-level folder (move the directory itself under dest) - fs::create_dir_all(dest)?; - let dest_top = dest.join(top_name); - if fs::rename(top[0].path(), &dest_top).is_err() { - copy_dir_recursive_local(&top[0].path(), &dest_top)?; - fs::remove_dir_all(top[0].path())?; - } - } - } else { - move_dir_contents(tmp.path(), dest)?; - } - Ok(()) + extract_tar_reader(decoder, dest) } fn extract_tar_bz2(path: &Path, dest: &Path) -> Result<()> { - let tmp = tempdir()?; let file = File::open(path)?; let decoder = bzip2::read::BzDecoder::new(file); - let mut archive = tar::Archive::new(decoder); - archive.unpack(tmp.path())?; - - let top = fs::read_dir(tmp.path())? - .filter_map(|r| r.ok()) - .collect::>(); - if top.len() == 1 && top[0].path().is_dir() { - let top_name = top[0].file_name().to_string_lossy().to_string(); - let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or(""); - - // blacklist of system-layout directories we should NOT strip - let sys_blacklist = [ - "usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev", - "proc", "sys", "boot", "srv", "home", - ]; - - let looks_like_versioned = - |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit()); - let should_strip = (!sys_blacklist.contains(&top_name.as_str())) - && (top_name == expected_basename - || (!expected_basename.is_empty() && top_name.contains(expected_basename)) - || looks_like_versioned(&top_name)); - - if should_strip { - // strip the single top-level folder (move its contents into dest) - move_dir_contents(&top[0].path(), dest)?; - } else { - // preserve the top-level folder (move the directory itself under dest) - fs::create_dir_all(dest)?; - let dest_top = dest.join(top_name); - if fs::rename(top[0].path(), &dest_top).is_err() { - copy_dir_recursive_local(&top[0].path(), &dest_top)?; - fs::remove_dir_all(top[0].path())?; - } - } - } else { - move_dir_contents(tmp.path(), dest)?; - } - Ok(()) + extract_tar_reader(decoder, dest) } fn extract_tar(path: &Path, dest: &Path) -> Result<()> { - let tmp = tempdir()?; let file = File::open(path)?; - let mut archive = tar::Archive::new(file); - archive.unpack(tmp.path())?; - - let top = fs::read_dir(tmp.path())? - .filter_map(|r| r.ok()) - .collect::>(); - if top.len() == 1 && top[0].path().is_dir() { - let top_name = top[0].file_name().to_string_lossy().to_string(); - let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or(""); - - // blacklist of system-layout directories we should NOT strip - let sys_blacklist = [ - "usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev", - "proc", "sys", "boot", "srv", "home", - ]; - - let looks_like_versioned = - |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit()); - let should_strip = (!sys_blacklist.contains(&top_name.as_str())) - && (top_name == expected_basename - || (!expected_basename.is_empty() && top_name.contains(expected_basename)) - || looks_like_versioned(&top_name)); - - if should_strip { - // strip the single top-level folder (move its contents into dest) - move_dir_contents(&top[0].path(), dest)?; - } else { - // preserve the top-level folder (move the directory itself under dest) - fs::create_dir_all(dest)?; - let dest_top = dest.join(top_name); - if fs::rename(top[0].path(), &dest_top).is_err() { - copy_dir_recursive_local(&top[0].path(), &dest_top)?; - fs::remove_dir_all(top[0].path())?; - } - } - } else { - move_dir_contents(tmp.path(), dest)?; - } - Ok(()) + extract_tar_reader(file, dest) } fn extract_zip(path: &Path, dest: &Path) -> Result<()> { @@ -263,60 +158,84 @@ fn extract_zip(path: &Path, dest: &Path) -> Result<()> { let file = File::open(path)?; let mut archive = zip::ZipArchive::new(file)?; archive.extract(tmp.path())?; - - let top = fs::read_dir(tmp.path())? - .filter_map(|r| r.ok()) - .collect::>(); - if top.len() == 1 && top[0].path().is_dir() { - let top_name = top[0].file_name().to_string_lossy().to_string(); - let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or(""); - - // blacklist of system-layout directories we should NOT strip - let sys_blacklist = [ - "usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev", - "proc", "sys", "boot", "srv", "home", - ]; - - let looks_like_versioned = - |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit()); - let should_strip = (!sys_blacklist.contains(&top_name.as_str())) - && (top_name == expected_basename - || (!expected_basename.is_empty() && top_name.contains(expected_basename)) - || looks_like_versioned(&top_name)); - - if should_strip { - // strip the single top-level folder (move its contents into dest) - move_dir_contents(&top[0].path(), dest)?; - } else { - // preserve the top-level folder (move the directory itself under dest) - fs::create_dir_all(dest)?; - let dest_top = dest.join(top_name); - if fs::rename(top[0].path(), &dest_top).is_err() { - copy_dir_recursive_local(&top[0].path(), &dest_top)?; - fs::remove_dir_all(top[0].path())?; - } - } - } else { - move_dir_contents(tmp.path(), dest)?; - } - Ok(()) + finalize_extracted_tree(tmp.path(), dest) } fn extract_tar_zst(path: &Path, dest: &Path) -> Result<()> { - let tmp = tempdir()?; let file = File::open(path)?; let decoder = ZstdDecoder::new(file)?; - let mut archive = tar::Archive::new(decoder); - archive.unpack(tmp.path())?; + extract_tar_reader(decoder, dest) +} - let top = fs::read_dir(tmp.path())? +fn extract_tar_lz4(path: &Path, dest: &Path) -> Result<()> { + let file = File::open(path)?; + let decoder = Lz4FrameDecoder::new(file); + extract_tar_reader(decoder, dest) +} + +fn extract_tar_lzma(path: &Path, dest: &Path) -> Result<()> { + let file = File::open(path)?; + let decoder = LzmaReader::new_mem_limit(file, u32::MAX, None)?; + extract_tar_reader(decoder, dest) +} + +fn extract_tar_lzip(path: &Path, dest: &Path) -> Result<()> { + let file = File::open(path)?; + let decoder = LzipReader::new(file); + extract_tar_reader(decoder, dest) +} + +fn extract_tar_compress(path: &Path, dest: &Path) -> Result<()> { + let tmp = tempdir()?; + let mut child = Command::new("gzip"); + child.arg("-cd").arg(path); + child.stdout(Stdio::piped()); + let mut child = child.spawn().with_context(|| { + format!( + "Failed to spawn gzip for .Z decompression: {}", + path.display() + ) + })?; + let stdout = child + .stdout + .take() + .context("Failed to capture gzip stdout")?; + let mut archive = tar::Archive::new(stdout); + archive + .unpack(tmp.path()) + .with_context(|| format!("Failed to unpack .tar.Z archive {}", path.display()))?; + drop(archive); + let status = child + .wait() + .with_context(|| format!("Failed waiting for gzip on {}", path.display()))?; + if !status.success() { + bail!("gzip failed while decompressing {}", path.display()); + } + finalize_extracted_tree(tmp.path(), dest) +} + +fn extract_cpio(path: &Path, dest: &Path) -> Result<()> { + let tmp = tempdir()?; + let file = File::open(path)?; + extract_cpio_newc_from_reader(file, tmp.path())?; + finalize_extracted_tree(tmp.path(), dest) +} + +fn extract_tar_reader(reader: R, dest: &Path) -> Result<()> { + let tmp = tempdir()?; + let mut archive = tar::Archive::new(reader); + archive.unpack(tmp.path())?; + finalize_extracted_tree(tmp.path(), dest) +} + +fn finalize_extracted_tree(src_root: &Path, dest: &Path) -> Result<()> { + let top = fs::read_dir(src_root)? .filter_map(|r| r.ok()) .collect::>(); if top.len() == 1 && top[0].path().is_dir() { let top_name = top[0].file_name().to_string_lossy().to_string(); let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or(""); - // blacklist of system-layout directories we should NOT strip let sys_blacklist = [ "usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev", "proc", "sys", "boot", "srv", "home", @@ -330,10 +249,8 @@ fn extract_tar_zst(path: &Path, dest: &Path) -> Result<()> { || looks_like_versioned(&top_name)); if should_strip { - // strip the single top-level folder (move its contents into dest) move_dir_contents(&top[0].path(), dest)?; } else { - // preserve the top-level folder (move the directory itself under dest) fs::create_dir_all(dest)?; let dest_top = dest.join(top_name); if fs::rename(top[0].path(), &dest_top).is_err() { @@ -342,7 +259,7 @@ fn extract_tar_zst(path: &Path, dest: &Path) -> Result<()> { } } } else { - move_dir_contents(tmp.path(), dest)?; + move_dir_contents(src_root, dest)?; } Ok(()) } @@ -684,10 +601,64 @@ fn copy_dir_recursive_local(src: &Path, dst: &Path) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use crate::package::{Build, BuildFlags, BuildType, Dependencies, PackageInfo}; + use lz4_flex::frame::FrameEncoder as Lz4FrameEncoder; + use lzma_rust2::{LzipOptions, LzipWriter, LzmaOptions, LzmaWriter}; use std::io::Write; use std::time::{Duration, SystemTime}; use tempfile::tempdir; + fn test_spec() -> PackageSpec { + PackageSpec { + package: PackageInfo { + name: "pkg".into(), + version: "1.0".into(), + revision: 1, + description: "d".into(), + homepage: "h".into(), + license: vec!["MIT".into()], + }, + packages: Vec::new(), + alternatives: Default::default(), + manual_sources: Vec::new(), + source: Vec::new(), + build: Build { + build_type: BuildType::Custom, + flags: BuildFlags::default(), + }, + dependencies: Dependencies::default(), + package_alternatives: Default::default(), + package_dependencies: Default::default(), + spec_dir: PathBuf::from("."), + } + } + + fn test_source(extract_dir: &str) -> Source { + Source { + url: "https://example.test/src.tar".into(), + sha256: "skip".into(), + extract_dir: extract_dir.into(), + patches: Vec::new(), + post_extract: Vec::new(), + cherry_pick: Vec::new(), + } + } + + fn simple_tar_bytes(top_dir: &str, file_name: &str, contents: &[u8]) -> Vec { + let mut tar_buf = Vec::new(); + { + let mut tar = tar::Builder::new(&mut tar_buf); + let mut header = tar::Header::new_gnu(); + header.set_size(contents.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, format!("{top_dir}/{file_name}"), contents) + .unwrap(); + tar.finish().unwrap(); + } + tar_buf + } + #[test] fn test_extract_deb_roundtrip() { let tmp = tempdir().unwrap(); @@ -733,6 +704,145 @@ mod tests { assert!(extract_dir.join("usr/bin/hello-deb").exists()); } + #[test] + fn test_archive_format_for_new_extensions() { + assert_eq!( + archive_format_for_filename("pkg.tar.lz4"), + Some(ArchiveFormat::TarLz4) + ); + assert_eq!( + archive_format_for_filename("pkg.tar.lzma"), + Some(ArchiveFormat::TarLzma) + ); + assert_eq!( + archive_format_for_filename("pkg.tar.lz"), + Some(ArchiveFormat::TarLzip) + ); + assert_eq!( + archive_format_for_filename("pkg.tar.Z"), + Some(ArchiveFormat::TarCompress) + ); + assert_eq!( + archive_format_for_filename("pkg.cpio"), + Some(ArchiveFormat::Cpio) + ); + } + + #[test] + fn test_extract_archive_tar_lz4_roundtrip() { + let tmp = tempdir().unwrap(); + let archive_path = tmp.path().join("pkg.tar.lz4"); + let build_dir = tmp.path().join("build"); + let tar_buf = simple_tar_bytes("pkg-1.0", "hello.txt", b"hello-lz4"); + + let mut compressed = Vec::new(); + { + let mut encoder = Lz4FrameEncoder::new(&mut compressed); + encoder.write_all(&tar_buf).unwrap(); + encoder.finish().unwrap(); + } + fs::write(&archive_path, compressed).unwrap(); + + let extracted = extract_archive( + &archive_path, + &test_spec(), + &test_source("pkg-1.0"), + &build_dir, + ) + .unwrap(); + + assert_eq!( + fs::read_to_string(extracted.join("hello.txt")).unwrap(), + "hello-lz4" + ); + } + + #[test] + fn test_extract_archive_tar_lzma_roundtrip() { + let tmp = tempdir().unwrap(); + let archive_path = tmp.path().join("pkg.tar.lzma"); + let build_dir = tmp.path().join("build"); + let tar_buf = simple_tar_bytes("pkg-1.0", "hello.txt", b"hello-lzma"); + + let mut compressed = Vec::new(); + { + let options = LzmaOptions::default(); + let mut writer = + LzmaWriter::new_use_header(&mut compressed, &options, Some(tar_buf.len() as u64)) + .unwrap(); + writer.write_all(&tar_buf).unwrap(); + writer.finish().unwrap(); + } + fs::write(&archive_path, compressed).unwrap(); + + let extracted = extract_archive( + &archive_path, + &test_spec(), + &test_source("pkg-1.0"), + &build_dir, + ) + .unwrap(); + + assert_eq!( + fs::read_to_string(extracted.join("hello.txt")).unwrap(), + "hello-lzma" + ); + } + + #[test] + fn test_extract_archive_tar_lzip_roundtrip() { + let tmp = tempdir().unwrap(); + let archive_path = tmp.path().join("pkg.tar.lz"); + let build_dir = tmp.path().join("build"); + let tar_buf = simple_tar_bytes("pkg-1.0", "hello.txt", b"hello-lzip"); + + let mut compressed = Vec::new(); + { + let mut writer = LzipWriter::new(&mut compressed, LzipOptions::default()); + writer.write_all(&tar_buf).unwrap(); + writer.finish().unwrap(); + } + fs::write(&archive_path, compressed).unwrap(); + + let extracted = extract_archive( + &archive_path, + &test_spec(), + &test_source("pkg-1.0"), + &build_dir, + ) + .unwrap(); + + assert_eq!( + fs::read_to_string(extracted.join("hello.txt")).unwrap(), + "hello-lzip" + ); + } + + #[test] + fn test_extract_archive_cpio_roundtrip() { + let tmp = tempdir().unwrap(); + let archive_path = tmp.path().join("pkg.cpio"); + let build_dir = tmp.path().join("build"); + + let mut cpio = Vec::new(); + write_cpio_newc_one_file(&mut cpio, "pkg-1.0/hello.txt", b"hello-cpio"); + write_cpio_trailer(&mut cpio); + fs::write(&archive_path, cpio).unwrap(); + + let extracted = extract_archive( + &archive_path, + &test_spec(), + &test_source("pkg-1.0"), + &build_dir, + ) + .unwrap(); + + assert_eq!( + fs::read_to_string(extracted.join("hello.txt")).unwrap(), + "hello-cpio" + ); + } + #[test] fn copy_file_preserve_metadata_keeps_mtime() { let tmp = tempdir().unwrap();