Update dependencies and enhance archive extraction support

- Bump version of the `depot` package from 0.15.2 to 0.16.0 in Cargo.toml and meson.build.
- Add new dependencies: `lz4_flex` and `lzma-rust2` in Cargo.toml and Cargo.lock.
- Implement support for extracting `.tar.lz4`, `.tar.lzma`, and `.tar.lzip` formats in the extractor module.
- Refactor archive extraction logic to streamline handling of various archive formats.
- Add tests for new archive formats to ensure extraction functionality works as expected.
This commit is contained in:
2026-03-10 20:09:03 -05:00
parent 7fad7c552b
commit ccd43fca6a
8 changed files with 607 additions and 243 deletions
Generated
+18 -1
View File
@@ -418,7 +418,7 @@ checksum = "807800ff3288b621186fe0a8f3392c4652068257302709c24efd918c3dffcdc2"
[[package]]
name = "depot"
version = "0.15.2"
version = "0.16.0"
dependencies = [
"anyhow",
"ar",
@@ -433,6 +433,8 @@ dependencies = [
"git2",
"indicatif",
"inquire",
"lz4_flex",
"lzma-rust2",
"md5",
"minisign",
"nix",
@@ -1299,6 +1301,15 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lz4_flex"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e"
dependencies = [
"twox-hash",
]
[[package]]
name = "lzma-rust2"
version = "0.16.2"
@@ -2358,6 +2369,12 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "twox-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
[[package]]
name = "typed-path"
version = "0.12.3"
+3 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "depot"
version = "0.15.2"
version = "0.16.0"
edition = "2024"
[lints.rust]
@@ -43,6 +43,8 @@ sys-mount = { version = "3.1.0", default-features = false }
time = { version = "0.3.47", features = ["formatting", "parsing"] }
b2sum-rust = "0.3.0"
serde_ignored = "0.1.14"
lz4_flex = "0.12.0"
lzma-rust2 = "0.16.2"
[dev-dependencies]
tempfile = "=3.27.0"
+1 -1
View File
@@ -1,6 +1,6 @@
project(
'depot',
version: '0.15.2',
version: '0.16.0',
meson_version: '>=0.60.0',
)
+36 -2
View File
@@ -128,7 +128,7 @@ pub fn build(
crate::log_info!("Skipping tests: disabled by build.flags.skip_tests");
} else {
let test_targets = phase_targets(&flags.make_test_target, &flags.make_test_targets);
if !test_targets.is_empty() {
if !cmake_uses_default_ctest(flags) {
let joined = test_targets.join(" ");
crate::log_info!("Running cmake test target(s): {}...", joined);
let mut test_cmd = Command::new("cmake");
@@ -149,7 +149,20 @@ pub fn build(
anyhow::bail!("cmake test target(s) '{}' failed", joined);
}
} else {
crate::log_info!("Skipping tests: no build.flags.make_test_target(s) for cmake");
crate::log_info!("Running ctest...");
let mut test_cmd = Command::new("ctest");
test_cmd.current_dir(&build_dir);
test_cmd.arg("--test-dir").arg(&build_dir);
test_cmd.arg("-j").arg(num_cpus().to_string());
test_cmd.arg("--output-on-failure");
crate::builder::prepare_tool_command(&mut test_cmd, &env_vars);
let status = test_cmd
.status()
.with_context(|| format!("Failed to run ctest for {}", spec.package.name))?;
if !status.success() {
anyhow::bail!("ctest failed");
}
}
}
@@ -251,6 +264,10 @@ fn phase_targets(single: &str, many: &[String]) -> Vec<String> {
targets
}
fn cmake_uses_default_ctest(flags: &crate::package::BuildFlags) -> bool {
phase_targets(&flags.make_test_target, &flags.make_test_targets).is_empty()
}
fn cmake_generator_for_make_exec(make_exec: &str) -> Option<&'static str> {
let tool = Path::new(make_exec)
.file_name()
@@ -434,6 +451,23 @@ mod tests {
assert!(phase_targets("", &[]).is_empty());
}
#[test]
fn test_cmake_uses_default_ctest_without_explicit_targets() {
assert!(cmake_uses_default_ctest(&BuildFlags::default()));
let explicit_single = BuildFlags {
make_test_target: "test".into(),
..BuildFlags::default()
};
assert!(!cmake_uses_default_ctest(&explicit_single));
let explicit_many = BuildFlags {
make_test_targets: vec!["check".into()],
..BuildFlags::default()
};
assert!(!cmake_uses_default_ctest(&explicit_many));
}
#[test]
fn test_cmake_generator_for_make_exec_detects_ninja_and_make() {
assert_eq!(cmake_generator_for_make_exec("ninja"), Some("Ninja"));
+67
View File
@@ -83,6 +83,36 @@ pub fn build(
anyhow::bail!("ninja build failed");
}
if flags.skip_tests {
crate::log_info!("Skipping tests: disabled by build.flags.skip_tests");
} else {
let test_suites = meson_test_suites(flags);
if test_suites.is_empty() {
crate::log_info!("Running meson test...");
} else {
crate::log_info!("Running meson test suite(s): {}...", test_suites.join(" "));
}
let mut test_cmd = Command::new("meson");
test_cmd.current_dir(&build_dir);
test_cmd.arg("test");
test_cmd.arg("-C").arg(&build_dir);
test_cmd.arg("--num-processes").arg(num_cpus().to_string());
test_cmd.arg("--print-errorlogs");
for suite in &test_suites {
test_cmd.arg("--suite").arg(suite);
}
crate::builder::prepare_tool_command(&mut test_cmd, &env_vars);
let status = test_cmd
.status()
.with_context(|| format!("Failed to run meson test for {}", spec.package.name))?;
if !status.success() {
anyhow::bail!("meson test failed");
}
}
crate::source::hooks::run_post_compile_commands(spec, &actual_src, destdir)?;
state.mark_done(BuildStep::PostCompileDone)?;
} else {
@@ -146,6 +176,21 @@ fn resolve_build_dir(actual_src: &Path, flags: &crate::package::BuildFlags) -> P
}
}
fn meson_test_suites(flags: &crate::package::BuildFlags) -> Vec<String> {
let mut suites = Vec::new();
let single = flags.make_test_target.trim();
if !single.is_empty() {
suites.push(single.to_string());
}
for suite in &flags.make_test_targets {
let trimmed = suite.trim();
if !trimmed.is_empty() {
suites.push(trimmed.to_string());
}
}
suites
}
fn has_option(configure: &[String], long: &str) -> bool {
let prefix = format!("{long}=");
for arg in configure {
@@ -407,6 +452,28 @@ mod tests {
);
}
#[test]
fn test_meson_test_suites_uses_single_and_multiple_targets() {
let flags = BuildFlags {
make_test_target: "unit".to_string(),
make_test_targets: vec!["integration".to_string(), " smoke ".to_string()],
..BuildFlags::default()
};
assert_eq!(
meson_test_suites(&flags),
vec![
"unit".to_string(),
"integration".to_string(),
"smoke".to_string()
]
);
}
#[test]
fn test_meson_test_suites_empty_without_targets() {
assert!(meson_test_suites(&BuildFlags::default()).is_empty());
}
#[test]
fn test_resolve_actual_src_uses_source_subdir_under_source() -> Result<()> {
let src = tempdir()?;
+73 -4
View File
@@ -489,10 +489,13 @@ fn extract_package_archive_to_staging(
Ok(tmp_dir)
}
fn build_type_runs_automatic_tests(build_type: package::BuildType) -> bool {
fn build_type_runs_automatic_tests(spec: &package::PackageSpec) -> bool {
matches!(
build_type,
package::BuildType::Autotools | package::BuildType::CMake | package::BuildType::Perl
spec.build.build_type,
package::BuildType::Autotools
| package::BuildType::CMake
| package::BuildType::Meson
| package::BuildType::Perl
)
}
@@ -501,7 +504,7 @@ fn maybe_disable_tests_for_missing_deps(
db_path: &Path,
) -> Result<()> {
if pkg_spec.build.flags.skip_tests
|| !build_type_runs_automatic_tests(pkg_spec.build.build_type)
|| !build_type_runs_automatic_tests(pkg_spec)
|| pkg_spec.dependencies.test.is_empty()
{
return Ok(());
@@ -4435,6 +4438,48 @@ pub fn run(cli: Cli) -> Result<()> {
#[cfg(test)]
mod tests {
use super::*;
fn test_package_spec(
build_type: package::BuildType,
make_test_target: Option<&str>,
make_test_targets: &[&str],
) -> package::PackageSpec {
let mut flags = package::BuildFlags::default();
if let Some(target) = make_test_target {
flags.make_test_target = target.to_string();
}
flags.make_test_targets = make_test_targets
.iter()
.map(|target| (*target).to_string())
.collect();
package::PackageSpec {
package: package::PackageInfo {
name: "pkg".into(),
version: "1.0".into(),
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
manual_sources: Vec::new(),
source: vec![package::Source {
url: "https://example.test/pkg.tar.gz".into(),
sha256: "skip".into(),
extract_dir: "pkg".into(),
patches: Vec::new(),
post_extract: Vec::new(),
cherry_pick: Vec::new(),
}],
build: package::Build { build_type, flags },
dependencies: package::Dependencies::default(),
package_alternatives: Default::default(),
package_dependencies: Default::default(),
spec_dir: PathBuf::from("."),
}
}
use anyhow::Context;
use std::io::Write;
@@ -5450,6 +5495,30 @@ optional = []
assert_eq!(install_success_action(true), "updated");
}
#[test]
fn build_type_runs_automatic_tests_matches_builder_behavior() {
assert!(build_type_runs_automatic_tests(&test_package_spec(
package::BuildType::Autotools,
None,
&[]
)));
assert!(build_type_runs_automatic_tests(&test_package_spec(
package::BuildType::Perl,
None,
&[]
)));
assert!(build_type_runs_automatic_tests(&test_package_spec(
package::BuildType::Meson,
None,
&[]
)));
assert!(build_type_runs_automatic_tests(&test_package_spec(
package::BuildType::CMake,
None,
&[]
)));
}
#[test]
fn rootfs_is_system_root_detects_live_rootfs() {
assert!(rootfs_is_system_root(Path::new("/")));
+66 -1
View File
@@ -142,7 +142,7 @@ fn is_dep_satisfied(
fn build_type_runs_automatic_tests(spec: &PackageSpec) -> bool {
matches!(
spec.build.build_type,
BuildType::Autotools | BuildType::CMake | BuildType::Perl
BuildType::Autotools | BuildType::CMake | BuildType::Meson | BuildType::Perl
)
}
@@ -324,6 +324,47 @@ pub fn require_test_deps(spec: &PackageSpec, db_path: &Path) -> Result<()> {
mod tests {
use super::*;
fn test_spec_with_build(
build_type: BuildType,
configure_test_target: Option<&str>,
configure_test_targets: &[&str],
) -> PackageSpec {
let mut flags = crate::package::BuildFlags::default();
if let Some(target) = configure_test_target {
flags.make_test_target = target.to_string();
}
flags.make_test_targets = configure_test_targets
.iter()
.map(|target| (*target).to_string())
.collect();
PackageSpec {
package: crate::package::PackageInfo {
name: "foo".into(),
version: "1.0".into(),
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
manual_sources: Vec::new(),
source: vec![crate::package::Source {
url: "https://example.test/foo.tar.gz".into(),
sha256: "skip".into(),
extract_dir: "foo".into(),
patches: Vec::new(),
post_extract: Vec::new(),
cherry_pick: Vec::new(),
}],
build: crate::package::Build { build_type, flags },
dependencies: crate::package::Dependencies::default(),
package_alternatives: Default::default(),
package_dependencies: Default::default(),
spec_dir: std::path::PathBuf::from("."),
}
}
#[test]
fn test_parse_dep() {
let cases = vec![
@@ -506,4 +547,28 @@ mod tests {
let missing = check_runtime_deps(&spec, Path::new("/definitely/not/a/real/db")).unwrap();
assert_eq!(missing, vec!["python".to_string()]);
}
#[test]
fn test_build_type_runs_automatic_tests_matches_builder_behavior() {
assert!(build_type_runs_automatic_tests(&test_spec_with_build(
BuildType::Autotools,
None,
&[]
)));
assert!(build_type_runs_automatic_tests(&test_spec_with_build(
BuildType::Perl,
None,
&[]
)));
assert!(build_type_runs_automatic_tests(&test_spec_with_build(
BuildType::Meson,
None,
&[]
)));
assert!(build_type_runs_automatic_tests(&test_spec_with_build(
BuildType::CMake,
None,
&[]
)));
}
}
+343 -233
View File
@@ -4,14 +4,37 @@ use crate::package::{PackageSpec, Source};
use anyhow::{Context, Result, bail};
use filetime::FileTime;
use flate2::read::GzDecoder;
use lz4_flex::frame::FrameDecoder as Lz4FrameDecoder;
use lzma_rust2::{LzipReader, LzmaReader};
use std::fs::{self, File};
use std::io::{Cursor, Read, Write};
use std::os::unix::fs as unix_fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use tempfile::{NamedTempFile, tempdir};
use walkdir::WalkDir;
use zstd::stream::read::Decoder as ZstdDecoder;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ArchiveFormat {
TarGz,
TarXz,
TarBz2,
TarZst,
TarLz4,
TarLzma,
TarLzip,
TarCompress,
Zip,
Tar,
Deb,
Rpm,
Cpio,
GzFile,
XzFile,
ZstFile,
}
/// Extract an archive source to the build directory.
pub fn extract_archive(
archive_path: &Path,
@@ -38,30 +61,24 @@ pub fn extract_archive(
crate::log_info!("Extracting: {}", filename);
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
extract_tar_gz(archive_path, &extract_path)?;
} else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") {
extract_tar_xz(archive_path, &extract_path)?;
} else if filename.ends_with(".tar.bz2") || filename.ends_with(".tbz2") {
extract_tar_bz2(archive_path, &extract_path)?;
} else if filename.ends_with(".tar.zst") || filename.ends_with(".tzst") {
extract_tar_zst(archive_path, &extract_path)?;
} else if filename.ends_with(".zip") {
extract_zip(archive_path, &extract_path)?;
} else if filename.ends_with(".tar") {
extract_tar(archive_path, &extract_path)?;
} else if filename.ends_with(".deb") {
extract_deb(archive_path, &extract_path)?;
} else if filename.ends_with(".rpm") {
extract_rpm(archive_path, &extract_path)?;
} else if filename.ends_with(".gz") {
extract_gz_file(archive_path, &extract_path)?;
} else if filename.ends_with(".xz") {
extract_xz_file(archive_path, &extract_path)?;
} else if filename.ends_with(".zst") {
extract_zst_file(archive_path, &extract_path)?;
} else {
bail!("Unsupported archive format: {}", filename);
match archive_format_for_filename(filename) {
Some(ArchiveFormat::TarGz) => extract_tar_gz(archive_path, &extract_path)?,
Some(ArchiveFormat::TarXz) => extract_tar_xz(archive_path, &extract_path)?,
Some(ArchiveFormat::TarBz2) => extract_tar_bz2(archive_path, &extract_path)?,
Some(ArchiveFormat::TarZst) => extract_tar_zst(archive_path, &extract_path)?,
Some(ArchiveFormat::TarLz4) => extract_tar_lz4(archive_path, &extract_path)?,
Some(ArchiveFormat::TarLzma) => extract_tar_lzma(archive_path, &extract_path)?,
Some(ArchiveFormat::TarLzip) => extract_tar_lzip(archive_path, &extract_path)?,
Some(ArchiveFormat::TarCompress) => extract_tar_compress(archive_path, &extract_path)?,
Some(ArchiveFormat::Zip) => extract_zip(archive_path, &extract_path)?,
Some(ArchiveFormat::Tar) => extract_tar(archive_path, &extract_path)?,
Some(ArchiveFormat::Deb) => extract_deb(archive_path, &extract_path)?,
Some(ArchiveFormat::Rpm) => extract_rpm(archive_path, &extract_path)?,
Some(ArchiveFormat::Cpio) => extract_cpio(archive_path, &extract_path)?,
Some(ArchiveFormat::GzFile) => extract_gz_file(archive_path, &extract_path)?,
Some(ArchiveFormat::XzFile) => extract_xz_file(archive_path, &extract_path)?,
Some(ArchiveFormat::ZstFile) => extract_zst_file(archive_path, &extract_path)?,
None => bail!("Unsupported archive format: {}", filename),
}
if !extract_path.exists() {
@@ -75,187 +92,65 @@ pub fn extract_archive(
Ok(extract_path)
}
fn archive_format_for_filename(filename: &str) -> Option<ArchiveFormat> {
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
Some(ArchiveFormat::TarGz)
} else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") {
Some(ArchiveFormat::TarXz)
} else if filename.ends_with(".tar.bz2") || filename.ends_with(".tbz2") {
Some(ArchiveFormat::TarBz2)
} else if filename.ends_with(".tar.zst") || filename.ends_with(".tzst") {
Some(ArchiveFormat::TarZst)
} else if filename.ends_with(".tar.lz4") {
Some(ArchiveFormat::TarLz4)
} else if filename.ends_with(".tar.lzma") {
Some(ArchiveFormat::TarLzma)
} else if filename.ends_with(".tar.lz") {
Some(ArchiveFormat::TarLzip)
} else if filename.ends_with(".tar.Z") {
Some(ArchiveFormat::TarCompress)
} else if filename.ends_with(".zip") {
Some(ArchiveFormat::Zip)
} else if filename.ends_with(".tar") {
Some(ArchiveFormat::Tar)
} else if filename.ends_with(".deb") {
Some(ArchiveFormat::Deb)
} else if filename.ends_with(".rpm") {
Some(ArchiveFormat::Rpm)
} else if filename.ends_with(".cpio") {
Some(ArchiveFormat::Cpio)
} else if filename.ends_with(".gz") {
Some(ArchiveFormat::GzFile)
} else if filename.ends_with(".xz") {
Some(ArchiveFormat::XzFile)
} else if filename.ends_with(".zst") {
Some(ArchiveFormat::ZstFile)
} else {
None
}
}
fn extract_tar_gz(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let decoder = GzDecoder::new(file);
let mut archive = tar::Archive::new(decoder);
archive.unpack(tmp.path())?;
// If the archive produced a single top-level directory, decide whether to
// strip that top directory (source tarballs like foo-1.2.3/) or preserve
// it (system-layout archives like usr/). Otherwise move all top-level
// entries into `dest` so `dest` always contains the source root.
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
extract_tar_reader(decoder, dest)
}
fn extract_tar_xz(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let decoder = xz2::read::XzDecoder::new(file);
let mut archive = tar::Archive::new(decoder);
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
extract_tar_reader(decoder, dest)
}
fn extract_tar_bz2(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let decoder = bzip2::read::BzDecoder::new(file);
let mut archive = tar::Archive::new(decoder);
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
extract_tar_reader(decoder, dest)
}
fn extract_tar(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let mut archive = tar::Archive::new(file);
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
extract_tar_reader(file, dest)
}
fn extract_zip(path: &Path, dest: &Path) -> Result<()> {
@@ -263,60 +158,84 @@ fn extract_zip(path: &Path, dest: &Path) -> Result<()> {
let file = File::open(path)?;
let mut archive = zip::ZipArchive::new(file)?;
archive.extract(tmp.path())?;
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(top[0].path(), &dest_top).is_err() {
copy_dir_recursive_local(&top[0].path(), &dest_top)?;
fs::remove_dir_all(top[0].path())?;
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
}
Ok(())
finalize_extracted_tree(tmp.path(), dest)
}
fn extract_tar_zst(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
let decoder = ZstdDecoder::new(file)?;
let mut archive = tar::Archive::new(decoder);
archive.unpack(tmp.path())?;
extract_tar_reader(decoder, dest)
}
let top = fs::read_dir(tmp.path())?
fn extract_tar_lz4(path: &Path, dest: &Path) -> Result<()> {
let file = File::open(path)?;
let decoder = Lz4FrameDecoder::new(file);
extract_tar_reader(decoder, dest)
}
fn extract_tar_lzma(path: &Path, dest: &Path) -> Result<()> {
let file = File::open(path)?;
let decoder = LzmaReader::new_mem_limit(file, u32::MAX, None)?;
extract_tar_reader(decoder, dest)
}
fn extract_tar_lzip(path: &Path, dest: &Path) -> Result<()> {
let file = File::open(path)?;
let decoder = LzipReader::new(file);
extract_tar_reader(decoder, dest)
}
fn extract_tar_compress(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let mut child = Command::new("gzip");
child.arg("-cd").arg(path);
child.stdout(Stdio::piped());
let mut child = child.spawn().with_context(|| {
format!(
"Failed to spawn gzip for .Z decompression: {}",
path.display()
)
})?;
let stdout = child
.stdout
.take()
.context("Failed to capture gzip stdout")?;
let mut archive = tar::Archive::new(stdout);
archive
.unpack(tmp.path())
.with_context(|| format!("Failed to unpack .tar.Z archive {}", path.display()))?;
drop(archive);
let status = child
.wait()
.with_context(|| format!("Failed waiting for gzip on {}", path.display()))?;
if !status.success() {
bail!("gzip failed while decompressing {}", path.display());
}
finalize_extracted_tree(tmp.path(), dest)
}
fn extract_cpio(path: &Path, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let file = File::open(path)?;
extract_cpio_newc_from_reader(file, tmp.path())?;
finalize_extracted_tree(tmp.path(), dest)
}
fn extract_tar_reader<R: Read>(reader: R, dest: &Path) -> Result<()> {
let tmp = tempdir()?;
let mut archive = tar::Archive::new(reader);
archive.unpack(tmp.path())?;
finalize_extracted_tree(tmp.path(), dest)
}
fn finalize_extracted_tree(src_root: &Path, dest: &Path) -> Result<()> {
let top = fs::read_dir(src_root)?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
@@ -330,10 +249,8 @@ fn extract_tar_zst(path: &Path, dest: &Path) -> Result<()> {
|| looks_like_versioned(&top_name));
if should_strip {
// strip the single top-level folder (move its contents into dest)
move_dir_contents(&top[0].path(), dest)?;
} else {
// preserve the top-level folder (move the directory itself under dest)
fs::create_dir_all(dest)?;
let dest_top = dest.join(top_name);
if fs::rename(top[0].path(), &dest_top).is_err() {
@@ -342,7 +259,7 @@ fn extract_tar_zst(path: &Path, dest: &Path) -> Result<()> {
}
}
} else {
move_dir_contents(tmp.path(), dest)?;
move_dir_contents(src_root, dest)?;
}
Ok(())
}
@@ -684,10 +601,64 @@ fn copy_dir_recursive_local(src: &Path, dst: &Path) -> Result<()> {
#[cfg(test)]
mod tests {
use super::*;
use crate::package::{Build, BuildFlags, BuildType, Dependencies, PackageInfo};
use lz4_flex::frame::FrameEncoder as Lz4FrameEncoder;
use lzma_rust2::{LzipOptions, LzipWriter, LzmaOptions, LzmaWriter};
use std::io::Write;
use std::time::{Duration, SystemTime};
use tempfile::tempdir;
fn test_spec() -> PackageSpec {
PackageSpec {
package: PackageInfo {
name: "pkg".into(),
version: "1.0".into(),
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
manual_sources: Vec::new(),
source: Vec::new(),
build: Build {
build_type: BuildType::Custom,
flags: BuildFlags::default(),
},
dependencies: Dependencies::default(),
package_alternatives: Default::default(),
package_dependencies: Default::default(),
spec_dir: PathBuf::from("."),
}
}
fn test_source(extract_dir: &str) -> Source {
Source {
url: "https://example.test/src.tar".into(),
sha256: "skip".into(),
extract_dir: extract_dir.into(),
patches: Vec::new(),
post_extract: Vec::new(),
cherry_pick: Vec::new(),
}
}
fn simple_tar_bytes(top_dir: &str, file_name: &str, contents: &[u8]) -> Vec<u8> {
let mut tar_buf = Vec::new();
{
let mut tar = tar::Builder::new(&mut tar_buf);
let mut header = tar::Header::new_gnu();
header.set_size(contents.len() as u64);
header.set_mode(0o644);
header.set_cksum();
tar.append_data(&mut header, format!("{top_dir}/{file_name}"), contents)
.unwrap();
tar.finish().unwrap();
}
tar_buf
}
#[test]
fn test_extract_deb_roundtrip() {
let tmp = tempdir().unwrap();
@@ -733,6 +704,145 @@ mod tests {
assert!(extract_dir.join("usr/bin/hello-deb").exists());
}
#[test]
fn test_archive_format_for_new_extensions() {
assert_eq!(
archive_format_for_filename("pkg.tar.lz4"),
Some(ArchiveFormat::TarLz4)
);
assert_eq!(
archive_format_for_filename("pkg.tar.lzma"),
Some(ArchiveFormat::TarLzma)
);
assert_eq!(
archive_format_for_filename("pkg.tar.lz"),
Some(ArchiveFormat::TarLzip)
);
assert_eq!(
archive_format_for_filename("pkg.tar.Z"),
Some(ArchiveFormat::TarCompress)
);
assert_eq!(
archive_format_for_filename("pkg.cpio"),
Some(ArchiveFormat::Cpio)
);
}
#[test]
fn test_extract_archive_tar_lz4_roundtrip() {
let tmp = tempdir().unwrap();
let archive_path = tmp.path().join("pkg.tar.lz4");
let build_dir = tmp.path().join("build");
let tar_buf = simple_tar_bytes("pkg-1.0", "hello.txt", b"hello-lz4");
let mut compressed = Vec::new();
{
let mut encoder = Lz4FrameEncoder::new(&mut compressed);
encoder.write_all(&tar_buf).unwrap();
encoder.finish().unwrap();
}
fs::write(&archive_path, compressed).unwrap();
let extracted = extract_archive(
&archive_path,
&test_spec(),
&test_source("pkg-1.0"),
&build_dir,
)
.unwrap();
assert_eq!(
fs::read_to_string(extracted.join("hello.txt")).unwrap(),
"hello-lz4"
);
}
#[test]
fn test_extract_archive_tar_lzma_roundtrip() {
let tmp = tempdir().unwrap();
let archive_path = tmp.path().join("pkg.tar.lzma");
let build_dir = tmp.path().join("build");
let tar_buf = simple_tar_bytes("pkg-1.0", "hello.txt", b"hello-lzma");
let mut compressed = Vec::new();
{
let options = LzmaOptions::default();
let mut writer =
LzmaWriter::new_use_header(&mut compressed, &options, Some(tar_buf.len() as u64))
.unwrap();
writer.write_all(&tar_buf).unwrap();
writer.finish().unwrap();
}
fs::write(&archive_path, compressed).unwrap();
let extracted = extract_archive(
&archive_path,
&test_spec(),
&test_source("pkg-1.0"),
&build_dir,
)
.unwrap();
assert_eq!(
fs::read_to_string(extracted.join("hello.txt")).unwrap(),
"hello-lzma"
);
}
#[test]
fn test_extract_archive_tar_lzip_roundtrip() {
let tmp = tempdir().unwrap();
let archive_path = tmp.path().join("pkg.tar.lz");
let build_dir = tmp.path().join("build");
let tar_buf = simple_tar_bytes("pkg-1.0", "hello.txt", b"hello-lzip");
let mut compressed = Vec::new();
{
let mut writer = LzipWriter::new(&mut compressed, LzipOptions::default());
writer.write_all(&tar_buf).unwrap();
writer.finish().unwrap();
}
fs::write(&archive_path, compressed).unwrap();
let extracted = extract_archive(
&archive_path,
&test_spec(),
&test_source("pkg-1.0"),
&build_dir,
)
.unwrap();
assert_eq!(
fs::read_to_string(extracted.join("hello.txt")).unwrap(),
"hello-lzip"
);
}
#[test]
fn test_extract_archive_cpio_roundtrip() {
let tmp = tempdir().unwrap();
let archive_path = tmp.path().join("pkg.cpio");
let build_dir = tmp.path().join("build");
let mut cpio = Vec::new();
write_cpio_newc_one_file(&mut cpio, "pkg-1.0/hello.txt", b"hello-cpio");
write_cpio_trailer(&mut cpio);
fs::write(&archive_path, cpio).unwrap();
let extracted = extract_archive(
&archive_path,
&test_spec(),
&test_source("pkg-1.0"),
&build_dir,
)
.unwrap();
assert_eq!(
fs::read_to_string(extracted.join("hello.txt")).unwrap(),
"hello-cpio"
);
}
#[test]
fn copy_file_preserve_metadata_keeps_mtime() {
let tmp = tempdir().unwrap();