Refactor and enhance source management and installation processes

- Improved error handling and context messages in `checkout` function in `git.rs`.
- Updated license field in package specification to use a vector in `hooks.rs`.
- Enhanced `copy_manual_sources` function in `mod.rs` to support both local file and remote URL sources, including checksum verification.
- Added new utility functions for path normalization and skipping installation of specific files in `staging/mod.rs`.
- Implemented logic to handle existing files during installation, allowing for `.depotnew` suffix for kept files.
- Added tests for manual source copying, installation behavior, and path validation.
- Introduced a new LICENSE file with MIT License terms.
This commit is contained in:
2026-02-21 13:25:14 -06:00
parent af0571c33b
commit c9bed308e2
27 changed files with 3868 additions and 939 deletions
+359 -70
View File
@@ -14,78 +14,28 @@ pub fn build(
src_dir: &Path,
destdir: &Path,
cross: Option<&CrossConfig>,
export_compiler_flags: bool,
) -> Result<()> {
let flags = &spec.build.flags;
// Determine actual source directory (support source_subdir)
let actual_src = if !flags.source_subdir.is_empty() {
src_dir.join(&flags.source_subdir)
} else {
src_dir.to_path_buf()
};
if !actual_src.exists() {
anyhow::bail!(
"Source directory not found: {} (source_subdir: {})",
actual_src.display(),
flags.source_subdir
);
}
let export_compiler_flags = export_compiler_flags && !flags.no_flags;
let actual_src = resolve_actual_src(spec, src_dir)?;
// Create destdir
fs::create_dir_all(destdir)?;
// Build environment variables
let mut env_vars: Vec<(&str, String)> = vec![];
// Use cross-compilation tools if configured
let (cc, ar) = if let Some(cc_cfg) = cross {
(cc_cfg.cc.clone(), cc_cfg.ar.clone())
let mut env_vars = crate::builder::standard_build_env(spec, cross, true, export_compiler_flags);
let cc = if let Some(cc_cfg) = cross {
cc_cfg.cc.clone()
} else {
(flags.cc.clone(), flags.ar.clone())
flags.cc.clone()
};
if !flags.cflags.is_empty() {
if export_compiler_flags && !flags.cflags.is_empty() {
// Expand shell command substitutions like $($CC -print-resource-dir)
let cflags_str = flags.cflags.join(" ");
let expanded = expand_shell_commands(&cflags_str, &cc)?;
env_vars.push(("CFLAGS", expanded));
}
if !flags.ldflags.is_empty() {
env_vars.push(("LDFLAGS", flags.ldflags.join(" ")));
}
env_vars.push(("CC", cc.clone()));
env_vars.push(("AR", ar));
// CARCH support
if !flags.carch.is_empty() {
env_vars.push(("CARCH", flags.carch.clone()));
}
// Export rootfs for build scripts
env_vars.push(("DEPOT_ROOTFS", flags.rootfs.clone()));
// Add cross-compilation environment
if let Some(cc_cfg) = cross {
env_vars.push(("CXX", cc_cfg.cxx.clone()));
env_vars.push(("RANLIB", cc_cfg.ranlib.clone()));
env_vars.push(("STRIP", cc_cfg.strip.clone()));
env_vars.push(("LD", cc_cfg.ld.clone()));
env_vars.push(("NM", cc_cfg.nm.clone()));
}
// Add dynamic loader flag if specified
if !flags.libc.is_empty() {
let ldflags = if flags.ldflags.is_empty() {
format!("-Wl,--dynamic-linker={}", flags.libc)
} else {
format!(
"{} -Wl,--dynamic-linker={}",
flags.ldflags.join(" "),
flags.libc
)
};
env_vars.push(("LDFLAGS", ldflags));
crate::builder::set_env_var(&mut env_vars, "CFLAGS", expanded);
}
use crate::builder::state::{BuildStep, StateTracker};
@@ -117,18 +67,47 @@ pub fn build(
configure_cmd.arg(format!("--prefix={}", flags.prefix));
if !flags.chost.is_empty() {
configure_cmd.arg(format!("--host={}", flags.chost));
}
if !flags.cbuild.is_empty() {
configure_cmd.arg(format!("--build={}", flags.cbuild));
// Some projects use non-GNU configure scripts that reject --host/--build.
// Probe support first and only add these options when advertised.
let help_text = configure_help_text(configure_path, &build_dir, &env_vars);
let supports_host = help_text
.as_deref()
.map(|s| configure_help_supports_option(s, "--host"))
.unwrap_or(true);
let supports_build = help_text
.as_deref()
.map(|s| configure_help_supports_option(s, "--build"))
.unwrap_or(true);
let requested_host = if let Some(cc_cfg) = cross {
Some(cc_cfg.host_triple().to_string())
} else if !flags.chost.is_empty() {
Some(flags.chost.clone())
} else {
None
};
let requested_build = if cross.is_some() {
CrossConfig::build_triple().ok()
} else if !flags.cbuild.is_empty() {
Some(flags.cbuild.clone())
} else {
None
};
if let Some(host) = requested_host {
if supports_host {
configure_cmd.arg(format!("--host={}", host));
} else {
println!(" configure does not support --host; skipping {}", host);
}
}
// Add cross-compilation flags
if let Some(cc_cfg) = cross {
configure_cmd.arg(format!("--host={}", cc_cfg.host_triple()));
if let Ok(build) = CrossConfig::build_triple() {
if let Some(build) = requested_build {
if supports_build {
configure_cmd.arg(format!("--build={}", build));
} else {
println!(" configure does not support --build; skipping {}", build);
}
}
@@ -154,6 +133,7 @@ pub fn build(
let mut make_cmd = Command::new("make");
make_cmd.current_dir(&build_dir);
make_cmd.arg("-j").arg(num_cpus().to_string());
add_make_variable_overrides(&mut make_cmd, &flags.make_vars, "build")?;
crate::builder::prepare_command(&mut make_cmd, &env_vars);
@@ -165,6 +145,30 @@ pub fn build(
anyhow::bail!("make failed with status: {}", status);
}
if let Some(test_target) = maybe_find_autotools_test_target(&build_dir, flags.skip_tests)? {
println!("Running make {}...", test_target);
let mut test_cmd = Command::new("make");
test_cmd.current_dir(&build_dir);
add_make_variable_overrides(&mut test_cmd, &flags.make_test_vars, "test")?;
test_cmd.arg(test_target);
crate::builder::prepare_command(&mut test_cmd, &env_vars);
let status = test_cmd.status().with_context(|| {
format!(
"Failed to run make {} in {}",
test_target,
build_dir.display()
)
})?;
if !status.success() {
anyhow::bail!("make {} failed with status: {}", test_target, status);
}
} else if flags.skip_tests {
println!("Skipping tests: disabled by build.flags.skip_tests");
} else {
println!("Skipping tests: no 'check' or 'test' target in Makefile");
}
// Run post-compile hooks (after make, before make install)
hooks::run_post_compile_commands(spec, &actual_src, destdir)?;
state.mark_done(BuildStep::PostCompileDone)?;
@@ -185,10 +189,17 @@ pub fn build(
let mut install_cmd = fakeroot::wrap_install_command("make", destdir);
install_cmd.current_dir(&build_dir);
if !has_make_variable_override(&flags.make_install_vars, "DESTDIR") {
install_cmd.arg(format!("DESTDIR={}", destdir.to_string_lossy()));
}
add_make_variable_overrides(&mut install_cmd, &flags.make_install_vars, "install")?;
install_cmd.arg("install");
let mut install_env = env_vars.clone();
install_env.push(("DESTDIR", destdir.to_string_lossy().into_owned()));
install_env.push((
"DESTDIR".to_string(),
destdir.to_string_lossy().into_owned(),
));
crate::builder::prepare_command(&mut install_cmd, &install_env);
let status = install_cmd
@@ -215,6 +226,155 @@ fn num_cpus() -> usize {
.unwrap_or(1)
}
fn resolve_actual_src(spec: &PackageSpec, src_dir: &Path) -> Result<std::path::PathBuf> {
let flags = &spec.build.flags;
let source_subdir = spec.expand_vars(&flags.source_subdir);
let actual_src = if source_subdir.is_empty() {
src_dir.to_path_buf()
} else {
src_dir.join(&source_subdir)
};
if !actual_src.exists() {
anyhow::bail!(
"Source directory not found: {} (source_subdir: {} -> {})",
actual_src.display(),
flags.source_subdir,
source_subdir
);
}
Ok(actual_src)
}
fn configure_help_text(
configure_path: &str,
build_dir: &Path,
env_vars: &crate::builder::EnvVars,
) -> Option<String> {
let mut help_cmd = Command::new(configure_path);
help_cmd.current_dir(build_dir);
help_cmd.arg("--help");
crate::builder::prepare_command(&mut help_cmd, env_vars);
let output = help_cmd.output().ok()?;
if !output.status.success() {
return None;
}
let mut text = String::new();
text.push_str(&String::from_utf8_lossy(&output.stdout));
text.push_str(&String::from_utf8_lossy(&output.stderr));
Some(text)
}
fn configure_help_supports_option(help_text: &str, option: &str) -> bool {
let with_eq = format!("{}=", option);
let with_space = format!("{} ", option);
help_text.contains(&with_eq) || help_text.contains(&with_space) || help_text.contains(option)
}
fn find_autotools_test_target(build_dir: &Path) -> Result<Option<&'static str>> {
for target in ["check", "test"] {
if makefile_has_target(build_dir, target)? {
return Ok(Some(target));
}
}
Ok(None)
}
fn maybe_find_autotools_test_target(
build_dir: &Path,
skip_tests: bool,
) -> Result<Option<&'static str>> {
if skip_tests {
return Ok(None);
}
find_autotools_test_target(build_dir)
}
fn makefile_has_target(build_dir: &Path, target: &str) -> Result<bool> {
for name in ["GNUmakefile", "Makefile", "makefile"] {
let path = build_dir.join(name);
if !path.exists() {
continue;
}
let content = fs::read_to_string(&path)
.with_context(|| format!("Failed to read makefile: {}", path.display()))?;
if makefile_content_has_target(&content, target) {
return Ok(true);
}
}
Ok(false)
}
fn makefile_content_has_target(content: &str, target: &str) -> bool {
for raw in content.lines() {
let line = raw.trim_start();
if line.is_empty() || line.starts_with('#') || line.starts_with('\t') {
continue;
}
if let Some(rest) = line.strip_prefix(".PHONY:") {
if rest.split_whitespace().any(|t| t == target) {
return true;
}
continue;
}
let Some(colon_pos) = line.find(':') else {
continue;
};
let rhs = &line[colon_pos + 1..];
if rhs.starts_with('=') {
// Variable assignment (e.g. FOO:=bar), not a make target.
continue;
}
let lhs = &line[..colon_pos];
if lhs.split_whitespace().any(|t| t == target) {
return true;
}
}
false
}
fn add_make_variable_overrides(cmd: &mut Command, vars: &[String], phase: &str) -> Result<()> {
for raw in vars {
let var = raw.trim();
if var.is_empty() {
continue;
}
let Some((name, _value)) = var.split_once('=') else {
anyhow::bail!(
"Invalid make variable override '{}' for {} phase; expected NAME=VALUE",
var,
phase
);
};
let name = name.trim();
if name.is_empty() || name.contains(char::is_whitespace) {
anyhow::bail!(
"Invalid make variable override '{}' for {} phase; expected NAME=VALUE",
var,
phase
);
}
cmd.arg(var);
}
Ok(())
}
fn has_make_variable_override(vars: &[String], name: &str) -> bool {
vars.iter().any(|raw| {
let var = raw.trim();
let Some((lhs, _rhs)) = var.split_once('=') else {
return false;
};
lhs.trim() == name
})
}
/// Expand shell command substitutions like $($CC -print-resource-dir) in a string
fn expand_shell_commands(input: &str, cc: &str) -> Result<String> {
let mut result = input.to_string();
@@ -253,6 +413,9 @@ fn expand_shell_commands(input: &str, cc: &str) -> Result<String> {
#[cfg(test)]
mod tests {
use super::*;
use crate::package::{Build, BuildFlags, BuildType, Dependencies, PackageInfo, PackageSpec};
use std::path::PathBuf;
use tempfile::tempdir;
#[test]
fn test_expand_shell_commands_simple() -> Result<()> {
@@ -266,7 +429,7 @@ mod tests {
// The command contains $CC which should be replaced with provided cc
let out = expand_shell_commands("start $($CC -v >/dev/null; echo OK) end", "mycc")?;
// Since the inner command echoes OK, after replacing $CC it should run and include OK
assert!(out.contains("OK") || out.contains("") );
assert!(out.contains("OK") || out.contains(""));
Ok(())
}
@@ -275,4 +438,130 @@ mod tests {
let n = num_cpus();
assert!(n >= 1);
}
#[test]
fn test_configure_help_supports_host_build() {
let help = "Usage: configure [OPTION]...\n --host=HOST cross host\n --build=BUILD";
assert!(configure_help_supports_option(help, "--host"));
assert!(configure_help_supports_option(help, "--build"));
assert!(!configure_help_supports_option(help, "--target"));
}
#[test]
fn test_makefile_content_has_target_detects_check_and_test() {
let content = r#"
.PHONY: all check
all:
@echo all
check:
@echo check
"#;
assert!(makefile_content_has_target(content, "check"));
assert!(!makefile_content_has_target(content, "test"));
}
#[test]
fn test_makefile_content_has_target_ignores_assignments() {
let content = r#"
TEST := value
VAR:=$(shell echo hi)
foo: bar
@true
"#;
assert!(!makefile_content_has_target(content, "TEST"));
assert!(!makefile_content_has_target(content, "VAR"));
assert!(!makefile_content_has_target(content, "check"));
}
#[test]
fn test_maybe_find_autotools_test_target_respects_skip_tests() -> Result<()> {
let tmp = tempdir().unwrap();
std::fs::write(tmp.path().join("Makefile"), "check:\n\t@true\n").unwrap();
let skipped = maybe_find_autotools_test_target(tmp.path(), true)?;
assert_eq!(skipped, None);
let detected = maybe_find_autotools_test_target(tmp.path(), false)?;
assert_eq!(detected, Some("check"));
Ok(())
}
#[test]
fn test_add_make_variable_overrides_accepts_valid_assignments() -> Result<()> {
let mut cmd = Command::new("make");
add_make_variable_overrides(
&mut cmd,
&[
"CC=clang".to_string(),
"V=1".to_string(),
" CFLAGS=-O2 -pipe ".to_string(),
],
"build",
)?;
let args: Vec<String> = cmd
.get_args()
.map(|a| a.to_string_lossy().to_string())
.collect();
assert_eq!(args, vec!["CC=clang", "V=1", "CFLAGS=-O2 -pipe"]);
Ok(())
}
#[test]
fn test_add_make_variable_overrides_rejects_invalid_assignment() {
let mut cmd = Command::new("make");
let err = add_make_variable_overrides(&mut cmd, &["not-an-assignment".to_string()], "test")
.expect_err("expected invalid assignment to fail");
assert!(err.to_string().contains("expected NAME=VALUE"));
}
#[test]
fn test_has_make_variable_override_detects_destdir() {
assert!(has_make_variable_override(
&["DESTDIR=/tmp/pkg".to_string()],
"DESTDIR"
));
assert!(has_make_variable_override(
&[" DESTDIR =/tmp/pkg ".to_string()],
"DESTDIR"
));
assert!(!has_make_variable_override(
&["V=1".to_string(), "PREFIX=/usr".to_string()],
"DESTDIR"
));
}
#[test]
fn test_resolve_actual_src_expands_source_subdir_vars() {
let tmp = tempdir().unwrap();
let src_root = tmp.path().join("srcroot");
let expanded = src_root.join("expect5.45.4").join("unix");
std::fs::create_dir_all(&expanded).unwrap();
let spec = PackageSpec {
package: PackageInfo {
name: "expect".into(),
version: "5.45.4".into(),
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
manual_sources: Vec::new(),
source: Vec::new(),
build: Build {
build_type: BuildType::Autotools,
flags: BuildFlags {
source_subdir: "$name$version/unix".into(),
..BuildFlags::default()
},
},
dependencies: Dependencies::default(),
spec_dir: PathBuf::from("."),
};
let resolved = resolve_actual_src(&spec, &src_root).unwrap();
assert_eq!(resolved, expanded);
}
}
+23 -10
View File
@@ -1,12 +1,12 @@
//! Binary package "build" system — used when package supplies a prebuilt binary installer
use crate::cross::CrossConfig;
use crate::package::PackageSpec;
use anyhow::{Context, Result};
use std::fs;
use std::os::unix::fs as unix_fs;
use std::path::Path;
use walkdir::WalkDir;
use crate::package::PackageSpec;
use crate::cross::CrossConfig;
/// For binary packages we simply copy the extracted files into DESTDIR (preserving
/// directory structure). This is useful for .deb packages where extract step
@@ -16,9 +16,16 @@ pub fn build(
src_dir: &Path,
destdir: &Path,
_cross: Option<&CrossConfig>,
_export_compiler_flags: bool,
) -> Result<()> {
println!("Binary install: copying files from {} to {} (pkg type={})", src_dir.display(), destdir.display(), _spec.build.flags.binary_type);
fs::create_dir_all(destdir).with_context(|| format!("Failed to create destdir: {}", destdir.display()))?;
println!(
"Binary install: copying files from {} to {} (pkg type={})",
src_dir.display(),
destdir.display(),
_spec.build.flags.binary_type
);
fs::create_dir_all(destdir)
.with_context(|| format!("Failed to create destdir: {}", destdir.display()))?;
for entry in WalkDir::new(src_dir) {
let entry = entry?;
@@ -28,12 +35,18 @@ pub fn build(
fs::create_dir_all(&target)?;
} else if entry.file_type().is_symlink() {
let link_target = fs::read_link(entry.path())?;
if let Some(parent) = target.parent() { fs::create_dir_all(parent)?; }
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
// overwrite existing links/files
if target.exists() { let _ = fs::remove_file(&target); }
if target.exists() {
let _ = fs::remove_file(&target);
}
unix_fs::symlink(link_target, &target)?;
} else {
if let Some(parent) = target.parent() { fs::create_dir_all(parent)?; }
if let Some(parent) = target.parent() {
fs::create_dir_all(parent)?;
}
fs::copy(entry.path(), &target)?;
}
}
@@ -45,9 +58,9 @@ pub fn build(
mod tests {
use super::*;
use crate::package::{Build, BuildFlags, BuildType, Dependencies, PackageInfo};
use tempfile::tempdir;
use std::fs;
use std::os::unix::fs as unix_fs;
use tempfile::tempdir;
fn mk_spec(name: &str, version: &str) -> PackageSpec {
PackageSpec {
@@ -57,7 +70,7 @@ mod tests {
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: "MIT".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
@@ -96,7 +109,7 @@ mod tests {
unix_fs::symlink(&target, src.join("usr/lib/libdummy.so.link"))?;
let spec = mk_spec("bin-test", "1.0");
build(&spec, src, dest, None)?;
build(&spec, src, dest, None, true)?;
// Check copied file
assert!(dest.join("usr/bin/hello").exists());
+51 -62
View File
@@ -13,13 +13,13 @@ pub fn build(
src_dir: &Path,
destdir: &Path,
cross: Option<&CrossConfig>,
export_compiler_flags: bool,
) -> Result<()> {
let flags = &spec.build.flags;
// Determine actual source directory (support source_subdir)
let actual_src = resolve_actual_src(spec, src_dir)?;
let build_dir = actual_src.join("build");
// Create directories
@@ -27,55 +27,7 @@ pub fn build(
fs::create_dir_all(destdir)?;
// Environment variables
let mut env_vars: Vec<(&str, String)> = vec![];
// Use cross-compilation tools if configured
let cc = if let Some(cc_cfg) = cross {
cc_cfg.cc.clone()
} else {
flags.cc.clone()
};
if !flags.cflags.is_empty() {
env_vars.push(("CFLAGS", flags.cflags.join(" ")));
}
if !flags.chost.is_empty() {
env_vars.push(("CHOST", flags.chost.clone()));
}
if !flags.cbuild.is_empty() {
env_vars.push(("CBUILD", flags.cbuild.clone()));
}
if !flags.ldflags.is_empty() {
let ldflags = if flags.libc.is_empty() {
flags.ldflags.join(" ")
} else {
format!(
"{} -Wl,--dynamic-linker={}",
flags.ldflags.join(" "),
flags.libc
)
};
env_vars.push(("LDFLAGS", ldflags));
}
env_vars.push(("CC", cc));
// Ensure CXX is exported for configure flags that reference $CXX
// prefer cross-config CXX when cross-compiling (handled earlier), otherwise use flags.cxx
env_vars.push(("CXX", flags.cxx.clone()));
// Export rootfs for build scripts
env_vars.push(("DEPOT_ROOTFS", flags.rootfs.clone()));
// CARCH support
if !flags.carch.is_empty() {
env_vars.push(("CARCH", flags.carch.clone()));
}
// Add cross-compilation env
if let Some(cc_cfg) = cross {
env_vars.push(("CXX", cc_cfg.cxx.clone()));
env_vars.push(("AR", cc_cfg.ar.clone()));
}
let env_vars = crate::builder::standard_build_env(spec, cross, true, export_compiler_flags);
// Extract prefix from configure flags (cmake-style -DCMAKE_INSTALL_PREFIX=)
let prefix = flags
@@ -168,7 +120,10 @@ pub fn build(
install_cmd.arg("--install").arg(&build_dir);
let mut install_env = env_vars.clone();
install_env.push(("DESTDIR", destdir.to_string_lossy().into_owned()));
install_env.push((
"DESTDIR".to_string(),
destdir.to_string_lossy().into_owned(),
));
crate::builder::prepare_command(&mut install_cmd, &install_env);
let status = install_cmd
@@ -199,7 +154,7 @@ fn expand_env_vars(input: &str) -> String {
}
/// Expand using a provided set of env vars (used to expand flags before spawning child).
fn expand_with_envs(input: &str, envs: &Vec<(&str, String)>) -> String {
fn expand_with_envs(input: &str, envs: &[(String, String)]) -> String {
let mut result = input.to_string();
for (k, v) in envs {
result = result.replace(&format!("${}", k), v);
@@ -220,29 +175,38 @@ fn num_cpus() -> usize {
/// - `src_dir/<sub>` -> use if exists
/// - `spec.spec_dir/<sub>` -> use if exists (supports `../llvm`)
/// - bare relative path (cwd)
fn resolve_actual_src(spec: &crate::package::PackageSpec, src_dir: &Path) -> anyhow::Result<std::path::PathBuf> {
fn resolve_actual_src(
spec: &crate::package::PackageSpec,
src_dir: &Path,
) -> anyhow::Result<std::path::PathBuf> {
let flags = &spec.build.flags;
if flags.source_subdir.is_empty() {
let source_subdir = spec.expand_vars(&flags.source_subdir);
if source_subdir.is_empty() {
return Ok(src_dir.to_path_buf());
}
let candidate = std::path::Path::new(&flags.source_subdir);
let candidate = std::path::Path::new(&source_subdir);
// 1) absolute path -> use directly
if candidate.is_absolute() {
if candidate.exists() {
return Ok(candidate.to_path_buf());
}
anyhow::bail!("Source directory not found: {} (source_subdir: {})", candidate.display(), flags.source_subdir);
anyhow::bail!(
"Source directory not found: {} (source_subdir: {} -> {})",
candidate.display(),
flags.source_subdir,
source_subdir
);
}
// 2) src_dir/<source_subdir>
let under_src = src_dir.join(&flags.source_subdir);
let under_src = src_dir.join(&source_subdir);
if under_src.exists() {
return Ok(under_src);
}
// 3) spec_dir/<source_subdir> (useful for ../llvm relative to spec)
let spec_path = spec.spec_dir.join(&flags.source_subdir);
let spec_path = spec.spec_dir.join(&source_subdir);
if spec_path.exists() {
return Ok(spec_path);
}
@@ -254,7 +218,8 @@ fn resolve_actual_src(spec: &crate::package::PackageSpec, src_dir: &Path) -> any
// fallback error
anyhow::bail!(
"Source directory not found: {} (tried src_dir, spec_dir, and absolute path)",
"Source directory not found: {} (expanded from '{}'; tried src_dir, spec_dir, and absolute path)",
source_subdir,
flags.source_subdir
);
}
@@ -277,7 +242,10 @@ mod tests {
#[test]
fn test_expand_with_envs_prefers_provided_envs() {
let envs = vec![("CXX", "my-cxx".to_string()), ("CC", "my-cc".to_string())];
let envs = vec![
("CXX".to_string(), "my-cxx".to_string()),
("CC".to_string(), "my-cc".to_string()),
];
let s = "-DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=${CXX} -DROOT=$HOME";
let out = expand_with_envs(s, &envs);
assert!(out.contains("my-cc"));
@@ -297,18 +265,33 @@ mod tests {
let src_root = tmp.path().join("srcroot");
let spec_dir = tmp.path().join("specdir");
let external = tmp.path().join("external");
let expanded = src_root.join("x-1.0").join("sub");
std::fs::create_dir_all(&src_root.join("sub")).unwrap();
std::fs::create_dir_all(&expanded).unwrap();
// create directories for candidates
std::fs::create_dir_all(&spec_dir.join("../llvm")).unwrap();
std::fs::create_dir_all(&external).unwrap();
let spec = PackageSpec {
package: PackageInfo { name: "x".into(), version: "1.0".into(), revision: 1, description: "".into(), homepage: "".into(), license: "MIT".into() },
package: PackageInfo {
name: "x".into(),
version: "1.0".into(),
revision: 1,
description: "".into(),
homepage: "".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
manual_sources: Vec::new(),
source: Vec::new(),
build: Build { build_type: BuildType::CMake, flags: BuildFlags { source_subdir: "sub".into(), ..BuildFlags::default() } },
build: Build {
build_type: BuildType::CMake,
flags: BuildFlags {
source_subdir: "sub".into(),
..BuildFlags::default()
},
},
dependencies: Default::default(),
spec_dir: spec_dir.clone(),
};
@@ -328,5 +311,11 @@ mod tests {
spec3.build.flags.source_subdir = external.to_string_lossy().into_owned();
let p3 = resolve_actual_src(&spec3, &src_root).unwrap();
assert_eq!(p3, external);
// case: variable expansion in source_subdir
let mut spec4 = spec.clone();
spec4.build.flags.source_subdir = "$name-$version/sub".into();
let p4 = resolve_actual_src(&spec4, &src_root).unwrap();
assert_eq!(p4, expanded);
}
}
+27 -61
View File
@@ -12,13 +12,14 @@ pub fn build(
src_dir: &Path,
destdir: &Path,
cross: Option<&CrossConfig>,
export_compiler_flags: bool,
) -> Result<()> {
let flags = &spec.build.flags;
// Create destdir
fs::create_dir_all(destdir)?;
let mut env_vars: Vec<(&str, String)> = vec![];
let mut env_vars = crate::builder::standard_build_env(spec, cross, true, export_compiler_flags);
// For custom builds, look for a build.sh script in the source directory
let build_script = src_dir.join("build.sh");
@@ -29,8 +30,12 @@ pub fn build(
let spec_build = spec.spec_dir.join("build.sh");
if !build_script.exists() && spec_build.exists() {
fs::create_dir_all(src_dir)?;
fs::copy(&spec_build, &build_script)
.with_context(|| format!("Failed to copy build.sh from spec dir: {}", spec_build.display()))?;
fs::copy(&spec_build, &build_script).with_context(|| {
format!(
"Failed to copy build.sh from spec dir: {}",
spec_build.display()
)
})?;
// Ensure executable bit
#[cfg(unix)]
{
@@ -50,7 +55,7 @@ pub fn build(
}
use crate::builder::state::{BuildStep, StateTracker};
let mut state = StateTracker::new(&src_dir)?;
let mut state = StateTracker::new(src_dir)?;
if !state.is_done(BuildStep::PostInstallDone) {
println!(
@@ -70,7 +75,8 @@ pub fn build(
src_dir.to_path_buf()
};
let mut cmd = fakeroot::wrap_install_command("bash", destdir);
// Use POSIX `sh` (more likely to be available in minimal/chroot environments)
let mut cmd = fakeroot::wrap_install_command("sh", destdir);
cmd.current_dir(&build_dir);
// Ensure build script path is absolute for when we are in a sub-build-dir
@@ -81,62 +87,22 @@ pub fn build(
};
cmd.arg(&abs_build_script);
if !flags.cflags.is_empty() {
env_vars.push(("CFLAGS", flags.cflags.join(" ")));
}
if !flags.ldflags.is_empty() {
let ldflags = if flags.libc.is_empty() {
flags.ldflags.join(" ")
} else {
format!(
"{} -Wl,--dynamic-linker={}",
flags.ldflags.join(" "),
flags.libc
)
};
env_vars.push(("LDFLAGS", ldflags));
}
env_vars.push(("DESTDIR", destdir.to_string_lossy().into_owned()));
env_vars.push(("PREFIX", spec.build.flags.prefix.clone()));
env_vars.push(("DEPOT_ROOTFS", spec.build.flags.rootfs.clone()));
env_vars.push((
"DEPOT_SPECDIR",
spec.spec_dir.to_string_lossy().into_owned(),
));
if !flags.chost.is_empty() {
env_vars.push(("CHOST", flags.chost.clone()));
}
if !flags.cbuild.is_empty() {
env_vars.push(("CBUILD", flags.cbuild.clone()));
}
if !flags.carch.is_empty() {
env_vars.push(("CARCH", flags.carch.clone()));
}
// Use cross-compilation tools if configured
if let Some(cc_cfg) = cross {
env_vars.push(("CC", cc_cfg.cc.clone()));
env_vars.push(("CXX", cc_cfg.cxx.clone()));
env_vars.push(("AR", cc_cfg.ar.clone()));
env_vars.push(("RANLIB", cc_cfg.ranlib.clone()));
env_vars.push(("STRIP", cc_cfg.strip.clone()));
env_vars.push(("LD", cc_cfg.ld.clone()));
env_vars.push(("NM", cc_cfg.nm.clone()));
env_vars.push(("CROSS_PREFIX", cc_cfg.prefix.clone()));
env_vars.push(("CROSS_COMPILE", format!("{}-", cc_cfg.prefix)));
} else {
env_vars.push(("CC", flags.cc.clone()));
env_vars.push(("AR", flags.ar.clone()));
}
crate::builder::set_env_var(
&mut env_vars,
"DESTDIR",
destdir.to_string_lossy().into_owned(),
);
crate::builder::prepare_command(&mut cmd, &env_vars);
let status = cmd
.status()
.with_context(|| format!("Failed to run build script: {}", build_script.display()))?;
// Run the command and include the OS error on spawn failures for clearer diagnostics
let status = cmd.status().map_err(|e| {
anyhow::anyhow!(
"Failed to run build script {}: {}",
build_script.display(),
e
)
})?;
if !status.success() {
anyhow::bail!("Custom build script failed with status: {}", status);
@@ -163,7 +129,7 @@ mod tests {
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: "MIT".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
@@ -191,7 +157,7 @@ mod tests {
let spec = mk_spec("custom-no-build", "1.0");
let res = build(&spec, tmp_src.path(), tmp_dest.path(), None);
let res = build(&spec, tmp_src.path(), tmp_dest.path(), None, true);
assert!(res.is_err());
Ok(())
}
@@ -217,7 +183,7 @@ mod tests {
spec.spec_dir = spec_dir.path().to_path_buf();
// src_dir is empty; build() should copy build.sh from spec_dir and run it (no-op)
let _ = build(&spec, tmp_src.path(), tmp_dest.path(), None)?;
let _ = build(&spec, tmp_src.path(), tmp_dest.path(), None, true)?;
// If we reached here, build() succeeded and build.sh was copied into src
assert!(tmp_src.path().join("build.sh").exists());
Ok(())
+14 -33
View File
@@ -10,43 +10,21 @@ pub fn build(
src_dir: &Path,
destdir: &Path,
cross: Option<&CrossConfig>,
export_compiler_flags: bool,
) -> Result<()> {
let mut state = StateTracker::new(src_dir)?;
let flags = &spec.build.flags;
// Use cross-compilation tools if configured, otherwise use flags from spec
// Note: Makefile builds might rely on CC/CXX/AR being set, or might use make flags.
// We set them in env vars.
let (cc, ar) = if let Some(cc_cfg) = cross {
(cc_cfg.cc.as_str(), cc_cfg.ar.as_str())
} else {
(flags.cc.as_str(), flags.ar.as_str())
};
let mut env_vars = crate::builder::standard_build_env(spec, cross, true, export_compiler_flags);
let mut env_vars: Vec<(&str, String)> = vec![("CC", cc.to_string()), ("AR", ar.to_string())];
let cflags;
if !flags.cflags.is_empty() {
cflags = flags.cflags.join(" ");
env_vars.push(("CFLAGS", cflags));
}
let ldflags;
if !flags.ldflags.is_empty() {
ldflags = flags.ldflags.join(" ");
env_vars.push(("LDFLAGS", ldflags));
}
// CARCH support
if !flags.carch.is_empty() {
env_vars.push(("CARCH", flags.carch.clone()));
}
// Export PKG_CONFIG_SYSROOT_DIR for pkg-config and a general DEPOT_ROOTFS
// Export PKG_CONFIG_SYSROOT_DIR for pkg-config
if !flags.rootfs.is_empty() && flags.rootfs != "/" {
env_vars.push(("PKG_CONFIG_SYSROOT_DIR", flags.rootfs.clone()));
crate::builder::set_env_var(
&mut env_vars,
"PKG_CONFIG_SYSROOT_DIR",
flags.rootfs.clone(),
);
}
env_vars.push(("DEPOT_ROOTFS", flags.rootfs.clone()));
if !state.is_done(BuildStep::PostCompileDone) {
println!("Running makefile build commands...");
@@ -95,7 +73,10 @@ pub fn build(
cmd.current_dir(src_dir);
let mut install_env = env_vars.clone();
install_env.push(("DESTDIR", destdir.to_string_lossy().into_owned()));
install_env.push((
"DESTDIR".to_string(),
destdir.to_string_lossy().into_owned(),
));
crate::builder::prepare_command(&mut cmd, &install_env);
let status = cmd
@@ -130,7 +111,7 @@ mod tests {
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: "MIT".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
@@ -168,7 +149,7 @@ mod tests {
"cp built.txt $DESTDIR/usr/bin/installed.txt".into(),
];
build(&spec, src_path, dest_path, None)?;
build(&spec, src_path, dest_path, None, true)?;
// Verify build step
let built_file = src_path.join("built.txt");
+6 -46
View File
@@ -13,6 +13,7 @@ pub fn build(
src_dir: &Path,
destdir: &Path,
cross: Option<&CrossConfig>,
export_compiler_flags: bool,
) -> Result<()> {
let flags = &spec.build.flags;
let build_dir = src_dir.join("builddir");
@@ -22,51 +23,7 @@ pub fn build(
fs::create_dir_all(destdir)?;
// Environment variables
let mut env_vars: Vec<(&str, String)> = vec![];
// Use cross-compilation tools if configured
let cc = if let Some(cc_cfg) = cross {
cc_cfg.cc.clone()
} else {
flags.cc.clone()
};
if !flags.cflags.is_empty() {
env_vars.push(("CFLAGS", flags.cflags.join(" ")));
}
if !flags.chost.is_empty() {
env_vars.push(("CHOST", flags.chost.clone()));
}
if !flags.cbuild.is_empty() {
env_vars.push(("CBUILD", flags.cbuild.clone()));
}
if !flags.ldflags.is_empty() {
let ldflags = if flags.libc.is_empty() {
flags.ldflags.join(" ")
} else {
format!(
"{} -Wl,--dynamic-linker={}",
flags.ldflags.join(" "),
flags.libc
)
};
env_vars.push(("LDFLAGS", ldflags));
}
env_vars.push(("CC", cc));
// Export rootfs for build scripts
env_vars.push(("DEPOT_ROOTFS", flags.rootfs.clone()));
// Add cross-compilation env
if let Some(cc_cfg) = cross {
env_vars.push(("CXX", cc_cfg.cxx.clone()));
env_vars.push(("AR", cc_cfg.ar.clone()));
}
// CARCH support
if !flags.carch.is_empty() {
env_vars.push(("CARCH", flags.carch.clone()));
}
let env_vars = crate::builder::standard_build_env(spec, cross, true, export_compiler_flags);
// Extract prefix from configure flags
let prefix = flags
@@ -150,7 +107,10 @@ pub fn build(
install_cmd.arg("-C").arg(&build_dir);
let mut install_env = env_vars.clone();
install_env.push(("DESTDIR", destdir.to_string_lossy().into_owned()));
install_env.push((
"DESTDIR".to_string(),
destdir.to_string_lossy().into_owned(),
));
crate::builder::prepare_command(&mut install_cmd, &install_env);
let status = install_cmd
+240 -18
View File
@@ -15,8 +15,112 @@ use anyhow::Result;
use std::path::Path;
use std::process::Command;
pub type EnvVars = Vec<(String, String)>;
pub fn set_env_var(env_vars: &mut EnvVars, key: &str, value: impl Into<String>) {
let value = value.into();
if let Some((_, existing)) = env_vars.iter_mut().find(|(k, _)| k == key) {
*existing = value;
} else {
env_vars.push((key.to_string(), value));
}
}
pub fn standard_build_env(
spec: &PackageSpec,
cross: Option<&CrossConfig>,
include_compiler_env: bool,
export_compiler_flags: bool,
) -> EnvVars {
let flags = &spec.build.flags;
let mut env_vars: EnvVars = Vec::new();
let export_compiler_flags = export_compiler_flags && !flags.no_flags;
if include_compiler_env && export_compiler_flags {
if !flags.cflags.is_empty() {
set_env_var(&mut env_vars, "CFLAGS", flags.cflags.join(" "));
}
let ldflags = if !flags.ldflags.is_empty() || !flags.libc.is_empty() {
if flags.libc.is_empty() {
flags.ldflags.join(" ")
} else if flags.ldflags.is_empty() {
format!("-Wl,--dynamic-linker={}", flags.libc)
} else {
format!(
"{} -Wl,--dynamic-linker={}",
flags.ldflags.join(" "),
flags.libc
)
}
} else {
String::new()
};
if !ldflags.is_empty() {
set_env_var(&mut env_vars, "LDFLAGS", ldflags);
}
}
if !flags.chost.is_empty() {
set_env_var(&mut env_vars, "CHOST", flags.chost.clone());
}
if !flags.cbuild.is_empty() {
set_env_var(&mut env_vars, "CBUILD", flags.cbuild.clone());
}
if !flags.carch.is_empty() {
set_env_var(&mut env_vars, "CARCH", flags.carch.clone());
}
if !flags.prefix.is_empty() {
set_env_var(&mut env_vars, "PREFIX", flags.prefix.clone());
}
set_env_var(&mut env_vars, "DEPOT_ROOTFS", flags.rootfs.clone());
set_env_var(
&mut env_vars,
"DEPOT_SPECDIR",
spec.spec_dir.to_string_lossy().into_owned(),
);
if include_compiler_env {
if let Some(cc_cfg) = cross {
set_env_var(&mut env_vars, "CC", cc_cfg.cc.clone());
set_env_var(&mut env_vars, "CXX", cc_cfg.cxx.clone());
set_env_var(&mut env_vars, "AR", cc_cfg.ar.clone());
set_env_var(&mut env_vars, "RANLIB", cc_cfg.ranlib.clone());
set_env_var(&mut env_vars, "STRIP", cc_cfg.strip.clone());
set_env_var(&mut env_vars, "LD", cc_cfg.ld.clone());
set_env_var(&mut env_vars, "NM", cc_cfg.nm.clone());
set_env_var(&mut env_vars, "CROSS_PREFIX", cc_cfg.prefix.clone());
set_env_var(
&mut env_vars,
"CROSS_COMPILE",
format!("{}-", cc_cfg.prefix),
);
} else {
set_env_var(&mut env_vars, "CC", flags.cc.clone());
set_env_var(&mut env_vars, "CXX", flags.cxx.clone());
set_env_var(&mut env_vars, "AR", flags.ar.clone());
}
}
for key in &flags.passthrough_env {
let key = key.trim();
if key.is_empty() || key.contains('=') {
continue;
}
if env_vars.iter().any(|(existing, _)| existing == key) {
continue;
}
if let Ok(value) = std::env::var(key) {
set_env_var(&mut env_vars, key, value);
}
}
env_vars
}
/// Prepare a Command with a hermetic environment and some essential variables preserved.
pub fn prepare_command(cmd: &mut Command, env_vars: &[(&str, String)]) {
pub fn prepare_command(cmd: &mut Command, env_vars: &EnvVars) {
cmd.env_clear();
// Preserve essential environment variables
for var in &["PATH", "LANG", "HOME", "SHELL", "DESTDIR", "DEPOT_ROOTFS"] {
@@ -36,6 +140,7 @@ pub fn build(
src_dir: &Path,
destdir: &Path,
cross: Option<&CrossConfig>,
export_compiler_flags: bool,
) -> Result<()> {
if let Some(cc) = cross {
println!(
@@ -52,18 +157,58 @@ pub fn build(
}
match spec.build.build_type {
BuildType::Autotools => autotools::build(spec, src_dir, destdir, cross),
BuildType::CMake => cmake::build(spec, src_dir, destdir, cross),
BuildType::Meson => meson::build(spec, src_dir, destdir, cross),
BuildType::Custom => custom::build(spec, src_dir, destdir, cross),
BuildType::Rust => rust::build(spec, src_dir, destdir, cross),
BuildType::Bin => bin::build(spec, src_dir, destdir, cross),
BuildType::Makefile => makefile::build(spec, src_dir, destdir, cross),
BuildType::Autotools => {
autotools::build(spec, src_dir, destdir, cross, export_compiler_flags)
}
BuildType::CMake => cmake::build(spec, src_dir, destdir, cross, export_compiler_flags),
BuildType::Meson => meson::build(spec, src_dir, destdir, cross, export_compiler_flags),
BuildType::Custom => custom::build(spec, src_dir, destdir, cross, export_compiler_flags),
BuildType::Rust => rust::build(spec, src_dir, destdir, cross, export_compiler_flags),
BuildType::Bin => bin::build(spec, src_dir, destdir, cross, export_compiler_flags),
BuildType::Makefile => {
makefile::build(spec, src_dir, destdir, cross, export_compiler_flags)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::package::{Build, BuildFlags, BuildType, Dependencies, PackageInfo, PackageSpec};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::path::PathBuf;
fn mk_spec(cflags: Vec<&str>, ldflags: Vec<&str>) -> PackageSpec {
let mut flags = BuildFlags::default();
flags.cflags = cflags.into_iter().map(String::from).collect();
flags.ldflags = ldflags.into_iter().map(String::from).collect();
PackageSpec {
package: PackageInfo {
name: "env-test".into(),
version: "1.0".into(),
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
manual_sources: Vec::new(),
source: vec![crate::package::Source {
url: "https://example.test/src.tar.gz".into(),
sha256: "abc".into(),
extract_dir: "src".into(),
patches: Vec::new(),
post_extract: Vec::new(),
}],
build: Build {
build_type: BuildType::Custom,
flags,
},
dependencies: Dependencies::default(),
spec_dir: PathBuf::from("."),
}
}
#[test]
fn test_prepare_command() {
@@ -77,19 +222,19 @@ mod tests {
std::env::set_var("DEPOT_ROOTFS", "/my/rootfs");
}
prepare_command(&mut cmd, &[("MYVAR", "myval".to_string())]);
prepare_command(&mut cmd, &vec![("MYVAR".to_string(), "myval".to_string())]);
let envs: std::collections::HashMap<_, _> = cmd.get_envs().collect();
assert!(envs.get(std::ffi::OsStr::new("PATH")).is_some());
assert!(envs.get(std::ffi::OsStr::new("HOME")).is_some());
assert!(envs.get(std::ffi::OsStr::new("FORBIDDEN")).is_none());
let envs: HashMap<_, _> = cmd.get_envs().collect();
assert!(envs.get(OsStr::new("PATH")).is_some());
assert!(envs.get(OsStr::new("HOME")).is_some());
assert!(envs.get(OsStr::new("FORBIDDEN")).is_none());
assert_eq!(
envs.get(std::ffi::OsStr::new("MYVAR")),
envs.get(OsStr::new("MYVAR")),
Some(&Some(std::ffi::OsString::from("myval").as_os_str()))
);
// DEPOT_ROOTFS should be preserved from the parent environment
assert_eq!(
envs.get(std::ffi::OsStr::new("DEPOT_ROOTFS")),
envs.get(OsStr::new("DEPOT_ROOTFS")),
Some(&Some(std::ffi::OsString::from("/my/rootfs").as_os_str()))
);
}
@@ -100,11 +245,88 @@ mod tests {
unsafe {
std::env::set_var("DESTDIR", "/tmp/dest");
}
prepare_command(&mut cmd, &[]);
let envs: std::collections::HashMap<_, _> = cmd.get_envs().collect();
prepare_command(&mut cmd, &Vec::new());
let envs: HashMap<_, _> = cmd.get_envs().collect();
assert_eq!(
envs.get(std::ffi::OsStr::new("DESTDIR")),
envs.get(OsStr::new("DESTDIR")),
Some(&Some(std::ffi::OsString::from("/tmp/dest").as_os_str()))
);
}
#[test]
fn test_standard_build_env_respects_export_compiler_flags_toggle() {
let spec = mk_spec(vec!["-O2"], vec!["-Wl,--as-needed"]);
let enabled = standard_build_env(&spec, None, true, true);
assert!(
enabled.iter().any(|(k, v)| k == "CFLAGS" && v == "-O2"),
"expected CFLAGS to be exported when enabled"
);
assert!(
enabled
.iter()
.any(|(k, v)| k == "LDFLAGS" && v == "-Wl,--as-needed"),
"expected LDFLAGS to be exported when enabled"
);
let disabled = standard_build_env(&spec, None, true, false);
assert!(
!disabled.iter().any(|(k, _)| k == "CFLAGS"),
"expected CFLAGS to be omitted when disabled"
);
assert!(
!disabled.iter().any(|(k, _)| k == "CXXFLAGS"),
"expected CXXFLAGS to be omitted when disabled"
);
assert!(
!disabled.iter().any(|(k, _)| k == "LDFLAGS"),
"expected LDFLAGS to be omitted when disabled"
);
let mut disabled_by_spec = spec.clone();
disabled_by_spec.build.flags.no_flags = true;
let disabled_env = standard_build_env(&disabled_by_spec, None, true, true);
assert!(
!disabled_env.iter().any(|(k, _)| k == "CFLAGS"),
"expected CFLAGS to be omitted when no_flags is set in spec"
);
assert!(
!disabled_env.iter().any(|(k, _)| k == "LDFLAGS"),
"expected LDFLAGS to be omitted when no_flags is set in spec"
);
}
#[test]
fn test_standard_build_env_exports_passthrough_env() {
let mut spec = mk_spec(Vec::new(), Vec::new());
spec.build.flags.passthrough_env = vec!["RUSTFLAGS".into()];
unsafe {
std::env::set_var("RUSTFLAGS", "-C target-cpu=native");
}
let env = standard_build_env(&spec, None, false, true);
assert!(
env.iter()
.any(|(k, v)| k == "RUSTFLAGS" && v == "-C target-cpu=native"),
"expected RUSTFLAGS to be copied from parent environment"
);
}
#[test]
fn test_standard_build_env_passthrough_does_not_override_default_vars() {
let mut spec = mk_spec(Vec::new(), Vec::new());
spec.build.flags.cc = "spec-cc".to_string();
spec.build.flags.passthrough_env = vec!["CC".into()];
unsafe {
std::env::set_var("CC", "host-cc");
}
let env = standard_build_env(&spec, None, true, true);
assert!(
env.iter().any(|(k, v)| k == "CC" && v == "spec-cc"),
"expected default CC to take precedence over passthrough CC"
);
}
}
+8 -14
View File
@@ -13,6 +13,7 @@ pub fn build(
src_dir: &Path,
destdir: &Path,
cross: Option<&CrossConfig>,
export_compiler_flags: bool,
) -> Result<()> {
let flags = &spec.build.flags;
@@ -45,16 +46,12 @@ pub fn build(
};
// Build environment
let mut env_vars: Vec<(&str, String)> = vec![];
let mut env_vars =
crate::builder::standard_build_env(spec, cross, false, export_compiler_flags);
// RUSTFLAGS
if !flags.rustflags.is_empty() {
env_vars.push(("RUSTFLAGS", flags.rustflags.join(" ")));
}
// CARCH support
if !flags.carch.is_empty() {
env_vars.push(("CARCH", flags.carch.clone()));
crate::builder::set_env_var(&mut env_vars, "RUSTFLAGS", flags.rustflags.join(" "));
}
// If cross-compiling, set linker via CARGO_TARGET_*_LINKER
@@ -66,19 +63,16 @@ pub fn build(
.to_uppercase()
.replace('-', "_");
let linker_var = format!("CARGO_TARGET_{}_LINKER", target_env);
env_vars.push((linker_var.leak(), cc_cfg.cc.clone()));
env_vars.push(("CC".to_string().leak(), cc_cfg.cc.clone()));
env_vars.push(("AR".to_string().leak(), cc_cfg.ar.clone()));
crate::builder::set_env_var(&mut env_vars, &linker_var, cc_cfg.cc.clone());
crate::builder::set_env_var(&mut env_vars, "CC", cc_cfg.cc.clone());
crate::builder::set_env_var(&mut env_vars, "AR", cc_cfg.ar.clone());
}
// Set default rustup toolchain if not already set
if std::env::var("RUSTUP_TOOLCHAIN").is_err() {
env_vars.push(("RUSTUP_TOOLCHAIN", "stable".to_string()));
crate::builder::set_env_var(&mut env_vars, "RUSTUP_TOOLCHAIN", "stable");
}
// Export DEPOT_ROOTFS so build scripts can detect the target rootfs
env_vars.push(("DEPOT_ROOTFS", flags.rootfs.clone()));
// Run cargo build
println!(
"Running cargo build ({})...",
+25 -7
View File
@@ -128,7 +128,10 @@ impl Config {
if host_build.exists() && host_build != build_path {
let content = fs::read_to_string(&host_build).with_context(|| {
format!("Failed to read system build config: {}", host_build.display())
format!(
"Failed to read system build config: {}",
host_build.display()
)
})?;
let (val, appends) = self.preprocess_toml(&content)?;
merge_toml_values(&mut self.build_overrides, &val);
@@ -139,7 +142,10 @@ impl Config {
if build_path.exists() {
let content = fs::read_to_string(&build_path).with_context(|| {
format!("Failed to read system build config: {}", build_path.display())
format!(
"Failed to read system build config: {}",
build_path.display()
)
})?;
let (val, appends) = self.preprocess_toml(&content)?;
merge_toml_values(&mut self.build_overrides, &val);
@@ -153,7 +159,10 @@ impl Config {
if host_package.exists() && host_package != package_path {
let content = fs::read_to_string(&host_package).with_context(|| {
format!("Failed to read system package config: {}", host_package.display())
format!(
"Failed to read system package config: {}",
host_package.display()
)
})?;
let (val, appends) = self.preprocess_toml(&content)?;
merge_toml_values(&mut self.package_overrides, &val);
@@ -164,7 +173,10 @@ impl Config {
if package_path.exists() {
let content = fs::read_to_string(&package_path).with_context(|| {
format!("Failed to read system package config: {}", package_path.display())
format!(
"Failed to read system package config: {}",
package_path.display()
)
})?;
let (val, appends) = self.preprocess_toml(&content)?;
merge_toml_values(&mut self.package_overrides, &val);
@@ -372,7 +384,9 @@ cflags += ["-g"]
// Save and restore the original value so other tests aren't affected.
let orig_home = std::env::var_os("HOME");
let fake_home = tempfile::tempdir().unwrap();
unsafe { std::env::set_var("HOME", &fake_home.path()); }
unsafe {
std::env::set_var("HOME", &fake_home.path());
}
let config = Config::for_rootfs(Path::new("/"));
@@ -392,9 +406,13 @@ cflags += ["-g"]
// restore original HOME
if let Some(v) = orig_home {
unsafe { std::env::set_var("HOME", v); }
unsafe {
std::env::set_var("HOME", v);
}
} else {
unsafe { std::env::remove_var("HOME"); }
unsafe {
std::env::remove_var("HOME");
}
}
}
+7 -12
View File
@@ -203,12 +203,16 @@ endian = 'little'
}
}
/// Find a cross-compilation tool in PATH
/// Find a cross-compilation tool in PATH and return its absolute path
fn find_tool(prefix: &str, suffixes: &[&str], required: bool) -> Result<String> {
for suffix in suffixes {
let tool_name = format!("{}-{}", prefix, suffix);
if tool_exists(&tool_name) {
return Ok(tool_name);
// Use `which` to resolve to an absolute path (so callers get a usable path)
if let Ok(output) = Command::new("which").arg(&tool_name).output() {
if output.status.success() {
let path = String::from_utf8_lossy(&output.stdout).trim().to_string();
return Ok(path);
}
}
}
@@ -222,12 +226,3 @@ fn find_tool(prefix: &str, suffixes: &[&str], required: bool) -> Result<String>
Err(anyhow::anyhow!("Tool not found"))
}
/// Check if a tool exists in PATH
fn tool_exists(name: &str) -> bool {
Command::new("which")
.arg(name)
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
+20 -20
View File
@@ -9,6 +9,10 @@ use rusqlite::{Connection, params};
use std::fs;
use std::path::Path;
fn format_licenses(licenses: &[String]) -> String {
licenses.join(", ")
}
/// Initialize database and register a package
pub fn register_package(db_path: &Path, spec: &PackageSpec, destdir: &Path) -> Result<()> {
// Create parent directory (auto-create db dir if missing)
@@ -40,7 +44,7 @@ pub fn register_package(db_path: &Path, spec: &PackageSpec, destdir: &Path) -> R
spec.package.revision,
spec.package.description,
spec.package.homepage,
spec.package.license,
format_licenses(&spec.package.license),
],
)?;
@@ -405,16 +409,11 @@ fn init_db(conn: &Connection) -> Result<()> {
}
/// Decide whether a conflicting path is safe to auto-remove ownership for.
/// This covers shared index files (e.g. `usr/share/info/dir`) and common
/// language-shared trees such as Perl site/vendor/lib directories.
/// This covers common language-shared trees such as Perl site/vendor/lib
/// directories.
fn is_auto_removable_path(path: &str) -> bool {
let p = path.trim_start_matches('/');
// Exact info index file (and compressed variants)
if p == "usr/share/info/dir" || p.starts_with("usr/share/info/dir.") {
return true;
}
// Perl shared trees (common multi-package locations)
if p.starts_with("usr/lib/perl")
|| p.starts_with("usr/share/perl")
@@ -475,7 +474,7 @@ mod tests {
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: "MIT".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Alternatives {
@@ -578,27 +577,27 @@ mod tests {
let tmp = tempfile::tempdir().unwrap();
let db_path = tmp.path().join("packages.db");
// Install package 'alpha' owning usr/share/info/dir
// Install package 'alpha' owning a known shared Perl path
let spec_a = mk_spec("alpha", "1.0");
let dest_a = tmp.path().join("dest_a");
std::fs::create_dir_all(dest_a.join("usr/share/info")).unwrap();
std::fs::write(dest_a.join("usr/share/info/dir"), "index").unwrap();
std::fs::create_dir_all(dest_a.join("usr/share/perl5")).unwrap();
std::fs::write(dest_a.join("usr/share/perl5/shared.pm"), "package A;").unwrap();
register_package(&db_path, &spec_a, &dest_a).unwrap();
// Now install package 'beta' that also provides usr/share/info/dir -> should auto-clear
// Now install package 'beta' that also provides the same shared path -> should auto-clear
let spec_b = mk_spec("beta", "1.0");
let dest_b = tmp.path().join("dest_b");
std::fs::create_dir_all(dest_b.join("usr/share/info")).unwrap();
std::fs::write(dest_b.join("usr/share/info/dir"), "index2").unwrap();
std::fs::create_dir_all(dest_b.join("usr/share/perl5")).unwrap();
std::fs::write(dest_b.join("usr/share/perl5/shared.pm"), "package B;").unwrap();
// This should succeed and transfer ownership of the 'dir' path to beta
// This should succeed and transfer ownership of the shared path to beta
register_package(&db_path, &spec_b, &dest_b).unwrap();
// Verify DB: alpha should no longer own the path, beta should
let files_a = get_package_files(&db_path, "alpha").unwrap();
assert!(!files_a.contains(&"usr/share/info/dir".to_string()));
assert!(!files_a.contains(&"usr/share/perl5/shared.pm".to_string()));
let files_b = get_package_files(&db_path, "beta").unwrap();
assert!(files_b.contains(&"usr/share/info/dir".to_string()));
assert!(files_b.contains(&"usr/share/perl5/shared.pm".to_string()));
}
#[test]
@@ -668,7 +667,7 @@ mod tests {
std::fs::write(dest1.join("usr/bin/shared_dir/old_file"), "old").unwrap();
register_package(&db_path, &spec_v1, &dest1).unwrap();
let _ = crate::staging::install_atomic(&dest1, &rootfs, &tx_base, &[]).unwrap();
let _ = crate::staging::install_atomic(&dest1, &rootfs, &tx_base, &[], &[]).unwrap();
assert!(rootfs.join("usr/bin/foo").exists());
assert!(rootfs.join("usr/bin/shared_dir/old_file").exists());
@@ -689,7 +688,8 @@ mod tests {
vec!["usr/bin/shared_dir/old_file".to_string()]
);
let tx = crate::staging::install_atomic(&dest2, &rootfs, &tx_base, &remove_paths).unwrap();
let tx =
crate::staging::install_atomic(&dest2, &rootfs, &tx_base, &remove_paths, &[]).unwrap();
register_package(&db_path, &spec_v2, &dest2).unwrap();
tx.commit().unwrap();
+80 -12
View File
@@ -6,6 +6,23 @@ use std::fs;
use std::path::{Path, PathBuf};
use zstd::stream::write::Encoder;
fn parse_license_text(metadata: &toml::Value) -> Option<String> {
if let Some(s) = metadata.get("license").and_then(|v| v.as_str()) {
return Some(s.to_string());
}
if let Some(arr) = metadata.get("license").and_then(|v| v.as_array()) {
let licenses: Vec<String> = arr
.iter()
.filter_map(|v| v.as_str())
.map(String::from)
.collect();
if !licenses.is_empty() {
return Some(licenses.join(", "));
}
}
None
}
pub struct RepoManager {
pub repo_dir: PathBuf,
}
@@ -129,10 +146,7 @@ impl RepoManager {
.get("homepage")
.and_then(|v| v.as_str())
.map(String::from);
license = metadata
.get("license")
.and_then(|v| v.as_str())
.map(String::from);
license = parse_license_text(&metadata);
if let Some(provides_arr) = metadata.get("provides").and_then(|v| v.as_array()) {
provides = provides_arr
@@ -209,8 +223,11 @@ impl RepoManager {
}
/// Synchronize git mirrors into /usr/src/depot/<reponame>
pub fn sync_mirrors(repo_dir: &std::path::Path, mirrors: &std::collections::HashMap<String, String>) -> Result<()> {
use git2::{Repository, FetchOptions, Cred, RemoteCallbacks, ResetType, build::RepoBuilder};
pub fn sync_mirrors(
repo_dir: &std::path::Path,
mirrors: &std::collections::HashMap<String, String>,
) -> Result<()> {
use git2::{Cred, FetchOptions, RemoteCallbacks, Repository, ResetType, build::RepoBuilder};
use std::os::unix::fs::PermissionsExt;
let base = repo_dir.to_path_buf();
@@ -235,7 +252,9 @@ pub fn sync_mirrors(repo_dir: &std::path::Path, mirrors: &std::collections::Hash
let mut builder = RepoBuilder::new();
builder.fetch_options(fo);
builder.clone(url, &target).with_context(|| format!("Failed to clone {}", url))?;
builder
.clone(url, &target)
.with_context(|| format!("Failed to clone {}", url))?;
} else {
println!("Updating mirror '{}' in {}", name, target.display());
// Open repository and fetch updates
@@ -251,8 +270,11 @@ pub fn sync_mirrors(repo_dir: &std::path::Path, mirrors: &std::collections::Hash
fo.remote_callbacks(cb);
// Fetch from origin
let mut remote = repo.find_remote("origin").or_else(|_| repo.remote_anonymous(url))?;
remote.fetch(&["refs/heads/*:refs/remotes/origin/*"], Some(&mut fo), None)
let mut remote = repo
.find_remote("origin")
.or_else(|_| repo.remote_anonymous(url))?;
remote
.fetch(&["refs/heads/*:refs/remotes/origin/*"], Some(&mut fo), None)
.with_context(|| format!("Failed to fetch updates for {}", url))?;
// Try to fast-forward HEAD to origin/HEAD by resetting to FETCH_HEAD if present
@@ -280,7 +302,10 @@ pub fn sync_mirrors(repo_dir: &std::path::Path, mirrors: &std::collections::Hash
}
/// Show status for each mirror repository: path, exists, branch/HEAD, latest commit, dirty
pub fn mirrors_status(repo_dir: &std::path::Path, mirrors: &std::collections::HashMap<String, String>) -> Result<()> {
pub fn mirrors_status(
repo_dir: &std::path::Path,
mirrors: &std::collections::HashMap<String, String>,
) -> Result<()> {
use git2::Repository;
let base = repo_dir.to_path_buf();
@@ -308,7 +333,9 @@ pub fn mirrors_status(repo_dir: &std::path::Path, mirrors: &std::collections::Ha
// Latest commit OID
let oid = repo.refname_to_id("HEAD").ok();
let short = oid.map(|o| format!("{}", o)).unwrap_or_else(|| "(unknown)".to_string());
let short = oid
.map(|o| format!("{}", o))
.unwrap_or_else(|| "(unknown)".to_string());
// Commit time (seconds since epoch) if available
let mut commit_time = String::new();
@@ -327,7 +354,11 @@ pub fn mirrors_status(repo_dir: &std::path::Path, mirrors: &std::collections::Ha
continue;
}
};
let dirty = statuses.iter().any(|s| s.status().intersects(git2::Status::WT_MODIFIED | git2::Status::WT_NEW | git2::Status::WT_DELETED));
let dirty = statuses.iter().any(|s| {
s.status().intersects(
git2::Status::WT_MODIFIED | git2::Status::WT_NEW | git2::Status::WT_DELETED,
)
});
println!("Path: {}", target.display());
println!("Branch/HEAD: {}", branch);
@@ -448,4 +479,41 @@ runtime = []
.unwrap();
assert_eq!(provides_count, 1);
}
#[test]
fn test_index_package_with_multiple_licenses() {
let tmp = tempfile::tempdir().unwrap();
let repo_dir = tmp.path();
let pkg_path = repo_dir.join("test-1.0-1-x86_64.depot.pkg.tar.zst");
let file = fs::File::create(&pkg_path).unwrap();
let encoder = zstd::stream::write::Encoder::new(file, 3).unwrap();
let mut tar = tar::Builder::new(encoder);
let metadata = r#"
name = "test"
version = "1.0"
revision = 1
license = ["MIT", "Apache-2.0"]
"#;
let mut header = tar::Header::new_gnu();
header.set_path(".metadata.toml").unwrap();
header.set_size(metadata.len() as u64);
header.set_mode(0o644);
header.set_cksum();
tar.append(&header, metadata.as_bytes()).unwrap();
let encoder = tar.into_inner().unwrap();
encoder.finish().unwrap();
let mut conn = Connection::open_in_memory().unwrap();
let manager = RepoManager::new(repo_dir.to_path_buf());
manager.init_repo_schema(&mut conn).unwrap();
manager.index_package(&mut conn, &pkg_path).unwrap();
let lic: Option<String> = conn
.query_row("SELECT license FROM packages", [], |r| r.get(0))
.unwrap();
assert_eq!(lic, Some("MIT, Apache-2.0".to_string()));
}
}
+79
View File
@@ -173,10 +173,31 @@ pub fn check_runtime_deps(spec: &PackageSpec, db_path: &Path) -> Result<Vec<Stri
Ok(missing)
}
/// Check if all test dependencies are satisfied
pub fn check_test_deps(spec: &PackageSpec, db_path: &Path) -> Result<Vec<String>> {
let mut missing = Vec::new();
if !db_path.exists() {
return Ok(spec.dependencies.test.clone());
}
let installed = db::get_installed_packages(db_path)?;
let provides = db::get_all_provides(db_path)?;
for dep in &spec.dependencies.test {
if !is_dep_satisfied(dep, &installed, &provides, db_path)? {
missing.push(dep.clone());
}
}
Ok(missing)
}
/// Print dependency status
pub fn print_dep_status(spec: &PackageSpec, db_path: &Path) -> Result<()> {
let missing_build = check_build_deps(spec, db_path)?;
let missing_runtime = check_runtime_deps(spec, db_path)?;
let missing_test = check_test_deps(spec, db_path)?;
if !spec.dependencies.build.is_empty() {
println!("Build dependencies: {}", spec.dependencies.build.join(", "));
@@ -195,6 +216,13 @@ pub fn print_dep_status(spec: &PackageSpec, db_path: &Path) -> Result<()> {
}
}
if !spec.dependencies.test.is_empty() {
println!("Test dependencies: {}", spec.dependencies.test.join(", "));
if !missing_test.is_empty() {
println!(" Missing: {}", missing_test.join(", "));
}
}
Ok(())
}
@@ -212,6 +240,20 @@ pub fn require_build_deps(spec: &PackageSpec, db_path: &Path) -> Result<()> {
Ok(())
}
/// Verify all test dependencies are installed, error if not
pub fn require_test_deps(spec: &PackageSpec, db_path: &Path) -> Result<()> {
let missing = check_test_deps(spec, db_path)?;
if !missing.is_empty() {
anyhow::bail!(
"Missing test dependencies: {}\nInstall them first with: nyapm install <package>",
missing.join(", ")
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -260,4 +302,41 @@ mod tests {
assert!(compare_versions("a", "b", VersionOp::Lt));
assert!(compare_versions("foo", "foo", VersionOp::Exact));
}
#[test]
fn test_check_test_deps_returns_test_deps_when_db_missing() {
let spec = PackageSpec {
package: crate::package::PackageInfo {
name: "foo".into(),
version: "1.0".into(),
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Default::default(),
manual_sources: Vec::new(),
source: vec![crate::package::Source {
url: "https://example.test/foo.tar.gz".into(),
sha256: "skip".into(),
extract_dir: "foo".into(),
patches: Vec::new(),
post_extract: Vec::new(),
}],
build: crate::package::Build {
build_type: crate::package::BuildType::Custom,
flags: crate::package::BuildFlags::default(),
},
dependencies: crate::package::Dependencies {
build: Vec::new(),
runtime: Vec::new(),
test: vec!["bats".into(), "python".into()],
},
spec_dir: std::path::PathBuf::from("."),
};
let missing = check_test_deps(&spec, Path::new("/definitely/not/a/real/db")).unwrap();
assert_eq!(missing, vec!["bats".to_string(), "python".to_string()]);
}
}
+13 -3
View File
@@ -45,7 +45,11 @@ impl PackageIndex {
.by_name
.insert(spec.package.name.clone(), path.clone());
for provided in &spec.alternatives.provides {
index.by_provides.entry(provided.clone()).or_default().push(path.clone());
index
.by_provides
.entry(provided.clone())
.or_default()
.push(path.clone());
}
}
}
@@ -63,9 +67,15 @@ impl PackageIndex {
let path = entry.path().to_path_buf();
if path.extension().map(|e| e == "toml").unwrap_or(false) {
if let Ok(spec) = PackageSpec::from_file(&path) {
index.by_name.insert(spec.package.name.clone(), path.clone());
index
.by_name
.insert(spec.package.name.clone(), path.clone());
for provided in &spec.alternatives.provides {
index.by_provides.entry(provided.clone()).or_default().push(path.clone());
index
.by_provides
.entry(provided.clone())
.or_default()
.push(path.clone());
}
}
}
+188 -41
View File
@@ -17,6 +17,86 @@ use clap::{Parser, Subcommand};
use std::fs;
use std::path::{Path, PathBuf};
fn parse_licenses_from_toml(metadata: &toml::Value) -> Vec<String> {
if let Some(s) = metadata.get("license").and_then(|v| v.as_str()) {
return vec![s.to_string()];
}
if let Some(arr) = metadata.get("license").and_then(|v| v.as_array()) {
return arr
.iter()
.filter_map(|v| v.as_str())
.map(String::from)
.collect();
}
Vec::new()
}
fn clean_build_workspace(config: &config::Config) -> Result<()> {
if config.build_dir.exists() {
fs::remove_dir_all(&config.build_dir).with_context(|| {
format!("Failed to clean build dir: {}", config.build_dir.display())
})?;
println!("Cleaned build workspace: {}", config.build_dir.display());
}
Ok(())
}
fn warn_if_running_as_root_for_build(command: &str, rootfs: &Path) {
if crate::fakeroot::is_root() {
eprintln!(
"\x1b[33mWARNING: Running '{}' as root is discouraged.\x1b[0m",
command
);
eprintln!(
"\x1b[33mA misconfigured build environment or malicious/buggy build file can overwrite or delete critical system files.\x1b[0m"
);
eprintln!(
"\x1b[33mRecommendation: use a non-root build user and only install as root.\x1b[0m"
);
eprintln!("\x1b[33mCurrent rootfs target: {}\x1b[0m", rootfs.display());
}
}
fn install_staged_to_rootfs(
pkg_spec: &package::PackageSpec,
destdir: &Path,
rootfs: &Path,
config: &config::Config,
) -> Result<()> {
std::fs::create_dir_all(&config.db_dir).with_context(|| {
format!(
"Failed to create database directory: {}",
config.db_dir.display()
)
})?;
let db_path = config.db_dir.join("packages.db");
let new_files = staging::generate_manifest_with_dirs(destdir)?;
let remove_paths =
db::calculate_upgrade_paths(&db_path, &pkg_spec.package.name, &new_files.files)?;
let tx_base = config.build_dir.join("tx");
let tx = staging::install_atomic(
destdir,
rootfs,
&tx_base,
&remove_paths,
&pkg_spec.build.flags.keep,
)?;
for out in pkg_spec.outputs() {
let mut spec_for_out = pkg_spec.clone();
spec_for_out.package = out;
if let Err(e) = db::register_package(&db_path, &spec_for_out, destdir) {
let _ = tx.rollback();
return Err(e);
}
}
tx.commit()?;
Ok(())
}
#[derive(Parser)]
#[command(name = "Depot")]
#[command(about = "Depot - Source-based package manager for Linux", long_about = None)]
@@ -30,10 +110,25 @@ struct Cli {
#[arg(long, global = true)]
no_deps: bool,
/// Do not export CFLAGS/CXXFLAGS/LDFLAGS to build commands
#[arg(
long,
global = true,
action = clap::ArgAction::Set,
num_args = 0..=1,
default_value_t = false,
default_missing_value = "true"
)]
no_flags: bool,
/// Cross-compilation prefix (e.g., x86_64-linux-musl, aarch64-linux-gnu)
#[arg(long, global = true)]
cross_prefix: Option<String>,
/// Clean build workspace after successful install/build
#[arg(long, global = true)]
clean: bool,
#[command(subcommand)]
command: Commands,
}
@@ -64,6 +159,10 @@ enum Commands {
/// Explicitly specify path to package spec (.toml file)
#[arg(short, long = "spec", visible_alias = "package", alias = "p")]
spec: Option<PathBuf>,
/// Install package to rootfs after creating package archive(s)
#[arg(long)]
install: bool,
},
/// Show information about a package
Info {
@@ -109,6 +208,7 @@ fn main() -> Result<()> {
spec_or_archive,
spec,
} => {
warn_if_running_as_root_for_build("install", &cli.rootfs);
let mut spec_path = spec.unwrap_or(spec_or_archive);
// Load configuration early so we can use the configured repo clone dir
@@ -122,7 +222,8 @@ fn main() -> Result<()> {
if !spec_path.exists() {
let name = spec_path.to_string_lossy().to_string();
println!("Looking up package '{}' in local indexes...", name);
let pkg_index = index::PackageIndex::build_with_repo_dir(Some(config.repo_clone_dir.clone()));
let pkg_index =
index::PackageIndex::build_with_repo_dir(Some(config.repo_clone_dir.clone()));
if let Some(found) = pkg_index.find(&name) {
spec_path = found;
}
@@ -159,10 +260,6 @@ fn main() -> Result<()> {
);
}
// We need to parse the metadata.toml but we don't have a direct "from_metadata"
// Let's implement a minimal reconstruction or use the metadata to fill a spec.
// Actually, PackageSpec needs a lot of fields.
// Let's extract the WHOLE archive to a temporary staging dir and use it.
let file = fs::File::open(&spec_path)?;
let zstd_decoder = zstd::stream::read::Decoder::new(file)?;
let mut archive = tar::Archive::new(zstd_decoder);
@@ -197,11 +294,7 @@ fn main() -> Result<()> {
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
license: metadata
.get("license")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
license: parse_licenses_from_toml(&metadata),
},
packages: Vec::new(),
alternatives: package::Alternatives::default(),
@@ -225,6 +318,18 @@ fn main() -> Result<()> {
} else {
Vec::new()
},
test: if let Some(deps) = metadata
.get("dependencies")
.and_then(|v| v.get("test"))
.and_then(|v| v.as_array())
{
deps.iter()
.filter_map(|v| v.as_str())
.map(String::from)
.collect()
} else {
Vec::new()
},
},
spec_dir: PathBuf::from("."),
};
@@ -266,6 +371,8 @@ fn main() -> Result<()> {
// Check dependencies and prompt for auto-install if needed
if !cli.no_deps {
let needs_test_deps =
matches!(pkg_spec.build.build_type, package::BuildType::Autotools);
deps::print_dep_status(&pkg_spec, &db_path)?;
// Collect all missing dependencies (build + runtime)
@@ -277,6 +384,14 @@ fn main() -> Result<()> {
missing.push(dep);
}
}
if needs_test_deps {
let missing_test = deps::check_test_deps(&pkg_spec, &db_path)?;
for dep in missing_test {
if !missing.contains(&dep) {
missing.push(dep);
}
}
}
if !missing.is_empty() {
// Check for dependency cycles via DEPOT_DEPCHAIN env var
@@ -301,7 +416,9 @@ fn main() -> Result<()> {
if input == "y" || input == "yes" || input.is_empty() {
// Build package index for fast lookups
let pkg_index = index::PackageIndex::build_with_repo_dir(Some(config.repo_clone_dir.clone()));
let pkg_index = index::PackageIndex::build_with_repo_dir(Some(
config.repo_clone_dir.clone(),
));
// Build new dep chain
let new_chain = if dep_chain.is_empty() {
@@ -324,9 +441,15 @@ fn main() -> Result<()> {
if cli.no_deps {
cmd.arg("--no-deps");
}
if cli.no_flags {
cmd.arg("--no-flags");
}
if let Some(ref p) = cli.cross_prefix {
cmd.arg("--cross-prefix").arg(p);
}
if cli.clean {
cmd.arg("--clean");
}
cmd.arg("install").arg(&dep_spec_path);
cmd.env("DEPOT_DEPCHAIN", &new_chain);
@@ -348,6 +471,9 @@ fn main() -> Result<()> {
// Enforce build dependencies (runtime deps are warnings only if not installed/prompt declined)
deps::require_build_deps(&pkg_spec, &db_path)?;
if needs_test_deps {
deps::require_test_deps(&pkg_spec, &db_path)?;
}
}
// Ensure database directory exists
@@ -377,7 +503,13 @@ fn main() -> Result<()> {
.as_ref()
.map(|p| cross::CrossConfig::from_prefix(p))
.transpose()?;
builder::build(&pkg_spec, &src_dir, &destdir, cross_config.as_ref())?;
builder::build(
&pkg_spec,
&src_dir,
&destdir,
cross_config.as_ref(),
!cli.no_flags,
)?;
// 3.1 Copy license files into staged tree
staging::add_licenses(&src_dir, &destdir, &pkg_spec.package.name)?;
@@ -388,25 +520,8 @@ fn main() -> Result<()> {
// 4. Stage (clean .la files, etc.)
staging::process(&destdir, &pkg_spec)?;
// 5. Install/update to rootfs (atomic)
let new_files = staging::generate_manifest_with_dirs(&destdir)?;
let remove_paths =
db::calculate_upgrade_paths(&db_path, &pkg_spec.package.name, &new_files.files)?;
let tx_base = config.build_dir.join("tx");
let tx = staging::install_atomic(&destdir, &cli.rootfs, &tx_base, &remove_paths)?;
// 6. Register in database (rollback install on DB error)
for out in pkg_spec.outputs() {
let mut spec_for_out = pkg_spec.clone();
spec_for_out.package = out;
if let Err(e) = db::register_package(&db_path, &spec_for_out, &destdir) {
let _ = tx.rollback();
return Err(e);
}
}
tx.commit()?;
// 5-6. Install/update to rootfs and register in DB
install_staged_to_rootfs(&pkg_spec, &destdir, &cli.rootfs, &config)?;
// 7. Check runtime dependencies (warn only)
if !cli.no_deps {
@@ -427,6 +542,10 @@ fn main() -> Result<()> {
"Successfully installed {} v{}",
pkg_spec.package.name, pkg_spec.package.version
);
if cli.clean {
clean_build_workspace(&config)?;
}
}
Commands::Remove { package } => {
println!("Removing package: {}", package);
@@ -435,12 +554,12 @@ fn main() -> Result<()> {
db::remove_package(&db_path, &package, &cli.rootfs)?;
println!("Successfully removed {}", package);
}
Commands::Build { spec_pos, spec } => {
if crate::fakeroot::is_root() {
anyhow::bail!(
"The 'build' command must be run as a non-root user to ensure a clean build environment."
);
}
Commands::Build {
spec_pos,
spec,
install,
} => {
warn_if_running_as_root_for_build("build", &cli.rootfs);
let spec_path = spec.or(spec_pos).context("No spec file provided")?;
println!("Building package from: {}", spec_path.display());
let mut pkg_spec = package::PackageSpec::from_file(&spec_path)?;
@@ -461,8 +580,13 @@ fn main() -> Result<()> {
// Check build dependencies
if !cli.no_deps {
let needs_test_deps =
matches!(pkg_spec.build.build_type, package::BuildType::Autotools);
deps::print_dep_status(&pkg_spec, &db_path)?;
deps::require_build_deps(&pkg_spec, &db_path)?;
if needs_test_deps {
deps::require_test_deps(&pkg_spec, &db_path)?;
}
}
let src_dir = source::prepare(&pkg_spec, &config.cache_dir, &config.build_dir)?;
@@ -477,7 +601,13 @@ fn main() -> Result<()> {
.as_ref()
.map(|p| cross::CrossConfig::from_prefix(p))
.transpose()?;
builder::build(&pkg_spec, &src_dir, &destdir, cross_config.as_ref())?;
builder::build(
&pkg_spec,
&src_dir,
&destdir,
cross_config.as_ref(),
!cli.no_flags,
)?;
staging::add_licenses(&src_dir, &destdir, &pkg_spec.package.name)?;
@@ -493,7 +623,8 @@ fn main() -> Result<()> {
for out in pkg_spec.outputs() {
let mut spec_for_out = pkg_spec.clone();
spec_for_out.package = out;
let packager = package::Packager::new(spec_for_out.clone(), destdir.clone(), config.clone());
let packager =
package::Packager::new(spec_for_out.clone(), destdir.clone(), config.clone());
let pkg_file = packager.create_package(Path::new("."), arch)?;
created_files.push(pkg_file);
}
@@ -501,6 +632,18 @@ fn main() -> Result<()> {
for f in &created_files {
println!("Build complete. Package created: {}", f.display());
}
if install {
install_staged_to_rootfs(&pkg_spec, &destdir, &cli.rootfs, &config)?;
println!(
"Successfully installed {} v{}",
pkg_spec.package.name, pkg_spec.package.version
);
}
if cli.clean {
clean_build_workspace(&config)?;
}
}
Commands::Info { package } => {
// Try as file first, then as installed package name
@@ -540,7 +683,10 @@ fn main() -> Result<()> {
println!("No mirrors configured in /etc/depot.d/mirrors.toml");
} else {
db::repo::sync_mirrors(&config.repo_clone_dir, &config.mirrors)?;
println!("Mirrors synchronized into {}", config.repo_clone_dir.display());
println!(
"Mirrors synchronized into {}",
config.repo_clone_dir.display()
);
}
}
RepoCommands::Status => {
@@ -571,7 +717,8 @@ fn main() -> Result<()> {
// Produce a minimal TOML for interactive-created specs (omit defaults)
let toml_string = package::spec_to_minimal_toml(&spec)?;
let output_path = output.unwrap_or_else(|| PathBuf::from(format!("{}.toml", spec.package.name)));
let output_path =
output.unwrap_or_else(|| PathBuf::from(format!("{}.toml", spec.package.name)));
if output_path.exists() {
println!(
+784 -129
View File
File diff suppressed because it is too large Load Diff
+49 -2
View File
@@ -8,6 +8,19 @@ use std::path::{Path, PathBuf};
use tar::Builder;
use zstd::stream::write::Encoder;
fn license_value(licenses: &[String]) -> toml::Value {
if licenses.len() == 1 {
toml::Value::String(licenses[0].clone())
} else {
toml::Value::Array(
licenses
.iter()
.map(|license| toml::Value::String(license.clone()))
.collect(),
)
}
}
pub struct Packager {
pub spec: PackageSpec,
pub destdir: PathBuf,
@@ -119,7 +132,7 @@ impl Packager {
);
map.insert(
"license".to_string(),
toml::Value::String(self.spec.package.license.clone()),
license_value(&self.spec.package.license),
);
// Add provides
@@ -159,6 +172,17 @@ impl Packager {
.collect(),
),
);
build_deps.insert(
"test".to_string(),
toml::Value::Array(
self.spec
.dependencies
.test
.iter()
.map(|s| toml::Value::String(s.clone()))
.collect(),
),
);
map.insert("dependencies".to_string(), toml::Value::Table(build_deps));
let toml_str = toml::to_string(&toml::Value::Table(map))
@@ -229,7 +253,7 @@ mod tests {
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: "MIT".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Alternatives::default(),
@@ -301,5 +325,28 @@ mod tests {
let deps = val.get("dependencies").unwrap();
assert!(deps.get("build").unwrap().as_array().unwrap().is_empty());
assert!(deps.get("runtime").unwrap().as_array().unwrap().is_empty());
assert!(deps.get("test").unwrap().as_array().unwrap().is_empty());
}
#[test]
fn test_generate_metadata_toml_with_multiple_licenses() {
let tmp = tempfile::tempdir().unwrap();
let dest = tmp.path();
let mut packager = mk_packager(dest.to_path_buf());
packager.spec.package.license = vec!["MIT".into(), "Apache-2.0".into()];
packager.generate_metadata_toml().unwrap();
let meta_path = dest.join(".metadata.toml");
let content = fs::read_to_string(meta_path).unwrap();
let val: toml::Value = toml::from_str(&content).unwrap();
let arr = val
.get("license")
.and_then(|v| v.as_array())
.expect("license should be an array");
assert_eq!(arr.len(), 2);
assert_eq!(arr[0].as_str(), Some("MIT"));
assert_eq!(arr[1].as_str(), Some("Apache-2.0"));
}
}
+765 -10
View File
@@ -3,6 +3,8 @@
use anyhow::{Context, Result};
use serde::Deserialize;
use serde::Deserializer;
use serde::Serialize;
use serde::Serializer;
use std::fmt;
use std::fs;
use std::path::Path;
@@ -52,10 +54,43 @@ impl PackageSpec {
if spec.source.is_empty() && spec.manual_sources.is_empty() {
anyhow::bail!("Package must have at least one source or manual_sources entry");
}
spec.validate_manual_sources()?;
Ok(spec)
}
fn validate_manual_sources(&self) -> Result<()> {
for (idx, manual) in self.manual_sources.iter().enumerate() {
let has_file = manual
.file
.as_ref()
.map(|s| !s.trim().is_empty())
.unwrap_or(false);
let has_url = manual
.url
.as_ref()
.map(|s| !s.trim().is_empty())
.unwrap_or(false);
match (has_file, has_url) {
(true, false) | (false, true) => {}
(false, false) => {
anyhow::bail!(
"manual_sources[{}] must specify exactly one of 'file' or 'url'",
idx
);
}
(true, true) => {
anyhow::bail!(
"manual_sources[{}] cannot specify both 'file' and 'url'",
idx
);
}
}
}
Ok(())
}
/// Expand variables like $name and $version in a string
pub fn expand_vars(&self, input: &str) -> String {
let specdir = self.spec_dir.to_string_lossy();
@@ -127,6 +162,17 @@ impl PackageSpec {
self.build.flags.ldflags = vec![s.to_string()];
}
}
"keep" => {
if let Some(arr) = v.as_array() {
self.build.flags.keep = arr
.iter()
.filter_map(|x| x.as_str())
.map(String::from)
.collect();
} else if let Some(s) = v.as_str() {
self.build.flags.keep = vec![s.to_string()];
}
}
"cc" => {
if let Some(s) = v.as_str() {
self.build.flags.cc = s.to_string();
@@ -162,6 +208,65 @@ impl PackageSpec {
self.build.flags.carch = s.to_string();
}
}
"make_vars" | "make-vars" | "make_build_vars" | "make-build-vars" => {
if let Some(arr) = v.as_array() {
self.build.flags.make_vars = arr
.iter()
.filter_map(|x| x.as_str())
.map(String::from)
.collect();
} else if let Some(s) = v.as_str() {
self.build.flags.make_vars =
s.split_whitespace().map(String::from).collect();
}
}
"make_test_vars" | "make-test-vars" => {
if let Some(arr) = v.as_array() {
self.build.flags.make_test_vars = arr
.iter()
.filter_map(|x| x.as_str())
.map(String::from)
.collect();
} else if let Some(s) = v.as_str() {
self.build.flags.make_test_vars =
s.split_whitespace().map(String::from).collect();
}
}
"make_install_vars" | "make-install-vars" => {
if let Some(arr) = v.as_array() {
self.build.flags.make_install_vars = arr
.iter()
.filter_map(|x| x.as_str())
.map(String::from)
.collect();
} else if let Some(s) = v.as_str() {
self.build.flags.make_install_vars =
s.split_whitespace().map(String::from).collect();
}
}
"passthrough_env" | "passthrough-env" | "pass_env" | "pass-env" | "export_env"
| "export-env" => {
if let Some(arr) = v.as_array() {
self.build.flags.passthrough_env = arr
.iter()
.filter_map(|x| x.as_str())
.map(String::from)
.collect();
} else if let Some(s) = v.as_str() {
self.build.flags.passthrough_env =
s.split_whitespace().map(String::from).collect();
}
}
"no_flags" | "no-flags" => {
if let Some(b) = v.as_bool() {
self.build.flags.no_flags = b;
}
}
"skip_tests" | "skip-tests" => {
if let Some(b) = v.as_bool() {
self.build.flags.skip_tests = b;
}
}
// Add more fields as needed
_ => {}
}
@@ -194,6 +299,18 @@ impl PackageSpec {
}
}
}
"keep" => {
for v in values {
if let Some(arr) = v.as_array() {
self.build
.flags
.keep
.extend(arr.iter().filter_map(|x| x.as_str()).map(String::from));
} else if let Some(s) = v.as_str() {
self.build.flags.keep.push(s.to_string());
}
}
}
"configure" => {
for v in values {
if let Some(arr) = v.as_array() {
@@ -260,6 +377,77 @@ impl PackageSpec {
self.build.flags.carch = s.to_string();
}
}
"make_vars" | "make-vars" | "make_build_vars" | "make-build-vars" => {
for v in values {
if let Some(arr) = v.as_array() {
self.build
.flags
.make_vars
.extend(arr.iter().filter_map(|x| x.as_str()).map(String::from));
} else if let Some(s) = v.as_str() {
self.build
.flags
.make_vars
.extend(s.split_whitespace().map(String::from));
}
}
}
"make_test_vars" | "make-test-vars" => {
for v in values {
if let Some(arr) = v.as_array() {
self.build
.flags
.make_test_vars
.extend(arr.iter().filter_map(|x| x.as_str()).map(String::from));
} else if let Some(s) = v.as_str() {
self.build
.flags
.make_test_vars
.extend(s.split_whitespace().map(String::from));
}
}
}
"make_install_vars" | "make-install-vars" => {
for v in values {
if let Some(arr) = v.as_array() {
self.build
.flags
.make_install_vars
.extend(arr.iter().filter_map(|x| x.as_str()).map(String::from));
} else if let Some(s) = v.as_str() {
self.build
.flags
.make_install_vars
.extend(s.split_whitespace().map(String::from));
}
}
}
"passthrough_env" | "passthrough-env" | "pass_env" | "pass-env" | "export_env"
| "export-env" => {
for v in values {
if let Some(arr) = v.as_array() {
self.build
.flags
.passthrough_env
.extend(arr.iter().filter_map(|x| x.as_str()).map(String::from));
} else if let Some(s) = v.as_str() {
self.build
.flags
.passthrough_env
.extend(s.split_whitespace().map(String::from));
}
}
}
"no_flags" | "no-flags" => {
if let Some(b) = values.last().and_then(|v| v.as_bool()) {
self.build.flags.no_flags = b;
}
}
"skip_tests" | "skip-tests" => {
if let Some(b) = values.last().and_then(|v| v.as_bool()) {
self.build.flags.skip_tests = b;
}
}
_ => {}
}
}
@@ -344,6 +532,39 @@ type = "custom"
assert_eq!(spec.sources()[1].extract_dir, "bar");
}
#[test]
fn parse_multiple_licenses() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = ["MIT", "Apache-2.0"]
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "custom"
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert_eq!(
spec.package.license,
vec!["MIT".to_string(), "Apache-2.0".to_string()]
);
}
#[test]
fn parse_rejects_empty_sources() {
let tmp = tempfile::tempdir().unwrap();
@@ -371,6 +592,101 @@ type = "custom"
assert!(PackageSpec::from_file(&path).is_err());
}
#[test]
fn parse_manual_source_with_url() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[[manual_sources]]
url = "https://example.com/manual.patch"
sha256 = "skip"
dest = "patches/manual.patch"
[build]
type = "custom"
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert_eq!(spec.manual_sources.len(), 1);
assert_eq!(
spec.manual_sources[0].url.as_deref(),
Some("https://example.com/manual.patch")
);
assert_eq!(spec.manual_sources[0].file, None);
}
#[test]
fn parse_manual_source_rejects_missing_file_and_url() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[[manual_sources]]
sha256 = "skip"
[build]
type = "custom"
"#,
)
.unwrap();
let err = PackageSpec::from_file(&path).expect_err("spec should be rejected");
assert!(err.to_string().contains("exactly one of 'file' or 'url'"));
}
#[test]
fn parse_manual_source_rejects_file_and_url_together() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[[manual_sources]]
file = "manual.patch"
url = "https://example.com/manual.patch"
[build]
type = "custom"
"#,
)
.unwrap();
let err = PackageSpec::from_file(&path).expect_err("spec should be rejected");
assert!(
err.to_string()
.contains("cannot specify both 'file' and 'url'")
);
}
#[test]
fn test_apply_config() {
let mut spec = mk_spec("foo", "1.0");
@@ -382,6 +698,11 @@ type = "custom"
[flags]
cc = "my-cc"
cflags = ["-O2"]
passthrough_env = ["RUSTFLAGS"]
make_vars = ["V=1"]
no_flags = true
skip_tests = true
keep = ["etc/locale.gen"]
"#,
)
.unwrap();
@@ -396,6 +717,24 @@ cflags = ["-O2"]
toml::Value::String("opt-level=3".to_string()),
])],
);
config.appends.insert(
"build.flags.keep".to_string(),
vec![toml::Value::Array(vec![toml::Value::String(
"etc/locale.gen".to_string(),
)])],
);
config.appends.insert(
"build.flags.passthrough_env".to_string(),
vec![toml::Value::String("CARGO_HOME".to_string())],
);
config.appends.insert(
"build.flags.make_test_vars".to_string(),
vec![toml::Value::String("TESTS=smoke".to_string())],
);
config.appends.insert(
"build.flags.make_install_vars".to_string(),
vec![toml::Value::String("DESTDIR=/tmp/pkg".to_string())],
);
spec.apply_config(&config);
@@ -409,6 +748,326 @@ cflags = ["-O2"]
.rustflags
.contains(&"opt-level=3".to_string())
);
assert!(spec.build.flags.no_flags);
assert!(
spec.build
.flags
.keep
.contains(&"etc/locale.gen".to_string())
);
assert!(
spec.build
.flags
.passthrough_env
.contains(&"RUSTFLAGS".to_string())
);
assert!(
spec.build
.flags
.passthrough_env
.contains(&"CARGO_HOME".to_string())
);
assert!(spec.build.flags.make_vars.contains(&"V=1".to_string()));
assert!(spec.build.flags.skip_tests);
assert!(
spec.build
.flags
.make_test_vars
.contains(&"TESTS=smoke".to_string())
);
assert!(
spec.build
.flags
.make_install_vars
.contains(&"DESTDIR=/tmp/pkg".to_string())
);
}
#[test]
fn parse_no_flags_from_spec() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "custom"
[build.flags]
no_flags = true
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert!(spec.build.flags.no_flags);
}
#[test]
fn parse_no_flags_alias_from_spec() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "custom"
[build.flags]
"no-flags" = true
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert!(spec.build.flags.no_flags);
}
#[test]
fn parse_skip_tests_from_spec() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "autotools"
[build.flags]
skip_tests = true
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert!(spec.build.flags.skip_tests);
}
#[test]
fn parse_skip_tests_alias_from_spec() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "autotools"
[build.flags]
"skip-tests" = true
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert!(spec.build.flags.skip_tests);
}
#[test]
fn parse_keep_from_spec() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "custom"
[build.flags]
keep = ["etc/locale.gen", "etc/resolv.conf"]
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert_eq!(
spec.build.flags.keep,
vec!["etc/locale.gen".to_string(), "etc/resolv.conf".to_string()]
);
}
#[test]
fn parse_passthrough_env_from_spec() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "custom"
[build.flags]
passthrough_env = ["RUSTFLAGS", "CARGO_HOME"]
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert_eq!(
spec.build.flags.passthrough_env,
vec!["RUSTFLAGS".to_string(), "CARGO_HOME".to_string()]
);
}
#[test]
fn parse_test_dependencies_from_spec() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "autotools"
[dependencies]
build = ["make"]
test = ["python", "bats"]
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert_eq!(
spec.dependencies.test,
vec!["python".to_string(), "bats".to_string()]
);
}
#[test]
fn parse_make_var_overrides_from_spec() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("pkg.toml");
std::fs::write(
&path,
r#"
[package]
name = "foo"
version = "1.0"
description = "d"
homepage = "h"
license = "MIT"
[source]
url = "https://example.com/foo.tar.gz"
sha256 = "skip"
extract_dir = "foo"
[build]
type = "autotools"
[build.flags]
make_vars = ["V=1", "CC=clang"]
make_test_vars = ["TESTS=unit"]
make_install_vars = ["STRIPPROG=true"]
"#,
)
.unwrap();
let spec = PackageSpec::from_file(&path).unwrap();
assert_eq!(
spec.build.flags.make_vars,
vec!["V=1".to_string(), "CC=clang".to_string()]
);
assert_eq!(
spec.build.flags.make_test_vars,
vec!["TESTS=unit".to_string()]
);
assert_eq!(
spec.build.flags.make_install_vars,
vec!["STRIPPROG=true".to_string()]
);
}
#[test]
@@ -444,9 +1103,12 @@ cbuild = "x86_64-pc-linux-gnu"
// Override via config
let mut config = crate::config::Config::for_rootfs(Path::new("/tmp/nonexistent"));
config.build_overrides = toml::from_str(r#"[flags]
config.build_overrides = toml::from_str(
r#"[flags]
carch = "armv7"
"#).unwrap();
"#,
)
.unwrap();
spec.apply_config(&config);
assert_eq!(spec.build.flags.carch, "armv7");
}
@@ -509,7 +1171,7 @@ type = "custom"
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: "MIT".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Alternatives::default(),
@@ -540,7 +1202,7 @@ impl fmt::Display for PackageSpec {
)?;
writeln!(f, "Description: {}", self.package.description)?;
writeln!(f, "Homepage: {}", self.package.homepage)?;
writeln!(f, "License: {}", self.package.license)?;
writeln!(f, "License: {}", self.package.license.join(", "))?;
writeln!(f, "Sources: {}", self.source.len())?;
writeln!(f, "Build Type: {:?}", self.build.build_type)?;
if !self.alternatives.provides.is_empty() {
@@ -560,13 +1222,45 @@ pub struct PackageInfo {
pub revision: u32,
pub description: String,
pub homepage: String,
pub license: String,
#[serde(
deserialize_with = "deserialize_licenses",
serialize_with = "serialize_licenses"
)]
pub license: Vec<String>,
}
fn default_revision() -> u32 {
1
}
fn deserialize_licenses<'de, D>(deserializer: D) -> std::result::Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum StringOrArray {
String(String),
Array(Vec<String>),
}
match StringOrArray::deserialize(deserializer)? {
StringOrArray::String(s) => Ok(vec![s]),
StringOrArray::Array(v) => Ok(v),
}
}
fn serialize_licenses<S>(licenses: &[String], serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: Serializer,
{
if licenses.len() == 1 {
serializer.serialize_str(&licenses[0])
} else {
licenses.serialize(serializer)
}
}
impl PackageSpec {
/// Generate the standard package filename: <name>-<version>-<revision>-<arch>.depot.pkg.tar.zst
pub fn package_filename(&self, arch: &str) -> String {
@@ -613,15 +1307,20 @@ pub struct Source {
pub post_extract: Vec<String>,
}
/// Manual (local) source file to copy before fetching remote sources.
/// Manual source copied before standard source fetching.
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
pub struct ManualSource {
/// Filename in the spec directory
pub file: String,
/// SHA256 checksum (optional, use "skip" to bypass verification)
/// Filename in the spec directory (local manual source mode).
#[serde(default)]
pub file: Option<String>,
/// Remote URL to fetch (remote manual source mode).
#[serde(default)]
pub url: Option<String>,
/// Checksum (optional, use "skip" to bypass verification).
#[serde(default)]
pub sha256: Option<String>,
/// Destination path relative to build work directory (default: same as file)
/// Destination path relative to build work directory.
/// Defaults to `file` for local mode or a derived filename for URL mode.
#[serde(default)]
pub dest: Option<String>,
}
@@ -675,6 +1374,15 @@ pub struct BuildFlags {
pub cflags: Vec<String>,
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub ldflags: Vec<String>,
/// Keep existing files and install package-provided replacement as `<path>.depotnew`.
#[serde(default, deserialize_with = "deserialize_string_or_array")]
pub keep: Vec<String>,
/// Disable exporting CFLAGS/CXXFLAGS/LDFLAGS for this package build.
#[serde(default, alias = "no-flags")]
pub no_flags: bool,
/// Skip automatic build-system test execution (e.g. Autotools `make check`/`make test`).
#[serde(default, alias = "skip-tests")]
pub skip_tests: bool,
#[serde(default)]
pub configure: Vec<String>,
/// C compiler
@@ -721,6 +1429,41 @@ pub struct BuildFlags {
/// CPU architecture short name (CARCH equivalent), e.g. "x86_64", "aarch64"
#[serde(default = "default_carch")]
pub carch: String,
/// Variable overrides passed directly to `make` (compile step), e.g. ["V=1", "CC=clang"].
#[serde(
default,
alias = "make-vars",
alias = "make_build_vars",
alias = "make-build-vars",
deserialize_with = "deserialize_string_or_array"
)]
pub make_vars: Vec<String>,
/// Variable overrides passed directly to `make check` / `make test`.
#[serde(
default,
alias = "make-test-vars",
deserialize_with = "deserialize_string_or_array"
)]
pub make_test_vars: Vec<String>,
/// Variable overrides passed directly to `make install`.
#[serde(
default,
alias = "make-install-vars",
deserialize_with = "deserialize_string_or_array"
)]
pub make_install_vars: Vec<String>,
/// Additional host environment variable names to export unchanged to build commands.
/// Example: ["RUSTFLAGS", "CARGO_HOME"].
#[serde(
default,
alias = "passthrough-env",
alias = "pass_env",
alias = "pass-env",
alias = "export_env",
alias = "export-env",
deserialize_with = "deserialize_string_or_array"
)]
pub passthrough_env: Vec<String>,
// Rust-specific fields
/// Rust build profile: "debug" or "release" (default: release)
@@ -756,6 +1499,9 @@ impl Default for BuildFlags {
BuildFlags {
cflags: Vec::new(),
ldflags: Vec::new(),
keep: Vec::new(),
no_flags: false,
skip_tests: false,
configure: Vec::new(),
cc: default_cc(),
cxx: default_cxx(),
@@ -770,6 +1516,10 @@ impl Default for BuildFlags {
chost: String::new(),
cbuild: String::new(),
carch: default_carch(),
make_vars: Vec::new(),
make_test_vars: Vec::new(),
make_install_vars: Vec::new(),
passthrough_env: Vec::new(),
profile: default_profile(),
target: String::new(),
rustflags: Vec::new(),
@@ -852,8 +1602,13 @@ fn default_cxx() -> String {
/// Package dependencies
#[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)]
pub struct Dependencies {
/// Dependencies required for building packages.
#[serde(default)]
pub build: Vec<String>,
/// Dependencies required at runtime.
#[serde(default)]
pub runtime: Vec<String>,
/// Dependencies required to run package test suites.
#[serde(default)]
pub test: Vec<String>,
}
+49 -49
View File
@@ -7,7 +7,7 @@ use std::fs::{self, File};
use std::io::{Cursor, Read, Write};
use std::os::unix::fs as unix_fs;
use std::path::{Path, PathBuf};
use tempfile::{tempdir, NamedTempFile};
use tempfile::{NamedTempFile, tempdir};
use walkdir::WalkDir;
use zstd::stream::read::Decoder as ZstdDecoder;
@@ -85,21 +85,21 @@ fn extract_tar_gz(path: &Path, dest: &Path) -> Result<()> {
// strip that top directory (source tarballs like foo-1.2.3/) or preserve
// it (system-layout archives like usr/). Otherwise move all top-level
// entries into `dest` so `dest` always contains the source root.
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
@@ -130,21 +130,21 @@ fn extract_tar_xz(path: &Path, dest: &Path) -> Result<()> {
let mut archive = tar::Archive::new(decoder);
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
@@ -175,21 +175,21 @@ fn extract_tar_bz2(path: &Path, dest: &Path) -> Result<()> {
let mut archive = tar::Archive::new(decoder);
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
@@ -219,21 +219,21 @@ fn extract_tar(path: &Path, dest: &Path) -> Result<()> {
let mut archive = tar::Archive::new(file);
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
@@ -263,21 +263,21 @@ fn extract_zip(path: &Path, dest: &Path) -> Result<()> {
let mut archive = zip::ZipArchive::new(file)?;
archive.extract(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
@@ -308,21 +308,21 @@ fn extract_tar_zst(path: &Path, dest: &Path) -> Result<()> {
let mut archive = tar::Archive::new(decoder);
archive.unpack(tmp.path())?;
let top = fs::read_dir(tmp.path())?.filter_map(|r| r.ok()).collect::<Vec<_>>();
let top = fs::read_dir(tmp.path())?
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
if top.len() == 1 && top[0].path().is_dir() {
let top_name = top[0].file_name().to_string_lossy().to_string();
let expected_basename = dest
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
let expected_basename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
// blacklist of system-layout directories we should NOT strip
let sys_blacklist = [
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run",
"dev", "proc", "sys", "boot", "srv", "home",
"usr", "bin", "sbin", "lib", "lib64", "etc", "share", "opt", "var", "run", "dev",
"proc", "sys", "boot", "srv", "home",
];
let looks_like_versioned = |s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let looks_like_versioned =
|s: &str| s.contains('-') && s.chars().any(|c| c.is_ascii_digit());
let should_strip = (!sys_blacklist.contains(&top_name.as_str()))
&& (top_name == expected_basename
|| (!expected_basename.is_empty() && top_name.contains(expected_basename))
+400 -131
View File
@@ -4,11 +4,14 @@ use crate::package::{PackageSpec, Source};
use anyhow::{Context, Result, bail};
use indicatif::{ProgressBar, ProgressStyle};
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::fs::{self, File};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use url::Url;
const MAX_MIRROR_RETRIES: usize = 8;
/// Fetch an archive source tarball, returning path to downloaded file.
pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> Result<PathBuf> {
let url = spec.expand_vars(&source.url);
@@ -30,33 +33,48 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
// Parse URL early so we can handle non-HTTP schemes (FTP support)
let parsed_url = Url::parse(&url).with_context(|| format!("Invalid URL: {}", url))?;
// If this is an FTP URL, fetch via the ftp crate into the cache and continue
// If this is an FTP URL, fetch via suppaftp into the cache and continue
if parsed_url.scheme() == "ftp" {
// Connect and login (anonymous fallback)
let host = parsed_url.host_str().context("FTP URL missing host")?;
let port = parsed_url.port_or_known_default().unwrap_or(21);
let addr = format!("{}:{}", host, port);
let mut ftp_stream = ftp::FtpStream::connect(addr.as_str())
let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str())
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
let user = if parsed_url.username().is_empty() { "anonymous" } else { parsed_url.username() };
let user = if parsed_url.username().is_empty() {
"anonymous"
} else {
parsed_url.username()
};
let pass = parsed_url.password().unwrap_or("anonymous@");
ftp_stream.login(user, pass).with_context(|| format!("FTP login failed for {}", host))?;
ftp_stream
.login(user, pass)
.with_context(|| format!("FTP login failed for {}", host))?;
// Retrieve the path (try with and without leading slash)
let path = parsed_url.path();
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
let mut retrieved = false;
for p in candidates.iter().filter(|s| !s.is_empty()) {
match ftp_stream.retr(p, |reader: &mut dyn Read| -> std::result::Result<(), ftp::FtpError> {
let mut file = File::create(&dest_path).map_err(ftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
loop {
let bytes_read = reader.read(&mut buffer).map_err(ftp::FtpError::ConnectionError)?;
if bytes_read == 0 { break; }
file.write_all(&buffer[..bytes_read]).map_err(ftp::FtpError::ConnectionError)?;
}
Ok(())
}) {
match ftp_stream.retr(
p,
|reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> {
let mut file =
File::create(&dest_path).map_err(suppaftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
loop {
let bytes_read = reader
.read(&mut buffer)
.map_err(suppaftp::FtpError::ConnectionError)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])
.map_err(suppaftp::FtpError::ConnectionError)?;
}
Ok(())
},
) {
Ok(_) => {
retrieved = true;
break;
@@ -74,10 +92,8 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
// Use a sensible default User-Agent so servers that reject empty/unknown agents (e.g. IANA)
// will accept requests. Include package name/version at compile time.
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
let client = reqwest::blocking::Client::builder()
.user_agent(ua)
.build()
.with_context(|| "Failed to build HTTP client")?;
let client =
super::build_blocking_client(&ua, None).with_context(|| "Failed to build HTTP client")?;
let mut response = client
.get(&url)
@@ -132,42 +148,71 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
// Quick validation: ensure the downloaded file looks like the expected
// archive (detect obvious HTML error pages or wrong formats by magic).
// If validate_downloaded_archive returns an alternate URL (e.g. SourceForge
// mirror), retry the download with that URL once.
if let Some(alt) = validate_downloaded_archive(&dest_path, &filename, &url)? {
// If validation returns an alternate URL (e.g. SourceForge mirror), follow
// and retry a few times.
let mut next_alt = validate_downloaded_archive(&dest_path, &filename, &url)?;
let mut seen_alts: HashSet<String> = HashSet::new();
let mut retries = 0usize;
while let Some(alt) = next_alt {
retries += 1;
if retries > MAX_MIRROR_RETRIES {
bail!(
"Exceeded mirror retry limit ({}) while fetching {}",
MAX_MIRROR_RETRIES,
url
);
}
if !seen_alts.insert(alt.clone()) {
bail!("Mirror retry loop detected for URL: {}", alt);
}
println!("Retrying download from mirror: {}", alt);
fs::remove_file(&dest_path).ok();
// If mirror URL is FTP -> use ftp crate; otherwise use HTTP retry.
// If mirror URL is FTP -> use suppaftp; otherwise use HTTP retry.
if let Ok(alt_url) = Url::parse(&alt) {
if alt_url.scheme() == "ftp" {
// FTP mirror retrieval
let host = alt_url.host_str().context("FTP mirror URL missing host")?;
let port = alt_url.port_or_known_default().unwrap_or(21);
let addr = format!("{}:{}", host, port);
let mut ftp_stream = ftp::FtpStream::connect(addr.as_str())
let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str())
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
let user = if alt_url.username().is_empty() { "anonymous" } else { alt_url.username() };
let user = if alt_url.username().is_empty() {
"anonymous"
} else {
alt_url.username()
};
let pass = alt_url.password().unwrap_or("anonymous@");
ftp_stream.login(user, pass).with_context(|| format!("FTP login failed for {}", host))?;
ftp_stream
.login(user, pass)
.with_context(|| format!("FTP login failed for {}", host))?;
let path = alt_url.path();
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
let mut retrieved = false;
for p in candidates.iter().filter(|s| !s.is_empty()) {
match ftp_stream.retr(p, |reader: &mut dyn Read| -> std::result::Result<(), ftp::FtpError> {
let mut file = File::create(&dest_path).map_err(ftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = reader.read(&mut buffer).map_err(ftp::FtpError::ConnectionError)?;
if bytes_read == 0 { break; }
file.write_all(&buffer[..bytes_read]).map_err(ftp::FtpError::ConnectionError)?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
Ok(())
}) {
match ftp_stream.retr(
p,
|reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> {
let mut file = File::create(&dest_path)
.map_err(suppaftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = reader
.read(&mut buffer)
.map_err(suppaftp::FtpError::ConnectionError)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])
.map_err(suppaftp::FtpError::ConnectionError)?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
Ok(())
},
) {
Ok(_) => {
retrieved = true;
break;
@@ -182,15 +227,30 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
} else {
// HTTP(S) mirror retry (recreate client for retry)
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
let client = reqwest::blocking::Client::builder()
.user_agent(ua)
.build()
let client = super::build_blocking_client(&ua, None)
.with_context(|| "Failed to build HTTP client")?;
let mut response = client
.get(&alt)
.send()
.with_context(|| format!("Failed to fetch mirror URL: {}", alt))?;
let status = response.status();
if !status.is_success() {
let mut preview_bytes = Vec::new();
let _ = response.take(1024).read_to_end(&mut preview_bytes);
let preview = String::from_utf8_lossy(&preview_bytes);
bail!(
"HTTP error fetching {}: {}{}",
alt,
status,
if preview.trim().is_empty() {
"".to_string()
} else {
format!(" — preview: {}", preview.trim())
}
);
}
let mut file = File::create(&dest_path)
.with_context(|| format!("Failed to create: {}", dest_path.display()))?;
let mut buffer = [0u8; 8192];
@@ -211,9 +271,7 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
}
pb.finish_with_message("Download complete (mirror)");
// Re-validate the mirrored file
validate_downloaded_archive(&dest_path, &filename, &alt)?;
next_alt = validate_downloaded_archive(&dest_path, &filename, &alt)?;
}
// Verify checksum
@@ -239,13 +297,12 @@ fn verify_checksum(path: &Path, expected: &str) -> Result<bool> {
super::verify_file_hash(path, expected)
}
/// Derive a stable filename from a URL.
///
/// Rules:
/// - Parse as URL and use the last path segment if it looks like a filename (contains a dot)
/// - Otherwise fall back to a stable hash-based name: source-{sha256(url)[..12]}.download
fn derive_filename_from_url(url: &str) -> String {
pub(crate) fn derive_filename_from_url(url: &str) -> String {
// try to parse the URL
if let Some(last) = Url::parse(url).ok().and_then(|parsed| {
parsed
@@ -267,7 +324,11 @@ fn derive_filename_from_url(url: &str) -> String {
/// Validate downloaded file's magic header to make sure it is the expected
/// archive format (avoids saving HTML pages or other unexpected content).
fn validate_downloaded_archive(path: &std::path::Path, filename: &str, orig_url: &str) -> Result<Option<String>> {
fn validate_downloaded_archive(
path: &std::path::Path,
filename: &str,
orig_url: &str,
) -> Result<Option<String>> {
use std::io::Read;
let mut f = std::fs::File::open(path)?;
let mut buf = [0u8; 4096];
@@ -275,114 +336,251 @@ fn validate_downloaded_archive(path: &std::path::Path, filename: &str, orig_url:
let head = &buf[..n.min(4096)];
// Detect obvious HTML error pages (case-insensitive)
let head_str = String::from_utf8_lossy(head).to_ascii_lowercase();
if head_str.starts_with("<!doctype html") || head_str.starts_with("<html") || head_str.contains("<html") {
// If this came from SourceForge project URL, try to extract a direct
// downloads.sourceforge.net mirror link and return it for a retry.
let body = String::from_utf8_lossy(head).to_string();
if orig_url.contains("sourceforge.net") {
// helper: find nearest href attribute before `pos` and support
// both double- and single-quoted attributes.
let find_href_before = |body: &str, pos: usize| -> Option<String> {
if let Some(start) = body[..pos].rfind("href=\"") {
let rest = &body[start + 6..];
if let Some(end) = rest.find('"') {
return Some(rest[..end].to_string());
}
}
if let Some(start) = body[..pos].rfind("href='") {
let rest = &body[start + 6..];
if let Some(end) = rest.find('\'') {
return Some(rest[..end].to_string());
}
}
None
};
// look for downloads.sourceforge.net links in HTML
if let Some(pos) = body.find("downloads.sourceforge.net") {
if let Some(href) = find_href_before(&body, pos) {
let alt = if href.starts_with("//") {
format!("https:{}", href)
} else if href.starts_with("/") {
format!("https://downloads.sourceforge.net{}", href)
} else {
href
};
return Ok(Some(alt));
}
if is_html_content(head) {
if url_contains_sourceforge_host(orig_url) {
let body = std::fs::read(path)
.map(|bytes| String::from_utf8_lossy(&bytes).to_string())
.unwrap_or_else(|_| String::from_utf8_lossy(head).to_string());
if let Some(alt) = sourceforge_alt_url_from_html(&body, orig_url) {
return Ok(Some(alt));
}
// Also try to find '/download' link anywhere in the body
if let Some(pos) = body.find("/download") {
if let Some(href) = find_href_before(&body, pos) {
let alt = if href.starts_with("//") {
format!("https:{}", href)
} else {
href
};
return Ok(Some(alt));
}
}
// Final fallback for SF project pages: append '/download' to the
// original URL (this normally triggers the mirror redirect).
return Ok(Some(format!("{}/download", orig_url.trim_end_matches('/'))));
}
let preview = String::from_utf8_lossy(&head[..head.len().min(1024)]);
anyhow::bail!(
"Downloaded file '{}' looks like HTML (not an archive). Preview: {}",
filename,
preview.trim()
html_preview(head)
);
}
// Validate by extension (best-effort)
let lower = filename.to_ascii_lowercase();
let is_ok = if lower.ends_with(".tar.xz") || lower.ends_with(".txz") || lower.ends_with(".xz") {
let is_ok = classify_archive_magic(head, &lower, path);
if !is_ok {
anyhow::bail!(
"Downloaded file '{}' does not match expected archive magic; preview: {}",
filename,
html_preview(head)
);
}
Ok(None)
}
fn sourceforge_alt_url_from_html(body: &str, orig_url: &str) -> Option<String> {
let lower = body.to_ascii_lowercase();
for href in extract_hrefs(body, &lower) {
if href.contains("downloads.sourceforge.net")
&& let Some(url) = sourceforge_candidate_from_href(&href)
{
return Some(url);
}
}
for href in extract_hrefs(body, &lower) {
if href.contains("/download")
&& let Some(url) = sourceforge_candidate_from_href(&href)
{
return Some(url);
}
}
sourceforge_download_fallback(orig_url)
}
fn extract_hrefs(body: &str, lower: &str) -> Vec<String> {
let mut out = Vec::new();
let mut i = 0usize;
while let Some(rel) = lower[i..].find("href=") {
let start = i + rel + 5;
if start >= body.len() {
break;
}
let quote = body.as_bytes()[start] as char;
if quote != '"' && quote != '\'' {
i = start + 1;
continue;
}
let val_start = start + 1;
if val_start > body.len() {
break;
}
if let Some(end_rel) = body[val_start..].find(quote) {
out.push(body[val_start..val_start + end_rel].to_string());
i = val_start + end_rel + 1;
} else {
break;
}
}
out
}
fn normalize_downloads_sf_href(href: &str) -> String {
let href = href.replace("&amp;", "&");
if href.starts_with("//") {
format!("https:{}", href)
} else if href.starts_with('/') {
format!("https://downloads.sourceforge.net{}", href)
} else {
href
}
}
fn normalize_sf_href(href: &str) -> String {
let href = href.replace("&amp;", "&");
if href.starts_with("//") {
format!("https:{}", href)
} else if href.starts_with('/') {
format!("https://sourceforge.net{}", href)
} else {
href
}
}
fn sourceforge_candidate_from_href(href: &str) -> Option<String> {
let normalized = if href.contains("downloads.sourceforge.net") {
normalize_downloads_sf_href(href)
} else {
normalize_sf_href(href)
};
let parsed = Url::parse(&normalized).ok()?;
let host = parsed.host_str()?.to_ascii_lowercase();
if is_sourceforge_host(&host) || is_downloads_sourceforge_host(&host) {
return Some(normalized);
}
// Some HTML pages use social share links that embed a real SourceForge URL
// in a query parameter (e.g. x.com/share?url=...).
if is_social_share_host(&host) {
for (k, v) in parsed.query_pairs() {
if (k.eq_ignore_ascii_case("url") || k.eq_ignore_ascii_case("u"))
&& let Ok(inner) = Url::parse(v.as_ref())
&& let Some(inner_host) = inner.host_str()
{
let inner_host = inner_host.to_ascii_lowercase();
if is_sourceforge_host(&inner_host) || is_downloads_sourceforge_host(&inner_host) {
return Some(inner.to_string());
}
}
}
}
None
}
fn sourceforge_download_fallback(orig_url: &str) -> Option<String> {
let parsed = Url::parse(orig_url).ok()?;
let host = parsed.host_str()?.to_ascii_lowercase();
if is_sourceforge_host(&host) || is_downloads_sourceforge_host(&host) {
return Some(format!("{}/download", orig_url.trim_end_matches('/')));
}
if is_social_share_host(&host) {
for (k, v) in parsed.query_pairs() {
if (k.eq_ignore_ascii_case("url") || k.eq_ignore_ascii_case("u"))
&& let Ok(inner) = Url::parse(v.as_ref())
&& let Some(inner_host) = inner.host_str()
{
let inner_host = inner_host.to_ascii_lowercase();
if is_sourceforge_host(&inner_host) || is_downloads_sourceforge_host(&inner_host) {
return Some(format!("{}/download", inner.as_str().trim_end_matches('/')));
}
}
}
}
None
}
fn is_sourceforge_host(host: &str) -> bool {
host == "sourceforge.net" || host.ends_with(".sourceforge.net")
}
fn is_downloads_sourceforge_host(host: &str) -> bool {
host == "downloads.sourceforge.net" || host.ends_with(".downloads.sourceforge.net")
}
fn is_social_share_host(host: &str) -> bool {
matches!(
host,
"x.com"
| "www.x.com"
| "twitter.com"
| "www.twitter.com"
| "facebook.com"
| "www.facebook.com"
| "linkedin.com"
| "www.linkedin.com"
| "reddit.com"
| "www.reddit.com"
| "t.me"
| "telegram.me"
)
}
fn url_contains_sourceforge_host(url: &str) -> bool {
Url::parse(url)
.ok()
.and_then(|u| u.host_str().map(|h| h.to_ascii_lowercase()))
.map(|h| is_sourceforge_host(&h) || is_downloads_sourceforge_host(&h))
.unwrap_or_else(|| {
url.contains("://sourceforge.net/")
|| url.contains("://downloads.sourceforge.net/")
|| url.contains(".sourceforge.net/")
})
}
fn html_preview(head: &[u8]) -> String {
String::from_utf8_lossy(&head[..head.len().min(1024)])
.trim()
.to_string()
}
fn is_html_content(head: &[u8]) -> bool {
let head_str = String::from_utf8_lossy(head).to_ascii_lowercase();
head_str.starts_with("<!doctype html")
|| head_str.starts_with("<html")
|| head_str.contains("<html")
}
fn classify_archive_magic(head: &[u8], lower_filename: &str, path: &Path) -> bool {
if lower_filename.ends_with(".tar.xz")
|| lower_filename.ends_with(".txz")
|| lower_filename.ends_with(".xz")
{
head.starts_with(&[0xFD, b'7', b'z', b'X', b'Z', 0x00])
} else if lower.ends_with(".tar.gz") || lower.ends_with(".tgz") || lower.ends_with(".gz") {
} else if lower_filename.ends_with(".tar.gz")
|| lower_filename.ends_with(".tgz")
|| lower_filename.ends_with(".gz")
{
head.starts_with(&[0x1F, 0x8B])
} else if lower.ends_with(".tar.zst") || lower.ends_with(".tzst") || lower.ends_with(".zst") {
} else if lower_filename.ends_with(".tar.zst")
|| lower_filename.ends_with(".tzst")
|| lower_filename.ends_with(".zst")
{
head.starts_with(&[0x28, 0xB5, 0x2F, 0xFD])
} else if lower.ends_with(".zip") {
} else if lower_filename.ends_with(".zip") {
head.starts_with(b"PK\x03\x04")
} else if lower.ends_with(".tar.bz2") || lower.ends_with(".tbz2") {
} else if lower_filename.ends_with(".tar.bz2") || lower_filename.ends_with(".tbz2") {
head.starts_with(&[0x42, 0x5A, 0x68])
} else if lower.ends_with(".tar") {
// check for ustar magic at offset 257
} else if lower_filename.ends_with(".tar") {
if let Ok(mut f2) = std::fs::File::open(path) {
let mut hdr = [0u8; 262];
if f2.read_exact(&mut hdr).is_ok() {
&hdr[257..262] == b"ustar"
} else {
true // can't validate; be permissive
true
}
} else {
true
}
} else if lower.ends_with(".deb") {
// ar archive starts with "!<arch>\n"
} else if lower_filename.ends_with(".deb") {
head.starts_with(b"!<arch>")
} else if lower.ends_with(".rpm") {
// rpm contains cpio magic later; best-effort: accept
true
} else {
// Unknown extension -> be permissive
// rpm/unknown extensions: keep permissive as before.
true
};
if !is_ok {
let preview = String::from_utf8_lossy(&head[..head.len().min(1024)]);
anyhow::bail!(
"Downloaded file '{}' does not match expected archive magic; preview: {}",
filename,
preview.trim()
);
}
Ok(None)
}
#[cfg(test)]
@@ -413,7 +611,10 @@ mod tests {
#[test]
fn filename_from_ftp_url() {
assert_eq!(derive_filename_from_url("ftp://example.com/foo-1.2.3.tar.gz"), "foo-1.2.3.tar.gz");
assert_eq!(
derive_filename_from_url("ftp://example.com/foo-1.2.3.tar.gz"),
"foo-1.2.3.tar.gz"
);
}
#[test]
@@ -426,7 +627,11 @@ mod tests {
fn sourceforge_html_no_link_falls_back_to_download_suffix() {
use std::io::Write;
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(tmp, "<!doctype html><html><body>No direct link</body></html>").unwrap();
write!(
tmp,
"<!doctype html><html><body>No direct link</body></html>"
)
.unwrap();
let alt = validate_downloaded_archive(
tmp.path(),
"zsh-5.9.tar.xz",
@@ -455,7 +660,73 @@ mod tests {
.unwrap();
assert_eq!(
alt,
Some("https://downloads.sourceforge.net/project/zsh/zsh/5.9/zsh-5.9.tar.xz?download".to_string())
Some(
"https://downloads.sourceforge.net/project/zsh/zsh/5.9/zsh-5.9.tar.xz?download"
.to_string()
)
);
}
#[test]
fn sourceforge_html_large_page_extracts_downloads_link_beyond_4k() {
use std::io::Write;
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
"<!doctype html><html><body>{}<a href='//downloads.sourceforge.net/project/tcl/tcl8.6.17-src.tar.gz?download'>download</a></body></html>",
"x".repeat(9000)
)
.unwrap();
let alt = validate_downloaded_archive(
tmp.path(),
"tcl8.6.17-src.tar.gz",
"https://sourceforge.net/projects/tcl/files/tcl8.6.17-src.tar.gz",
)
.unwrap();
assert_eq!(
alt,
Some(
"https://downloads.sourceforge.net/project/tcl/tcl8.6.17-src.tar.gz?download"
.to_string()
)
);
}
#[test]
fn sourceforge_html_ignores_social_share_links_and_unwraps_url_param() {
use std::io::Write;
let mut tmp = tempfile::NamedTempFile::new().unwrap();
write!(
tmp,
"<!doctype html><a href='https://x.com/share?url=https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz/download&amp;text=share'>share</a>"
)
.unwrap();
let alt = validate_downloaded_archive(
tmp.path(),
"tcl8.6.17-src.tar.gz",
"https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz",
)
.unwrap();
assert_eq!(
alt,
Some(
"https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz/download"
.to_string()
)
);
}
#[test]
fn sourceforge_share_url_fallback_uses_embedded_sourceforge_url() {
let fallback = sourceforge_download_fallback(
"https://x.com/share?url=https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz",
);
assert_eq!(
fallback,
Some(
"https://sourceforge.net/projects/tcl/files/Tcl/8.6.17/tcl8.6.17-src.tar.gz/download"
.to_string()
)
);
}
@@ -495,5 +766,3 @@ mod tests {
assert!(!verify_checksum(tmp.path(), "md5:deadbeef").unwrap());
}
}
+13 -6
View File
@@ -19,15 +19,23 @@ pub fn checkout(
git_cache_dir: &Path,
pkgname: &str,
) -> Result<()> {
fs::create_dir_all(git_cache_dir)
.with_context(|| format!("Failed to create git cache dir: {}", git_cache_dir.display()))?;
fs::create_dir_all(git_cache_dir).with_context(|| {
format!(
"Failed to create git cache dir: {}",
git_cache_dir.display()
)
})?;
let mirror_dir = git_cache_dir.join(mirror_key(url));
ensure_mirror(url, &mirror_dir, pkgname)?;
if checkout_dir.exists() {
fs::remove_dir_all(checkout_dir)
.with_context(|| format!("Failed to remove existing checkout: {}", checkout_dir.display()))?;
fs::remove_dir_all(checkout_dir).with_context(|| {
format!(
"Failed to remove existing checkout: {}",
checkout_dir.display()
)
})?;
}
// Clone from local mirror for speed.
@@ -40,8 +48,7 @@ pub fn checkout(
.with_context(|| format!("Failed to clone from mirror for {}", url))?;
let repo = Repository::open(checkout_dir)?;
checkout_rev(&repo, rev)
.with_context(|| format!("Failed to checkout revision '{}'", rev))?;
checkout_rev(&repo, rev).with_context(|| format!("Failed to checkout revision '{}'", rev))?;
Ok(())
}
+1 -1
View File
@@ -253,7 +253,7 @@ mod tests {
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: "MIT".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Alternatives::default(),
+194 -31
View File
@@ -11,13 +11,15 @@ use sha2::{Digest, Sha256};
use std::fs;
use std::os::unix::fs as unix_fs;
use std::path::{Path, PathBuf};
use url::Url;
use walkdir::WalkDir;
/// Copy manual (local) sources to the build directory before fetching remote sources.
/// Copy manual sources to the build directory before fetching remote sources.
///
/// Manual sources are checked first, allowing local files like utility source code
/// to be available during the build process.
pub fn copy_manual_sources(spec: &PackageSpec, build_dir: &Path) -> Result<()> {
/// Manual sources support:
/// - local mode: `file = "..."` (path relative to spec directory)
/// - remote mode: `url = "..."` (downloaded first, then copied)
pub fn copy_manual_sources(spec: &PackageSpec, cache_dir: &Path, build_dir: &Path) -> Result<()> {
if spec.manual_sources.is_empty() {
return Ok(());
}
@@ -26,40 +28,83 @@ pub fn copy_manual_sources(spec: &PackageSpec, build_dir: &Path) -> Result<()> {
println!("Copying {} manual source(s)...", spec.manual_sources.len());
for manual in &spec.manual_sources {
let src_path = spec.spec_dir.join(&manual.file);
if !src_path.exists() {
bail!(
"Manual source not found: {} (expected at {})",
manual.file,
src_path.display()
);
}
let (source_path, source_label, default_dest): (PathBuf, String, String) =
if let Some(file) = manual.file.as_ref() {
let file = spec.expand_vars(file);
let src_path = spec.spec_dir.join(&file);
if !src_path.exists() {
bail!(
"Manual source not found: {} (expected at {})",
file,
src_path.display()
);
}
// Verify checksum if provided (supports `sha256:...`, `sha512:...`, `md5:...`, or raw SHA256).
if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip") {
if !verify_file_hash(&src_path, expected_hash)? {
bail!(
"Checksum mismatch for {}: expected {}",
manual.file,
expected_hash
);
}
}
// Verify checksum if provided.
if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip")
&& !verify_file_hash(&src_path, expected_hash)?
{
bail!("Checksum mismatch for {}: expected {}", file, expected_hash);
}
(src_path, file.clone(), file)
} else if let Some(url_raw) = manual.url.as_ref() {
let expanded_url = spec.expand_vars(url_raw);
let parsed = Url::parse(&expanded_url)
.with_context(|| format!("Invalid URL: {}", expanded_url))?;
if parsed.scheme() == "file" {
let src_path = parsed
.to_file_path()
.map_err(|_| anyhow::anyhow!("Invalid file URL: {}", expanded_url))?;
if !src_path.exists() {
bail!("Manual source file URL not found: {}", src_path.display());
}
if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip")
&& !verify_file_hash(&src_path, expected_hash)?
{
bail!(
"Checksum mismatch for {}: expected {}",
expanded_url,
expected_hash
);
}
let default_dest = fetcher::derive_filename_from_url(&expanded_url);
(src_path, expanded_url, default_dest)
} else {
let source = crate::package::Source {
url: expanded_url.clone(),
sha256: manual.sha256.clone().unwrap_or_else(|| "skip".to_string()),
extract_dir: "manual-source".to_string(),
patches: Vec::new(),
post_extract: Vec::new(),
};
let fetched = fetcher::fetch_archive(spec, &source, &cache_dir.join("manual"))?;
let default_dest = fetcher::derive_filename_from_url(&expanded_url);
(fetched, expanded_url, default_dest)
}
} else {
bail!("Manual source must define either 'file' or 'url'");
};
// Determine destination
let dest_name = manual.dest.as_deref().unwrap_or(&manual.file);
let dest_path = build_dir.join(dest_name);
let dest_name = if let Some(dest) = manual.dest.as_ref() {
spec.expand_vars(dest)
} else {
default_dest
};
let dest_path = build_dir.join(&dest_name);
// Create parent directories if needed
if let Some(parent) = dest_path.parent() {
fs::create_dir_all(parent)?;
}
println!(" {} -> {}", manual.file, dest_path.display());
fs::copy(&src_path, &dest_path).with_context(|| {
println!(" {} -> {}", source_label, dest_path.display());
fs::copy(&source_path, &dest_path).with_context(|| {
format!(
"Failed to copy {} to {}",
src_path.display(),
source_path.display(),
dest_path.display()
)
})?;
@@ -85,7 +130,11 @@ fn verify_file_hash(path: &Path, expected: &str) -> Result<bool> {
let (alg, hex) = if let Some(pos) = exp.find(':') {
let a = exp[..pos].trim().to_ascii_lowercase();
let h = exp[pos + 1..].trim().to_ascii_lowercase();
let alg = if a.is_empty() { "sha256".to_string() } else { a };
let alg = if a.is_empty() {
"sha256".to_string()
} else {
a
};
(alg, h.to_string())
} else {
("sha256".to_string(), exp.to_ascii_lowercase())
@@ -140,6 +189,22 @@ fn verify_file_hash(path: &Path, expected: &str) -> Result<bool> {
}
}
/// Build a blocking reqwest HTTP client using the platform-default TLS backend.
/// Any error building the client is returned directly (no fallback).
pub(crate) fn build_blocking_client(
user_agent: &str,
timeout: Option<std::time::Duration>,
) -> anyhow::Result<reqwest::blocking::Client> {
let mut builder = reqwest::blocking::Client::builder().user_agent(user_agent.to_string());
if let Some(t) = timeout {
builder = builder.timeout(t);
}
builder
.build()
.map_err(|e| anyhow::anyhow!("Failed to build HTTP client: {}", e))
}
/// Fetch + extract all sources.
///
/// Returns the primary source directory (the first source entry, or work_dir for manual-only packages).
@@ -148,12 +213,12 @@ pub fn prepare(spec: &PackageSpec, cache_dir: &Path, build_dir: &Path) -> Result
if spec.sources().is_empty() {
let work_dir = build_dir.join(&spec.package.name);
fs::create_dir_all(&work_dir)?;
copy_manual_sources(spec, &work_dir)?;
copy_manual_sources(spec, cache_dir, &work_dir)?;
return Ok(work_dir);
}
// Copy manual sources first (before any remote fetching)
copy_manual_sources(spec, build_dir)?;
copy_manual_sources(spec, cache_dir, build_dir)?;
let mut primary: Option<PathBuf> = None;
@@ -311,6 +376,39 @@ fn split_git_url(url: &str) -> Option<(String, String)> {
#[cfg(test)]
mod tests {
use super::*;
use crate::package::{
Alternatives, Build, BuildFlags, BuildType, Dependencies, ManualSource, PackageInfo,
PackageSpec, Source,
};
fn mk_spec_with_manuals(spec_dir: PathBuf, manuals: Vec<ManualSource>) -> PackageSpec {
PackageSpec {
package: PackageInfo {
name: "foo".into(),
version: "1.0".into(),
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Alternatives::default(),
manual_sources: manuals,
source: vec![Source {
url: "https://example.com/src.tar.gz".into(),
sha256: "skip".into(),
extract_dir: "src".into(),
patches: Vec::new(),
post_extract: Vec::new(),
}],
build: Build {
build_type: BuildType::Custom,
flags: BuildFlags::default(),
},
dependencies: Dependencies::default(),
spec_dir,
}
}
#[test]
fn split_git_url_accepts_git_with_rev() {
@@ -365,5 +463,70 @@ mod tests {
assert!(verify_file_hash(tmp.path(), &format!(":{}", sha256_hex)).unwrap());
assert!(!verify_file_hash(tmp.path(), "md5:deadbeef").unwrap());
}
}
#[test]
fn build_blocking_client_with_and_without_timeout() {
use std::time::Duration;
let ua = "depot/test";
let c1 = build_blocking_client(ua, None).expect("client build failed");
assert!(c1.get("https://example.com").build().is_ok());
let c2 =
build_blocking_client(ua, Some(Duration::from_secs(5))).expect("client build failed");
assert!(c2.get("https://example.com").build().is_ok());
}
#[test]
fn copy_manual_sources_local_file_mode() {
let tmp = tempfile::tempdir().unwrap();
let spec_dir = tmp.path().join("spec");
let cache_dir = tmp.path().join("cache");
let build_dir = tmp.path().join("build");
std::fs::create_dir_all(&spec_dir).unwrap();
std::fs::write(spec_dir.join("manual.patch"), "patch-data").unwrap();
let spec = mk_spec_with_manuals(
spec_dir.clone(),
vec![ManualSource {
file: Some("manual.patch".into()),
url: None,
sha256: None,
dest: None,
}],
);
copy_manual_sources(&spec, &cache_dir, &build_dir).unwrap();
assert_eq!(
std::fs::read_to_string(build_dir.join("manual.patch")).unwrap(),
"patch-data"
);
}
#[test]
fn copy_manual_sources_url_mode_file_scheme() {
let tmp = tempfile::tempdir().unwrap();
let spec_dir = tmp.path().join("spec");
let cache_dir = tmp.path().join("cache");
let build_dir = tmp.path().join("build");
std::fs::create_dir_all(&spec_dir).unwrap();
let remote_file = tmp.path().join("remote-resource.txt");
std::fs::write(&remote_file, "remote-data").unwrap();
let url = format!("file://{}", remote_file.display());
let spec = mk_spec_with_manuals(
spec_dir,
vec![ManualSource {
file: None,
url: Some(url),
sha256: Some("skip".into()),
dest: Some("assets/manual.txt".into()),
}],
);
copy_manual_sources(&spec, &cache_dir, &build_dir).unwrap();
assert_eq!(
std::fs::read_to_string(build_dir.join("assets/manual.txt")).unwrap(),
"remote-data"
);
}
}
+272 -19
View File
@@ -2,12 +2,59 @@
use crate::package::PackageSpec;
use anyhow::{Context, Result};
use std::collections::HashSet;
use std::fs;
use std::path::Component;
use std::path::Path;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use walkdir::WalkDir;
fn is_skipped_install_path(rel_path: &str) -> bool {
let p = rel_path.trim_start_matches('/');
p == ".metadata.toml"
|| p == ".files.yaml"
|| p == "usr/share/info/dir"
|| p.starts_with("usr/share/info/dir.")
}
fn normalize_relative_path(path: &str) -> Result<String> {
let trimmed = path.trim();
if trimmed.is_empty() {
anyhow::bail!("keep paths must not be empty");
}
let p = Path::new(trimmed);
if p.is_absolute() {
anyhow::bail!("keep paths must be relative: {}", trimmed);
}
let mut normalized = PathBuf::new();
for comp in p.components() {
match comp {
Component::Normal(seg) => normalized.push(seg),
Component::CurDir => {}
_ => {
anyhow::bail!(
"keep paths must not contain traversal or root components: {}",
trimmed
);
}
}
}
let s = normalized
.to_str()
.context("keep paths must be valid UTF-8")?
.to_string();
if s.is_empty() {
anyhow::bail!("keep paths must not resolve to empty paths: {}", trimmed);
}
Ok(s)
}
/// Process staged files - remove .la files, strip binaries, etc.
pub fn process(destdir: &Path, _spec: &PackageSpec) -> Result<()> {
println!("Processing staged files...");
@@ -112,7 +159,7 @@ impl FsTransaction {
for rel in &self.removed {
let src = self.removed_backup_path(rel);
let dst = self.rootfs.join(rel);
if src.exists() {
if src.symlink_metadata().is_ok() {
if let Some(parent) = dst.parent() {
fs::create_dir_all(parent)?;
}
@@ -132,7 +179,7 @@ impl FsTransaction {
for rel in &self.backed_up {
let src = self.backup_path(rel);
let dst = self.rootfs.join(rel);
if src.exists() {
if src.symlink_metadata().is_ok() {
if let Some(parent) = dst.parent() {
fs::create_dir_all(parent)?;
}
@@ -174,7 +221,13 @@ pub fn install_atomic(
rootfs: &Path,
tx_base_dir: &Path,
remove_paths: &[String],
keep_paths: &[String],
) -> Result<FsTransaction> {
let keep_set: HashSet<String> = keep_paths
.iter()
.map(|p| normalize_relative_path(p))
.collect::<Result<HashSet<_>>>()?;
fs::create_dir_all(tx_base_dir)
.with_context(|| format!("Failed to create tx dir: {}", tx_base_dir.display()))?;
@@ -236,7 +289,18 @@ pub fn install_atomic(
.to_string_lossy()
.to_string();
let dest_path = rootfs.join(&rel_path);
if is_skipped_install_path(&rel_path) {
continue;
}
let keep_as_depotnew = keep_set.contains(&rel_path) && rootfs.join(&rel_path).exists();
let install_rel_path = if keep_as_depotnew {
format!("{}.depotnew", rel_path)
} else {
rel_path.clone()
};
let dest_path = rootfs.join(&install_rel_path);
if let Some(parent) = dest_path.parent() {
fs::create_dir_all(parent)?;
@@ -245,7 +309,7 @@ pub fn install_atomic(
if dest_path.symlink_metadata().is_ok() {
// lexists checks existence without following symlinks
// Backup existing
let backup_path = tx.backup_path(&rel_path);
let backup_path = tx.backup_path(&install_rel_path);
if let Some(parent) = backup_path.parent() {
fs::create_dir_all(parent)?;
}
@@ -259,9 +323,9 @@ pub fn install_atomic(
fs::copy(&dest_path, &backup_path)?;
}
tx.backed_up.push(rel_path.clone());
tx.backed_up.push(install_rel_path.clone());
} else {
tx.created.push(rel_path.clone());
tx.created.push(install_rel_path.clone());
}
// Install new file/symlink
@@ -277,10 +341,10 @@ pub fn install_atomic(
if file_type.is_symlink() {
let target = fs::read_link(src_path)?;
std::os::unix::fs::symlink(target, &dest_path)
.with_context(|| format!("Failed to create symlink: {}", rel_path))?;
.with_context(|| format!("Failed to create symlink: {}", install_rel_path))?;
} else {
fs::copy(src_path, &dest_path)
.with_context(|| format!("Failed to install: {}", rel_path))?;
.with_context(|| format!("Failed to install: {}", install_rel_path))?;
}
}
@@ -288,21 +352,38 @@ pub fn install_atomic(
for rel in remove_paths {
let rel = rel.as_str();
let dest_path = rootfs.join(rel);
if !dest_path.exists() {
continue;
}
if !dest_path.is_file() {
// Only track files for now.
let dest_meta = match dest_path.symlink_metadata() {
Ok(m) => m,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(e) => {
return Err(e).with_context(|| {
format!("Failed to inspect obsolete path before removal: {}", rel)
});
}
};
if dest_meta.file_type().is_dir() {
// Only obsolete files/symlinks are removed here.
continue;
}
let backup_path = tx.removed_backup_path(rel);
if let Some(parent) = backup_path.parent() {
fs::create_dir_all(parent)?;
}
fs::copy(&dest_path, &backup_path)
.with_context(|| format!("Failed to backup removed file: {}", rel))?;
if dest_meta.file_type().is_symlink() {
let target = fs::read_link(&dest_path)
.with_context(|| format!("Failed to read obsolete symlink target: {}", rel))?;
std::os::unix::fs::symlink(&target, &backup_path)
.with_context(|| format!("Failed to backup removed symlink: {}", rel))?;
} else {
fs::copy(&dest_path, &backup_path)
.with_context(|| format!("Failed to backup removed file: {}", rel))?;
}
fs::remove_file(&dest_path)
.with_context(|| format!("Failed to remove obsolete file: {}", rel))?;
.with_context(|| format!("Failed to remove obsolete file/symlink: {}", rel))?;
tx.removed.push(rel.to_string());
}
@@ -362,7 +443,7 @@ mod tests {
std::fs::write(destdir.join("usr/bin/new_only"), "added").unwrap();
let remove_paths = vec!["usr/bin/old_only".to_string()];
let tx = install_atomic(&destdir, &rootfs, &tx_base, &remove_paths).unwrap();
let tx = install_atomic(&destdir, &rootfs, &tx_base, &remove_paths, &[]).unwrap();
// After install: updated + new present, obsolete removed
assert_eq!(
@@ -382,6 +463,94 @@ mod tests {
assert!(rootfs.join("usr/bin/old_only").exists());
}
#[test]
fn install_atomic_keep_existing_installs_depotnew() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = tmp.path().join("root");
let destdir = tmp.path().join("dest");
let tx_base = tmp.path().join("tx");
std::fs::create_dir_all(rootfs.join("etc")).unwrap();
std::fs::create_dir_all(destdir.join("etc")).unwrap();
std::fs::write(rootfs.join("etc/locale.gen"), "existing").unwrap();
std::fs::write(destdir.join("etc/locale.gen"), "from-package").unwrap();
let keep = vec!["etc/locale.gen".to_string()];
let tx = install_atomic(&destdir, &rootfs, &tx_base, &[], &keep).unwrap();
assert_eq!(
std::fs::read_to_string(rootfs.join("etc/locale.gen")).unwrap(),
"existing"
);
assert_eq!(
std::fs::read_to_string(rootfs.join("etc/locale.gen.depotnew")).unwrap(),
"from-package"
);
tx.rollback().unwrap();
assert_eq!(
std::fs::read_to_string(rootfs.join("etc/locale.gen")).unwrap(),
"existing"
);
assert!(!rootfs.join("etc/locale.gen.depotnew").exists());
}
#[test]
fn install_atomic_rejects_unsafe_keep_paths() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = tmp.path().join("root");
let destdir = tmp.path().join("dest");
let tx_base = tmp.path().join("tx");
std::fs::create_dir_all(&rootfs).unwrap();
std::fs::create_dir_all(destdir.join("etc")).unwrap();
std::fs::write(destdir.join("etc/locale.gen"), "x").unwrap();
let keep = vec!["../etc/shadow".to_string()];
let err = install_atomic(&destdir, &rootfs, &tx_base, &[], &keep)
.expect_err("expected keep path traversal to be rejected");
assert!(
err.to_string()
.contains("keep paths must not contain traversal")
);
}
#[test]
fn install_atomic_removes_obsolete_symlink_paths() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = tmp.path().join("root");
let destdir = tmp.path().join("dest");
let tx_base = tmp.path().join("tx");
std::fs::create_dir_all(rootfs.join("usr/lib")).unwrap();
std::fs::create_dir_all(destdir.join("usr/bin")).unwrap();
std::fs::write(destdir.join("usr/bin/new"), "ok").unwrap();
std::os::unix::fs::symlink("../lib/libold.so", rootfs.join("usr/lib/libold.so.link"))
.unwrap();
assert!(
rootfs
.join("usr/lib/libold.so.link")
.symlink_metadata()
.is_ok()
);
let remove_paths = vec!["usr/lib/libold.so.link".to_string()];
let tx = install_atomic(&destdir, &rootfs, &tx_base, &remove_paths, &[]).unwrap();
assert!(
rootfs
.join("usr/lib/libold.so.link")
.symlink_metadata()
.is_err()
);
tx.rollback().unwrap();
let restored = rootfs
.join("usr/lib/libold.so.link")
.symlink_metadata()
.expect("symlink should be restored");
assert!(restored.file_type().is_symlink());
}
#[test]
fn install_atomic_commit_removes_tx_dir() {
let tmp = tempfile::tempdir().unwrap();
@@ -392,7 +561,7 @@ mod tests {
std::fs::create_dir_all(destdir.join("usr/bin")).unwrap();
std::fs::write(destdir.join("usr/bin/foo"), "x").unwrap();
let tx = install_atomic(&destdir, &rootfs, &tx_base, &[]).unwrap();
let tx = install_atomic(&destdir, &rootfs, &tx_base, &[], &[]).unwrap();
let tx_dir = tx.tx_dir.clone();
assert!(tx_dir.exists());
tx.commit().unwrap();
@@ -410,7 +579,7 @@ mod tests {
// Create a symlink bin -> usr/bin in destdir
std::os::unix::fs::symlink("usr/bin", destdir.join("bin")).unwrap();
let _tx = install_atomic(&destdir, &rootfs, &tx_base, &[]).unwrap();
let _tx = install_atomic(&destdir, &rootfs, &tx_base, &[], &[]).unwrap();
// Verify rootfs/bin is a symlink, not a directory
let meta = rootfs
@@ -442,6 +611,86 @@ mod tests {
}
}
}
#[test]
fn install_atomic_skips_info_dir_index() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = tmp.path().join("root");
let destdir = tmp.path().join("dest");
let tx_base = tmp.path().join("tx");
std::fs::create_dir_all(&rootfs).unwrap();
std::fs::create_dir_all(destdir.join("usr/share/info")).unwrap();
std::fs::write(destdir.join("usr/share/info/dir"), "index").unwrap();
std::fs::write(destdir.join("usr/share/info/dir.gz"), "index gz").unwrap();
std::fs::write(destdir.join("usr/share/info/ok.info"), "ok").unwrap();
let _tx = install_atomic(&destdir, &rootfs, &tx_base, &[], &[]).unwrap();
assert!(!rootfs.join("usr/share/info/dir").exists());
assert!(!rootfs.join("usr/share/info/dir.gz").exists());
assert!(rootfs.join("usr/share/info/ok.info").exists());
}
#[test]
fn install_atomic_skips_package_metadata_files() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = tmp.path().join("root");
let destdir = tmp.path().join("dest");
let tx_base = tmp.path().join("tx");
std::fs::create_dir_all(&rootfs).unwrap();
std::fs::create_dir_all(&destdir).unwrap();
std::fs::create_dir_all(destdir.join("usr/bin")).unwrap();
std::fs::write(destdir.join(".metadata.toml"), "name='foo'").unwrap();
std::fs::write(destdir.join(".files.yaml"), "files: []").unwrap();
std::fs::write(destdir.join("usr/bin/ok"), "ok").unwrap();
let _tx = install_atomic(&destdir, &rootfs, &tx_base, &[], &[]).unwrap();
assert!(!rootfs.join(".metadata.toml").exists());
assert!(!rootfs.join(".files.yaml").exists());
assert!(rootfs.join("usr/bin/ok").exists());
}
#[test]
fn generate_manifest_skips_info_dir_index() {
let tmp = tempfile::tempdir().unwrap();
let destdir = tmp.path().join("dest");
std::fs::create_dir_all(destdir.join("usr/share/info")).unwrap();
std::fs::write(destdir.join("usr/share/info/dir"), "index").unwrap();
std::fs::write(destdir.join("usr/share/info/dir.xz"), "index xz").unwrap();
std::fs::write(destdir.join("usr/share/info/ok.info"), "ok").unwrap();
let manifest = generate_manifest_with_dirs(&destdir).unwrap();
assert!(!manifest.files.contains(&"usr/share/info/dir".to_string()));
assert!(
!manifest
.files
.contains(&"usr/share/info/dir.xz".to_string())
);
assert!(
manifest
.files
.contains(&"usr/share/info/ok.info".to_string())
);
}
#[test]
fn generate_manifest_skips_package_metadata_files() {
let tmp = tempfile::tempdir().unwrap();
let destdir = tmp.path().join("dest");
std::fs::create_dir_all(&destdir).unwrap();
std::fs::create_dir_all(destdir.join("usr/bin")).unwrap();
std::fs::write(destdir.join(".metadata.toml"), "name='foo'").unwrap();
std::fs::write(destdir.join(".files.yaml"), "files: []").unwrap();
std::fs::write(destdir.join("usr/bin/ok"), "ok").unwrap();
let manifest = generate_manifest_with_dirs(&destdir).unwrap();
assert!(!manifest.files.contains(&".metadata.toml".to_string()));
assert!(!manifest.files.contains(&".files.yaml".to_string()));
assert!(manifest.files.contains(&"usr/bin/ok".to_string()));
}
}
/// Manifest containing files and directories for a package
@@ -468,6 +717,10 @@ pub fn generate_manifest_with_dirs(destdir: &Path) -> Result<Manifest> {
continue;
}
if is_skipped_install_path(&rel_path) {
continue;
}
let file_type = entry.file_type();
// Check for symlink first