feat: improve depot UX and packaging workflow

- add shared terminal UI helpers for colored info/warn/success logs and yes/no prompts
- add Python build backend (PEP 517/setup.py wheel build + install) and expose python in interactive spec creation
- expand spec/build flags (post_configure, make phase targets/dirs, configure_file, no_strip, no_compress_man)
- support split-output staging with internal .depot output dirs, shell helpers (haul/subdestdir), and per-output deps/provides overrides
- improve staging with ELF auto-strip and zstd manpage compression (with opt-out flags)
- enforce runtime deps before install and support legacy lifecycle hook script names
- improve manual source handling with files/urls lists and stricter validation
This commit is contained in:
2026-02-22 14:52:45 -06:00
parent 0c676f6743
commit d63ad03e98
28 changed files with 4639 additions and 412 deletions
+2 -2
View File
@@ -35,7 +35,7 @@ pub fn extract_archive(
.and_then(|n| n.to_str())
.unwrap_or("");
println!("Extracting: {}", filename);
crate::log_info!("Extracting: {}", filename);
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
extract_tar_gz(archive_path, &extract_path)?;
@@ -70,7 +70,7 @@ pub fn extract_archive(
);
}
println!("Extracted to: {}", extract_path.display());
crate::log_info!("Extracted to: {}", extract_path.display());
Ok(extract_path)
}
+4 -4
View File
@@ -24,11 +24,11 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
// Check if already cached and verified
if dest_path.exists() && verify_checksum(&dest_path, &source.sha256)? {
println!("Using cached source: {}", dest_path.display());
crate::log_info!("Using cached source: {}", dest_path.display());
return Ok(dest_path);
}
println!("Fetching: {}", url);
crate::log_info!("Fetching: {}", url);
// Parse URL early so we can handle non-HTTP schemes (FTP support)
let parsed_url = Url::parse(&url).with_context(|| format!("Invalid URL: {}", url))?;
@@ -165,7 +165,7 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
if !seen_alts.insert(alt.clone()) {
bail!("Mirror retry loop detected for URL: {}", alt);
}
println!("Retrying download from mirror: {}", alt);
crate::log_info!("Retrying download from mirror: {}", alt);
fs::remove_file(&dest_path).ok();
// If mirror URL is FTP -> use suppaftp; otherwise use HTTP retry.
@@ -280,7 +280,7 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
bail!("Checksum verification failed for {}", filename);
}
println!("Checksum verified: {}", filename);
crate::log_info!("Checksum verified: {}", filename);
Ok(dest_path)
}
+2 -2
View File
@@ -43,7 +43,7 @@ pub fn checkout(
.to_str()
.ok_or_else(|| anyhow::anyhow!("Invalid mirror path"))?;
println!("Cloning git source into {}...", checkout_dir.display());
crate::log_info!("Cloning git source into {}...", checkout_dir.display());
Repository::clone(mirror_url, checkout_dir)
.with_context(|| format!("Failed to clone from mirror for {}", url))?;
@@ -62,7 +62,7 @@ fn mirror_key(url: &str) -> String {
fn ensure_mirror(url: &str, mirror_dir: &Path, pkgname: &str) -> Result<()> {
if !mirror_dir.exists() {
println!("Cloning git mirror for {} ({})...", pkgname, url);
crate::log_info!("Cloning git mirror for {} ({})...", pkgname, url);
let mut fo = FetchOptions::new();
fo.remote_callbacks(remote_callbacks());
let mut builder = git2::build::RepoBuilder::new();
+95 -23
View File
@@ -50,13 +50,13 @@ fn apply_patches(
)
})?;
println!("Applying {} patch(es)...", source.patches.len());
crate::log_info!("Applying {} patch(es)...", source.patches.len());
for p in &source.patches {
let p = spec.expand_vars(p);
let patch_path = resolve_patch_path(spec, &p, &patch_cache_dir)?;
println!(" patch: {}", patch_path.display());
crate::log_info!(" patch: {}", patch_path.display());
// Apply with patch(1). Keep it simple: -p1 is the common case.
let status = Command::new("patch")
@@ -80,14 +80,14 @@ fn run_post_extract_commands(spec: &PackageSpec, source: &Source, src_dir: &Path
return Ok(());
}
println!(
crate::log_info!(
"Running {} post-extract command(s)...",
source.post_extract.len()
);
for cmd in &source.post_extract {
let cmd = spec.expand_vars(cmd);
println!(" post_extract: {}", cmd);
crate::log_info!(" post_extract: {}", cmd);
// Use a shell for convenience; this is a package manager, so specs are trusted input.
let status = Command::new("sh")
@@ -107,32 +107,75 @@ fn run_post_extract_commands(spec: &PackageSpec, source: &Source, src_dir: &Path
}
/// Run post-compile commands (after make, before make install).
pub fn run_post_compile_commands(spec: &PackageSpec, src_dir: &Path, destdir: &Path) -> Result<()> {
let commands = &spec.build.flags.post_compile;
pub fn run_post_configure_commands(
spec: &PackageSpec,
src_dir: &Path,
destdir: &Path,
) -> Result<()> {
let commands = &spec.build.flags.post_configure;
if commands.is_empty() {
return Ok(());
}
println!("Running {} post-compile command(s)...", commands.len());
crate::log_info!("Running {} post-configure command(s)...", commands.len());
let shell_helpers = crate::shell_helpers::ShellHelpers::new(destdir)?;
for cmd in commands {
let cmd = spec.expand_vars(cmd);
println!(" post_compile: {}", cmd);
let cmd_str = spec.expand_vars(cmd);
crate::log_info!(" post_configure: {}", cmd_str);
let status = Command::new("sh")
.current_dir(src_dir)
let mut shell_cmd = Command::new("sh");
shell_cmd.current_dir(src_dir);
shell_helpers.apply_to_command(&mut shell_cmd);
let status = shell_cmd
.env("DEPOT_SPECDIR", &spec.spec_dir)
.env("DESTDIR", destdir)
.env("DEPOT_ROOTFS", &spec.build.flags.rootfs)
.env("CC", &spec.build.flags.cc)
.env("AR", &spec.build.flags.ar)
.arg("-c")
.arg(&cmd)
.arg(&cmd_str)
.status()
.with_context(|| format!("Failed to run post_compile command: {}", cmd))?;
.with_context(|| format!("Failed to run post_configure command: {}", cmd_str))?;
if !status.success() {
bail!("post_compile command failed: {}", cmd);
bail!("post_configure command failed: {}", cmd_str);
}
}
Ok(())
}
/// Run post-compile commands (after make, before make install).
pub fn run_post_compile_commands(spec: &PackageSpec, src_dir: &Path, destdir: &Path) -> Result<()> {
let commands = &spec.build.flags.post_compile;
if commands.is_empty() {
return Ok(());
}
crate::log_info!("Running {} post-compile command(s)...", commands.len());
let shell_helpers = crate::shell_helpers::ShellHelpers::new(destdir)?;
for cmd in commands {
let cmd_str = spec.expand_vars(cmd);
crate::log_info!(" post_compile: {}", cmd_str);
let mut shell_cmd = Command::new("sh");
shell_cmd.current_dir(src_dir);
shell_helpers.apply_to_command(&mut shell_cmd);
let status = shell_cmd
.env("DEPOT_SPECDIR", &spec.spec_dir)
.env("DESTDIR", destdir)
.env("DEPOT_ROOTFS", &spec.build.flags.rootfs)
.env("CC", &spec.build.flags.cc)
.env("AR", &spec.build.flags.ar)
.arg("-c")
.arg(&cmd_str)
.status()
.with_context(|| format!("Failed to run post_compile command: {}", cmd_str))?;
if !status.success() {
bail!("post_compile command failed: {}", cmd_str);
}
}
@@ -146,26 +189,29 @@ pub fn run_post_install_commands(spec: &PackageSpec, src_dir: &Path, destdir: &P
return Ok(());
}
println!("Running {} post-install command(s)...", commands.len());
crate::log_info!("Running {} post-install command(s)...", commands.len());
let shell_helpers = crate::shell_helpers::ShellHelpers::new(destdir)?;
for cmd in commands {
let cmd = spec.expand_vars(cmd);
println!(" post_install: {}", cmd);
let cmd_str = spec.expand_vars(cmd);
crate::log_info!(" post_install: {}", cmd_str);
let status = Command::new("sh")
.current_dir(src_dir)
let mut shell_cmd = Command::new("sh");
shell_cmd.current_dir(src_dir);
shell_helpers.apply_to_command(&mut shell_cmd);
let status = shell_cmd
.env("DEPOT_SPECDIR", &spec.spec_dir)
.env("DESTDIR", destdir)
.env("DEPOT_ROOTFS", &spec.build.flags.rootfs)
.env("CC", &spec.build.flags.cc)
.env("AR", &spec.build.flags.ar)
.arg("-c")
.arg(&cmd)
.arg(&cmd_str)
.status()
.with_context(|| format!("Failed to run post_install command: {}", cmd))?;
.with_context(|| format!("Failed to run post_install command: {}", cmd_str))?;
if !status.success() {
bail!("post_install command failed: {}", cmd);
bail!("post_install command failed: {}", cmd_str);
}
}
@@ -240,7 +286,7 @@ fn download(url: &str, dest: &Path) -> Result<()> {
#[cfg(test)]
mod tests {
use super::resolve_patch_path;
use super::{resolve_patch_path, run_post_install_commands};
use crate::package::{
Alternatives, Build, BuildFlags, BuildType, Dependencies, PackageInfo, PackageSpec, Source,
};
@@ -270,6 +316,8 @@ mod tests {
flags: BuildFlags::default(),
},
dependencies: Dependencies::default(),
package_alternatives: Default::default(),
package_dependencies: Default::default(),
spec_dir: spec_dir.to_path_buf(),
}
}
@@ -308,4 +356,28 @@ mod tests {
let resolved = resolve_patch_path(&spec, url, &patch_cache_dir).unwrap();
assert_eq!(resolved, cached);
}
#[test]
fn post_install_commands_can_haul_into_output_staging() {
let tmp = tempfile::tempdir().unwrap();
let spec_dir = tmp.path().join("spec");
let src_dir = tmp.path().join("src");
let destdir = tmp.path().join("dest");
std::fs::create_dir_all(&spec_dir).unwrap();
std::fs::create_dir_all(&src_dir).unwrap();
std::fs::create_dir_all(destdir.join("usr/lib")).unwrap();
std::fs::write(destdir.join("usr/lib/libLLVM.so.1"), "llvm").unwrap();
let mut spec = dummy_spec(&spec_dir);
spec.build.flags.post_install = vec!["haul llvm-libs 'usr/lib/libLLVM*.so*'".into()];
run_post_install_commands(&spec, &src_dir, &destdir).unwrap();
assert!(!destdir.join("usr/lib/libLLVM.so.1").exists());
assert!(
destdir
.join(".depot/outputs/llvm-libs/usr/lib/libLLVM.so.1")
.exists()
);
}
}
+203 -64
View File
@@ -25,12 +25,44 @@ pub fn copy_manual_sources(spec: &PackageSpec, cache_dir: &Path, build_dir: &Pat
}
fs::create_dir_all(build_dir)?;
println!("Copying {} manual source(s)...", spec.manual_sources.len());
let manual_entry_count: usize = spec
.manual_sources
.iter()
.map(|m| {
usize::from(m.file.as_ref().is_some_and(|s| !s.trim().is_empty()))
+ m.files.iter().filter(|s| !s.trim().is_empty()).count()
+ usize::from(m.url.as_ref().is_some_and(|s| !s.trim().is_empty()))
+ m.urls.iter().filter(|s| !s.trim().is_empty()).count()
})
.sum();
crate::log_info!("Copying {} manual source(s)...", manual_entry_count);
for manual in &spec.manual_sources {
let (source_path, source_label, default_dest): (PathBuf, String, String) =
if let Some(file) = manual.file.as_ref() {
let file = spec.expand_vars(file);
let mut local_entries: Vec<String> = Vec::new();
if let Some(file) = manual.file.as_ref() {
if !file.trim().is_empty() {
local_entries.push(file.clone());
}
}
local_entries.extend(
manual
.files
.iter()
.filter(|s| !s.trim().is_empty())
.cloned(),
);
let mut url_entries: Vec<String> = Vec::new();
if let Some(url) = manual.url.as_ref() {
if !url.trim().is_empty() {
url_entries.push(url.clone());
}
}
url_entries.extend(manual.urls.iter().filter(|s| !s.trim().is_empty()).cloned());
if !local_entries.is_empty() {
for raw_file in local_entries {
let file = spec.expand_vars(&raw_file);
let src_path = spec.spec_dir.join(&file);
if !src_path.exists() {
bail!(
@@ -40,79 +72,105 @@ pub fn copy_manual_sources(spec: &PackageSpec, cache_dir: &Path, build_dir: &Pat
);
}
// Verify checksum if provided.
if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip")
&& !verify_file_hash(&src_path, expected_hash)?
{
bail!("Checksum mismatch for {}: expected {}", file, expected_hash);
}
(src_path, file.clone(), file)
} else if let Some(url_raw) = manual.url.as_ref() {
let expanded_url = spec.expand_vars(url_raw);
let default_dest = file.clone();
copy_manual_source_file(spec, build_dir, &src_path, &file, manual, &default_dest)?;
}
continue;
}
if !url_entries.is_empty() {
for raw_url in url_entries {
let expanded_url = spec.expand_vars(&raw_url);
let parsed = Url::parse(&expanded_url)
.with_context(|| format!("Invalid URL: {}", expanded_url))?;
if parsed.scheme() == "file" {
let src_path = parsed
.to_file_path()
.map_err(|_| anyhow::anyhow!("Invalid file URL: {}", expanded_url))?;
if !src_path.exists() {
bail!("Manual source file URL not found: {}", src_path.display());
}
if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip")
&& !verify_file_hash(&src_path, expected_hash)?
{
bail!(
"Checksum mismatch for {}: expected {}",
expanded_url,
expected_hash
);
}
let default_dest = fetcher::derive_filename_from_url(&expanded_url);
(src_path, expanded_url, default_dest)
} else {
let source = crate::package::Source {
url: expanded_url.clone(),
sha256: manual.sha256.clone().unwrap_or_else(|| "skip".to_string()),
extract_dir: "manual-source".to_string(),
patches: Vec::new(),
post_extract: Vec::new(),
let (source_path, source_label, default_dest): (PathBuf, String, String) =
if parsed.scheme() == "file" {
let src_path = parsed
.to_file_path()
.map_err(|_| anyhow::anyhow!("Invalid file URL: {}", expanded_url))?;
if !src_path.exists() {
bail!("Manual source file URL not found: {}", src_path.display());
}
if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip")
&& !verify_file_hash(&src_path, expected_hash)?
{
bail!(
"Checksum mismatch for {}: expected {}",
expanded_url,
expected_hash
);
}
let default_dest = fetcher::derive_filename_from_url(&expanded_url);
(src_path, expanded_url, default_dest)
} else {
let source = crate::package::Source {
url: expanded_url.clone(),
sha256: manual.sha256.clone().unwrap_or_else(|| "skip".to_string()),
extract_dir: "manual-source".to_string(),
patches: Vec::new(),
post_extract: Vec::new(),
};
let fetched =
fetcher::fetch_archive(spec, &source, &cache_dir.join("manual"))?;
let default_dest = fetcher::derive_filename_from_url(&expanded_url);
(fetched, expanded_url, default_dest)
};
let fetched = fetcher::fetch_archive(spec, &source, &cache_dir.join("manual"))?;
let default_dest = fetcher::derive_filename_from_url(&expanded_url);
(fetched, expanded_url, default_dest)
}
} else {
bail!("Manual source must define either 'file' or 'url'");
};
// Determine destination
let dest_name = if let Some(dest) = manual.dest.as_ref() {
spec.expand_vars(dest)
} else {
default_dest
};
let dest_path = build_dir.join(&dest_name);
// Create parent directories if needed
if let Some(parent) = dest_path.parent() {
fs::create_dir_all(parent)?;
copy_manual_source_file(
spec,
build_dir,
&source_path,
&source_label,
manual,
&default_dest,
)?;
}
continue;
}
println!(" {} -> {}", source_label, dest_path.display());
fs::copy(&source_path, &dest_path).with_context(|| {
format!(
"Failed to copy {} to {}",
source_path.display(),
dest_path.display()
)
})?;
bail!("Manual source must define one of 'file', 'files', 'url', or 'urls'");
}
Ok(())
}
fn copy_manual_source_file(
spec: &PackageSpec,
build_dir: &Path,
source_path: &Path,
source_label: &str,
manual: &crate::package::ManualSource,
default_dest: &str,
) -> Result<()> {
let dest_name = if let Some(dest) = manual.dest.as_ref() {
spec.expand_vars(dest)
} else {
default_dest.to_string()
};
let dest_path = build_dir.join(&dest_name);
if let Some(parent) = dest_path.parent() {
fs::create_dir_all(parent)?;
}
crate::log_info!(" {} -> {}", source_label, dest_path.display());
fs::copy(source_path, &dest_path).with_context(|| {
format!(
"Failed to copy {} to {}",
source_path.display(),
dest_path.display()
)
})?;
Ok(())
}
/// Verify a file against an `expected` checksum string.
///
/// Formats accepted: `sha256:HEX`, `sha512:HEX`, `md5:HEX`, or plain `HEX` (assumed sha256).
@@ -122,7 +180,7 @@ fn verify_file_hash(path: &Path, expected: &str) -> Result<bool> {
let exp = expected.trim();
if exp.eq_ignore_ascii_case("skip") {
println!("Checksum verification skipped");
crate::log_info!("Checksum verification skipped");
return Ok(true);
}
@@ -250,7 +308,7 @@ fn prepare_one(
if dst.exists() {
// If it exists and has a state file, assume we are resuming
if dst.join(".depot_state").exists() {
println!(
crate::log_info!(
"Resuming build in existing source directory: {}",
dst.display()
);
@@ -265,7 +323,7 @@ fn prepare_one(
let src_dir = build_dir.join(&extract_dir_name);
if src_dir.exists() {
if src_dir.join(".depot_state").exists() {
println!(
crate::log_info!(
"Resuming build in existing source directory: {}",
src_dir.display()
);
@@ -288,7 +346,7 @@ fn prepare_one(
if let Some((base, rev)) = split_git_url(&url) {
let checkout_dir = build_dir.join(&extract_dir_name);
if checkout_dir.exists() && checkout_dir.join(".depot_state").exists() {
println!(
crate::log_info!(
"Resuming build in existing git directory: {}",
checkout_dir.display()
);
@@ -307,7 +365,7 @@ fn prepare_one(
let src_dir = build_dir.join(&extract_dir_name);
if src_dir.exists() && src_dir.join(".depot_state").exists() {
println!(
crate::log_info!(
"Resuming build in existing source directory: {}",
src_dir.display()
);
@@ -406,6 +464,8 @@ mod tests {
flags: BuildFlags::default(),
},
dependencies: Dependencies::default(),
package_alternatives: Default::default(),
package_dependencies: Default::default(),
spec_dir,
}
}
@@ -437,6 +497,48 @@ mod tests {
assert_eq!(rev, "HEAD");
}
#[test]
fn split_git_url_accepts_expanded_tag_or_hash_revision() {
let spec = PackageSpec {
package: PackageInfo {
name: "json".into(),
version: "3.11.3".into(),
revision: 1,
description: "d".into(),
homepage: "h".into(),
license: vec!["MIT".into()],
},
packages: Vec::new(),
alternatives: Alternatives::default(),
manual_sources: Vec::new(),
source: vec![Source {
url: "https://github.com/nlohmann/json.git#v$version".into(),
sha256: "skip".into(),
extract_dir: "json-$version".into(),
patches: Vec::new(),
post_extract: Vec::new(),
}],
build: Build {
build_type: BuildType::Custom,
flags: BuildFlags::default(),
},
dependencies: Dependencies::default(),
package_alternatives: Default::default(),
package_dependencies: Default::default(),
spec_dir: PathBuf::from("."),
};
let expanded = spec.expand_vars(&spec.source[0].url);
let (base, rev) = split_git_url(&expanded).unwrap();
assert_eq!(base, "https://github.com/nlohmann/json.git");
assert_eq!(rev, "v3.11.3");
let (base, rev) =
split_git_url("https://github.com/nlohmann/json.git#0123456789abcdef").unwrap();
assert_eq!(base, "https://github.com/nlohmann/json.git");
assert_eq!(rev, "0123456789abcdef");
}
#[test]
fn verify_file_hash_accepts_multiple_algorithms() {
use sha2::{Digest, Sha256, Sha512};
@@ -489,7 +591,9 @@ mod tests {
spec_dir.clone(),
vec![ManualSource {
file: Some("manual.patch".into()),
files: Vec::new(),
url: None,
urls: Vec::new(),
sha256: None,
dest: None,
}],
@@ -517,7 +621,9 @@ mod tests {
spec_dir,
vec![ManualSource {
file: None,
files: Vec::new(),
url: Some(url),
urls: Vec::new(),
sha256: Some("skip".into()),
dest: Some("assets/manual.txt".into()),
}],
@@ -529,4 +635,37 @@ mod tests {
"remote-data"
);
}
#[test]
fn copy_manual_sources_multi_files_in_one_block() {
let tmp = tempfile::tempdir().unwrap();
let spec_dir = tmp.path().join("spec");
let cache_dir = tmp.path().join("cache");
let build_dir = tmp.path().join("build");
std::fs::create_dir_all(spec_dir.join("pam")).unwrap();
std::fs::write(spec_dir.join("pam/other"), "other").unwrap();
std::fs::write(spec_dir.join("pam/system-auth"), "auth").unwrap();
let spec = mk_spec_with_manuals(
spec_dir.clone(),
vec![ManualSource {
file: None,
files: vec!["pam/other".into(), "pam/system-auth".into()],
url: None,
urls: Vec::new(),
sha256: None,
dest: None,
}],
);
copy_manual_sources(&spec, &cache_dir, &build_dir).unwrap();
assert_eq!(
std::fs::read_to_string(build_dir.join("pam/other")).unwrap(),
"other"
);
assert_eq!(
std::fs::read_to_string(build_dir.join("pam/system-auth")).unwrap(),
"auth"
);
}
}