Implement system state management with TOML serialization

- Introduced `SystemState` struct to manage system state, including stage, target, architecture, and layers.
- Added functions to load and save system state from/to a TOML file.
- Implemented layer management functions: add, set, and remove packages from layers.
- Created initialization function for LBI layout, including directory and symlink creation.
- Added tests for layer management and LBI layout initialization.
This commit is contained in:
2026-05-20 19:54:00 -05:00
parent d61d59f4f5
commit 1833a2c42d
18 changed files with 8820 additions and 354 deletions
+6700
View File
File diff suppressed because it is too large Load Diff
+45 -6
View File
@@ -54,14 +54,12 @@ pub fn build(
crate::builder::apply_build_helper_context_env(&mut env_vars, spec)?;
crate::builder::apply_build_helper_dirs_env(&mut env_vars, Some(src_dir), Some(&build_dir));
// For custom builds, look for a build.sh script in the source directory
// For custom builds, the spec's build.sh is authoritative when present.
// Upstream source archives sometimes ship unrelated helper scripts with the
// same name, and the package spec needs to override them deterministically.
let build_script = src_dir.join("build.sh");
// If the extracted source doesn't include build.sh but the spec directory does,
// copy it into the source dir (this makes `depot install <local-spec>` behave
// like the spec's build.sh being part of the package when appropriate).
let spec_build = spec.spec_dir.join("build.sh");
if !build_script.exists() && spec_build.exists() {
if spec_build.exists() {
fs::create_dir_all(src_dir)?;
fs::copy(&spec_build, &build_script).with_context(|| {
format!(
@@ -477,6 +475,47 @@ mod tests {
Ok(())
}
#[test]
fn test_build_prefers_spec_build_sh_over_source_build_sh() -> Result<()> {
let tmp_src = tempdir()?;
let tmp_dest = tempdir()?;
let spec_dir = tempdir()?;
let source_build_sh = tmp_src.path().join("build.sh");
std::fs::write(&source_build_sh, "#!/bin/sh\nexit 77\n")?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = std::fs::metadata(&source_build_sh)?.permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&source_build_sh, perms)?;
}
let spec_build_sh = spec_dir.path().join("build.sh");
std::fs::write(
&spec_build_sh,
"#!/bin/sh\nprintf 'from spec\\n' > \"$DESTDIR/spec-won\"\n",
)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = std::fs::metadata(&spec_build_sh)?.permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&spec_build_sh, perms)?;
}
let mut spec = mk_spec("custom-prefer-spec", "1.0");
spec.spec_dir = spec_dir.path().to_path_buf();
build(&spec, tmp_src.path(), tmp_dest.path(), None, true, None)?;
assert_eq!(
std::fs::read_to_string(tmp_dest.path().join("spec-won"))?,
"from spec\n"
);
Ok(())
}
#[test]
fn test_build_function_mode_uses_per_output_destdirs() -> Result<()> {
let tmp_src = tempdir()?;
+105
View File
@@ -144,6 +144,30 @@ pub struct BuildArgs {
pub cleanup_deps: bool,
}
/// Arguments for importing the Linux by Intent book package plan into Depot layers.
#[derive(Debug, Clone, Args)]
pub struct BootstrapArgs {
/// Target sysroot whose Depot system state should receive the parsed layers
#[arg(value_name = "SYSROOT")]
pub sysroot: PathBuf,
/// Target triple used for cross and staged bootstrap builds
#[arg(long)]
pub target: Option<String>,
/// Target architecture component used by build defaults
#[arg(long)]
pub arch: Option<String>,
/// URL of the Linux by Intent PDF to parse
#[arg(long, default_value = "https://www.vertexlinux.net/lbi/book.pdf")]
pub book_url: String,
/// Use a local PDF instead of fetching book-url
#[arg(long, value_name = "PDF")]
pub book_pdf: Option<PathBuf>,
}
#[derive(Debug, Clone, Args)]
pub struct UpdateArgs {
#[command(flatten)]
@@ -226,6 +250,70 @@ pub struct ConfigArgs {
pub rootfs_args: RootfsArgs,
}
/// Arguments for Depot system-build state management commands.
#[derive(Debug, Clone, Args)]
pub struct SystemArgs {
/// Root filesystem whose system state should be inspected or modified.
#[command(flatten)]
pub rootfs_args: RootfsArgs,
/// Requested system state subcommand.
#[command(subcommand)]
pub command: SystemCommands,
}
/// System-build state and Linux by Intent layout commands.
#[derive(Subcommand, Debug, Clone)]
pub enum SystemCommands {
/// Show tracked system build stage and package layers
Status,
/// Move the tracked system status to a build stage
Stage {
/// New stage name, such as cross-tools, minimal, chroot, stage2, or bootable
stage: String,
},
/// Manage package layer membership
Layer {
#[command(subcommand)]
command: SystemLayerCommands,
},
/// Initialize the Linux by Intent /system layout and Depot build defaults
InitLbi {
/// Target triple used for cross and staged builds
#[arg(long, default_value = "x86_64-unknown-linux-musl")]
target: String,
/// Target architecture component used by build defaults
#[arg(long)]
arch: Option<String>,
/// Replace an existing Depot build config generated for the target rootfs
#[arg(long)]
force: bool,
},
}
/// Package layer membership commands.
#[derive(Subcommand, Debug, Clone)]
pub enum SystemLayerCommands {
/// Add packages to a named layer
Add {
/// Layer name, such as base, devel, toolchain, or boot
layer: String,
/// Package names to add to the layer
#[arg(required = true, num_args = 1..)]
packages: Vec<String>,
},
/// Remove packages from a named layer
Remove {
/// Layer name
layer: String,
/// Package names to remove from the layer
#[arg(required = true, num_args = 1..)]
packages: Vec<String>,
},
/// List all tracked layers
List,
}
#[derive(Debug, Clone, Args)]
pub struct GenerateArtifactsArgs {
/// Output directory for generated files
@@ -265,6 +353,8 @@ pub enum Commands {
Remove(RemoveArgs),
/// Build a package without installing
Build(BuildArgs),
/// Parse the Linux by Intent book and populate temp/base/devel layers
Bootstrap(BootstrapArgs),
/// Update installed packages from configured repositories
Update(UpdateArgs),
/// Scan package specs for upstream version updates
@@ -283,6 +373,8 @@ pub enum Commands {
Repo(RepoArgs),
/// Show current configuration
Config(ConfigArgs),
/// Manage system build stage, package layers, and book-style layout state
System(SystemArgs),
/// Generate shell completion scripts and a man page into an output directory.
#[command(hide = true)]
GenerateArtifacts(GenerateArtifactsArgs),
@@ -317,6 +409,19 @@ pub enum InternalCommands {
#[command(hide = true)]
Clone { repo: String, dest: Option<PathBuf> },
#[command(hide = true)]
BootstrapChroot {
#[arg(long)]
rootfs: PathBuf,
#[arg(long)]
sources: PathBuf,
#[arg(long)]
destdir: PathBuf,
#[arg(long)]
workdir: String,
#[arg(long)]
script: String,
},
#[command(hide = true)]
AutotoolsConfigure {
#[arg(value_name = "ARG", num_args = 0.., allow_hyphen_values = true)]
args: Vec<String>,
+43 -9
View File
@@ -1,10 +1,11 @@
use crate::cli::{
BuildArgs, Cli, Commands, ConfigArgs, ConvertArgs, InfoArgs, InstallArgs, InternalCommands,
ListArgs, OwnsArgs, RemoveArgs, RepoCommands, RepoKindArg, SearchArgs, SignArgs, UpdateArgs,
ListArgs, OwnsArgs, RemoveArgs, RepoCommands, RepoKindArg, SearchArgs, SignArgs, SystemArgs,
UpdateArgs,
};
use crate::{
builder, cli_assets, config, cross, db, deps, index, install, locking, package, planner,
signing, source, staging, ui,
bootstrap, builder, cli_assets, config, cross, db, deps, index, install, locking, package,
planner, signing, source, staging, ui,
};
use anyhow::{Context, Result};
use git2::Direction;
@@ -19,10 +20,11 @@ use url::Url;
use walkdir::WalkDir;
use build_cmd::support::{
automatic_tests_disabled_for_outputs, build_lib32_companion_package, clean_build_workspace,
effective_lib32_only, ensure_requested_development_package_installed, make_lib32_package_spec,
maybe_disable_tests_for_missing_deps, maybe_prompt_to_skip_tests_for_missing_requested_deps,
merge_missing_dependencies, requested_outputs, should_install_test_deps,
automatic_tests_disabled_for_outputs, build_lib32_companion_package, clean_build_source_dirs,
clean_build_workspace, effective_lib32_only, ensure_requested_development_package_installed,
make_lib32_package_spec, maybe_disable_tests_for_missing_deps,
maybe_prompt_to_skip_tests_for_missing_requested_deps, merge_missing_dependencies,
requested_outputs, should_install_test_deps,
};
use install_cmd::archive::{
extract_package_archive_to_staging, load_package_archive_into_staging,
@@ -89,6 +91,7 @@ fn command_rootfs(command: &Commands) -> Option<&Path> {
Commands::Install(args) => Some(&args.rootfs_args.rootfs),
Commands::Remove(args) => Some(&args.rootfs_args.rootfs),
Commands::Build(args) => Some(&args.rootfs_args.rootfs),
Commands::Bootstrap(args) => Some(&args.sysroot),
Commands::Update(args) => Some(&args.rootfs_args.rootfs),
Commands::Info(args) => Some(&args.rootfs_args.rootfs),
Commands::Search(args) => Some(&args.rootfs_args.rootfs),
@@ -97,6 +100,7 @@ fn command_rootfs(command: &Commands) -> Option<&Path> {
Commands::Sign(args) => Some(&args.rootfs_args.rootfs),
Commands::Repo(args) => Some(repo_command_rootfs(&args.command)),
Commands::Config(args) => Some(&args.rootfs_args.rootfs),
Commands::System(args) => Some(&args.rootfs_args.rootfs),
Commands::Check(_)
| Commands::Convert(_)
| Commands::GenerateArtifacts(_)
@@ -110,6 +114,7 @@ fn command_assume_yes(command: &Commands) -> bool {
Commands::Install(args) => args.prompt_args.yes,
Commands::Remove(args) => args.prompt_args.yes,
Commands::Build(args) => args.prompt_args.yes,
Commands::Bootstrap(_) => false,
Commands::Update(args) => args.prompt_args.yes,
Commands::Check(_)
| Commands::Convert(_)
@@ -120,6 +125,7 @@ fn command_assume_yes(command: &Commands) -> bool {
| Commands::Sign(_)
| Commands::Repo(_)
| Commands::Config(_)
| Commands::System(_)
| Commands::GenerateArtifacts(_)
| Commands::MakeSpec(_)
| Commands::Internal(_) => false,
@@ -913,7 +919,7 @@ fn collect_installed_replacement_packages(
Ok(replacements)
}
fn remove_installed_package_with_hooks(
pub(crate) fn remove_installed_package_with_hooks(
package: &str,
rootfs: &Path,
config: &config::Config,
@@ -2054,6 +2060,27 @@ fn run_direct_archive_install_requests(
Ok(true)
}
struct SourceBuildCleanupGuard<'a> {
config: &'a config::Config,
enabled: bool,
}
impl<'a> SourceBuildCleanupGuard<'a> {
fn new(config: &'a config::Config, enabled: bool) -> Self {
Self { config, enabled }
}
}
impl Drop for SourceBuildCleanupGuard<'_> {
fn drop(&mut self) {
if self.enabled
&& let Err(err) = clean_build_source_dirs(self.config)
{
crate::log_warn!("Failed to clean build source dirs: {}", err);
}
}
}
fn run_direct_install_request(
options: DirectInstallOptions<'_>,
config: &config::Config,
@@ -2142,6 +2169,8 @@ fn run_direct_install_request(
pkg_spec.apply_config(config);
(pkg_spec, None)
};
let built_from_source = staging_dir.is_none();
let _source_cleanup_guard = SourceBuildCleanupGuard::new(config, built_from_source);
if options.lib32_only && staging_dir.is_some() {
anyhow::bail!("--lib32-only is only supported when installing from a package spec");
@@ -2438,6 +2467,7 @@ fn run_direct_install_request(
dir.path().to_path_buf()
} else {
// 1-2. Fetch + extract sources (supports archives and git URL#rev)
clean_build_source_dirs(config)?;
source::preflight_manual_sources(&pkg_spec, &config.cache_dir)?;
let src_dir = source::prepare(&pkg_spec, &config.cache_dir, &config.build_dir)?;
built_src_dir = Some(src_dir.clone());
@@ -2551,7 +2581,8 @@ pub fn run(cli: Cli) -> Result<()> {
Commands::Install(args) => args.build_exec_args.test_deps,
Commands::Build(args) => args.build_exec_args.test_deps,
Commands::Update(args) => args.build_exec_args.test_deps,
Commands::Check(_)
Commands::Bootstrap(_)
| Commands::Check(_)
| Commands::Remove(_)
| Commands::Info(_)
| Commands::Search(_)
@@ -2560,6 +2591,7 @@ pub fn run(cli: Cli) -> Result<()> {
| Commands::Sign(_)
| Commands::Repo(_)
| Commands::Config(_)
| Commands::System(_)
| Commands::GenerateArtifacts(_)
| Commands::Convert(_)
| Commands::MakeSpec(_)
@@ -2570,6 +2602,7 @@ pub fn run(cli: Cli) -> Result<()> {
Commands::Install(args) => install_cmd::run_install(args, cli_test_deps)?,
Commands::Remove(args) => install_cmd::run_remove(args)?,
Commands::Build(args) => build_cmd::run_build(args, cli_test_deps)?,
Commands::Bootstrap(args) => bootstrap::run(args)?,
Commands::Update(args) => update::run_update(args, cli_test_deps)?,
Commands::Check(args) => check::run_check(args)?,
Commands::Info(args) => misc::run_info(args)?,
@@ -2580,6 +2613,7 @@ pub fn run(cli: Cli) -> Result<()> {
Commands::Repo(args) => repo::run_repo(args.command)?,
Commands::GenerateArtifacts(args) => misc::run_generate_artifacts(args)?,
Commands::Config(args) => misc::run_config(args)?,
Commands::System(args) => misc::run_system(args)?,
Commands::MakeSpec(args) => misc::run_make_spec(args)?,
Commands::Convert(args) => misc::run_convert(args)?,
Commands::Internal(args) => misc::run_internal(args)?,
+13 -5
View File
@@ -4,11 +4,11 @@ pub(crate) mod support;
use self::support::{
RequestedBuildToolPackageInstall, automatic_tests_disabled_for_outputs,
build_lib32_companion_package, clean_build_workspace, effective_lib32_only,
ensure_requested_build_tool_package_installed, ensure_requested_development_package_installed,
maybe_disable_tests_for_missing_deps, maybe_prompt_to_skip_tests_for_missing_requested_deps,
merge_missing_dependencies, requested_outputs, should_install_test_deps,
warn_if_running_as_root_for_build,
build_lib32_companion_package, clean_build_source_dirs, clean_build_workspace,
effective_lib32_only, ensure_requested_build_tool_package_installed,
ensure_requested_development_package_installed, maybe_disable_tests_for_missing_deps,
maybe_prompt_to_skip_tests_for_missing_requested_deps, merge_missing_dependencies,
requested_outputs, should_install_test_deps, warn_if_running_as_root_for_build,
};
pub(super) fn run_build(args: BuildArgs, cli_test_deps: bool) -> Result<()> {
@@ -303,6 +303,7 @@ pub(super) fn run_build(args: BuildArgs, cli_test_deps: bool) -> Result<()> {
watcher.check()?;
}
clean_build_source_dirs(&config)?;
source::preflight_manual_sources(&pkg_spec, &config.cache_dir)?;
let src_dir = source::prepare(&pkg_spec, &config.cache_dir, &config.build_dir)?;
if let Some(watcher) = interrupt_watcher.as_ref() {
@@ -507,6 +508,7 @@ pub(super) fn run_build(args: BuildArgs, cli_test_deps: bool) -> Result<()> {
false,
)?;
}
clean_build_source_dirs(&config)?;
if clean {
clean_build_workspace(&config)?;
}
@@ -529,6 +531,12 @@ pub(super) fn run_build(args: BuildArgs, cli_test_deps: bool) -> Result<()> {
));
}
}
if let Err(clean_err) = clean_build_source_dirs(&config) {
ui::warn(format!(
"Failed to clean build source dirs after failed build: {}",
clean_err
));
}
if interrupted {
anyhow::bail!("Build interrupted by Ctrl-C");
}
+16
View File
@@ -115,6 +115,22 @@ pub(crate) fn clean_build_workspace(config: &config::Config) -> Result<()> {
Ok(())
}
pub(crate) fn clean_build_source_dirs(config: &config::Config) -> Result<()> {
if config.build_dir.exists() {
fs::remove_dir_all(&config.build_dir).with_context(|| {
format!(
"Failed to clean build source dirs: {}",
config.build_dir.display()
)
})?;
ui::success(format!(
"Cleaned build source dirs: {}",
config.build_dir.display()
));
}
Ok(())
}
pub(crate) fn warn_if_running_as_root_for_build(command: &str, rootfs: &Path) {
if crate::fakeroot::is_root() {
ui::warn(format!("Running '{}' as root is discouraged.", command));
+111
View File
@@ -112,6 +112,117 @@ pub(super) fn run_config(args: ConfigArgs) -> Result<()> {
Ok(())
}
pub(super) fn run_system(args: SystemArgs) -> Result<()> {
let SystemArgs {
rootfs_args,
command,
} = args;
let rootfs = rootfs_args.rootfs;
let config = config::Config::for_rootfs(&rootfs);
let mut system_lock = locking::open_lock(&config)?;
let system_lock_path = locking::lock_path(&config);
match command {
crate::cli::SystemCommands::Status => {
let _guard = locking::try_read(&system_lock, &system_lock_path, "system status")?;
let state = crate::system_state::load(&config)?;
print_system_state(&state);
}
crate::cli::SystemCommands::Stage { stage } => {
let _guard = locking::try_write(&mut system_lock, &system_lock_path, "system stage")?;
let state = crate::system_state::set_stage(&config, stage)?;
ui::success(format!(
"Moved system status to stage {}",
state.stage.as_deref().unwrap_or("unknown")
));
}
crate::cli::SystemCommands::Layer { command } => {
let _guard = locking::try_write(&mut system_lock, &system_lock_path, "system layer")?;
match command {
crate::cli::SystemLayerCommands::Add { layer, packages } => {
let package_count = packages.len();
crate::system_state::add_packages_to_layer(&config, layer.clone(), &packages)?;
ui::success(format!(
"Added {} package(s) to layer {}",
package_count, layer
));
}
crate::cli::SystemLayerCommands::Remove { layer, packages } => {
let package_count = packages.len();
crate::system_state::remove_packages_from_layer(
&config,
layer.clone(),
&packages,
)?;
ui::success(format!(
"Removed {} package(s) from layer {}",
package_count, layer
));
}
crate::cli::SystemLayerCommands::List => {
drop(_guard);
let _guard =
locking::try_read(&system_lock, &system_lock_path, "system layer list")?;
let state = crate::system_state::load(&config)?;
print_system_layers(&state);
}
}
}
crate::cli::SystemCommands::InitLbi {
target,
arch,
force,
} => {
let _guard =
locking::try_write(&mut system_lock, &system_lock_path, "system init-lbi")?;
let state = crate::system_state::init_lbi_layout(
&rootfs,
&config,
&target,
arch.as_deref(),
force,
)?;
ui::success(format!(
"Initialized Linux by Intent layout for {} ({})",
state.target.as_deref().unwrap_or("unknown"),
state.arch.as_deref().unwrap_or("unknown")
));
}
}
Ok(())
}
fn print_system_state(state: &crate::system_state::SystemState) {
println!(
"Stage: {}",
state.stage.as_deref().unwrap_or("uninitialized")
);
if let Some(target) = &state.target {
println!("Target: {target}");
}
if let Some(arch) = &state.arch {
println!("Arch: {arch}");
}
print_system_layers(state);
}
fn print_system_layers(state: &crate::system_state::SystemState) {
if state.layers.is_empty() {
println!("Layers: none");
return;
}
println!("Layers:");
for (layer, packages) in &state.layers {
if packages.is_empty() {
println!(" {layer}:");
} else {
println!(" {}: {}", layer, packages.join(", "));
}
}
}
pub(super) fn run_make_spec(args: crate::cli::MakeSpecArgs) -> Result<()> {
let output = args.output;
let spec = package::create_interactive()?;
+7
View File
@@ -143,6 +143,13 @@ pub(crate) fn run_internal_command(command: InternalCommands) -> Result<()> {
&[],
)
}
InternalCommands::BootstrapChroot {
rootfs,
sources,
destdir,
workdir,
script,
} => crate::bootstrap::run_bootstrap_chroot(&rootfs, &sources, &destdir, &workdir, &script),
InternalCommands::AutotoolsConfigure { args } => {
let env_vars = current_process_env_vars();
let context = current_build_helper_context()?;
+36
View File
@@ -282,6 +282,42 @@ fn clean_build_workspace_noops_when_dirs_are_missing() -> Result<()> {
Ok(())
}
#[test]
fn clean_build_source_dirs_removes_build_dir_only() -> Result<()> {
let rootfs = tempfile::tempdir().context("Failed to create temp rootfs")?;
let mut cfg = config::Config::for_rootfs(rootfs.path());
cfg.build_dir = rootfs.path().join("tmp/build");
cfg.cache_dir = rootfs.path().join("tmp/sources");
fs::create_dir_all(&cfg.build_dir)
.with_context(|| format!("Failed to create {}", cfg.build_dir.display()))?;
fs::create_dir_all(&cfg.cache_dir)
.with_context(|| format!("Failed to create {}", cfg.cache_dir.display()))?;
clean_build_source_dirs(&cfg)?;
assert!(!cfg.build_dir.exists());
assert!(cfg.cache_dir.exists());
Ok(())
}
#[test]
fn clean_build_source_dirs_noops_when_build_dir_missing() -> Result<()> {
let rootfs = tempfile::tempdir().context("Failed to create temp rootfs")?;
let mut cfg = config::Config::for_rootfs(rootfs.path());
cfg.build_dir = rootfs.path().join("tmp/build");
cfg.cache_dir = rootfs.path().join("tmp/sources");
fs::create_dir_all(&cfg.cache_dir)
.with_context(|| format!("Failed to create {}", cfg.cache_dir.display()))?;
clean_build_source_dirs(&cfg)?;
assert!(!cfg.build_dir.exists());
assert!(cfg.cache_dir.exists());
Ok(())
}
#[test]
fn binary_install_path_uses_repo_record_metadata_without_archive_metadata() -> Result<()> {
let rootfs = tempfile::tempdir().context("Failed to create temp rootfs")?;
+34 -1
View File
@@ -41,20 +41,27 @@ impl CrossConfig {
let cc = find_tool(prefix, &["gcc", "clang"], true)?;
let cxx = find_tool(prefix, &["g++", "clang++"], false)
.unwrap_or_else(|_| format!("{}-g++", prefix));
let ar = find_tool(prefix, &["ar", "llvm-ar"], true)?;
let ar =
find_tool_with_unprefixed_fallback(prefix, &["ar", "llvm-ar"], &["llvm-ar"], true)?;
let ranlib = find_tool(prefix, &["ranlib", "llvm-ranlib"], false)
.or_else(|_| find_unprefixed_tool(&["llvm-ranlib", "llvm-ar"], false))
.unwrap_or_else(|_| format!("{}-ranlib", prefix));
let strip = find_tool(prefix, &["strip", "llvm-strip"], false)
.or_else(|_| find_unprefixed_tool(&["llvm-strip", "llvm-objcopy"], false))
.unwrap_or_else(|_| format!("{}-strip", prefix));
let ld = find_tool(prefix, &["ld", "ld.lld"], false)
.unwrap_or_else(|_| format!("{}-ld", prefix));
let nm = find_tool(prefix, &["nm", "llvm-nm"], false)
.or_else(|_| find_unprefixed_tool(&["llvm-nm"], false))
.unwrap_or_else(|_| format!("{}-nm", prefix));
let objcopy = find_tool(prefix, &["objcopy", "llvm-objcopy"], false)
.or_else(|_| find_unprefixed_tool(&["llvm-objcopy"], false))
.unwrap_or_else(|_| format!("{}-objcopy", prefix));
let objdump = find_tool(prefix, &["objdump", "llvm-objdump"], false)
.or_else(|_| find_unprefixed_tool(&["llvm-objdump"], false))
.unwrap_or_else(|_| format!("{}-objdump", prefix));
let readelf = find_tool(prefix, &["readelf", "llvm-readelf"], false)
.or_else(|_| find_unprefixed_tool(&["llvm-readelf", "llvm-readobj"], false))
.unwrap_or_else(|_| format!("{}-readelf", prefix));
crate::log_info!("Cross-compilation tools discovered:");
@@ -244,6 +251,32 @@ fn find_tool(prefix: &str, suffixes: &[&str], required: bool) -> Result<String>
Err(anyhow::anyhow!("Tool not found"))
}
fn find_tool_with_unprefixed_fallback(
prefix: &str,
suffixes: &[&str],
unprefixed: &[&str],
required: bool,
) -> Result<String> {
find_tool(prefix, suffixes, false).or_else(|_| find_unprefixed_tool(unprefixed, required))
}
fn find_unprefixed_tool(names: &[&str], required: bool) -> Result<String> {
for name in names {
if let Ok(output) = Command::new("which").arg(name).output()
&& output.status.success()
{
let path = String::from_utf8_lossy(&output.stdout).trim().to_string();
return Ok(path);
}
}
if required {
anyhow::bail!("Could not find tool: {{{}}}", names.join("|"));
}
Err(anyhow::anyhow!("Tool not found"))
}
#[cfg(test)]
mod tests {
use super::lib32_target_triple;
+84 -1
View File
@@ -10,6 +10,8 @@ use std::fs;
use std::path::Path;
use walkdir::WalkDir;
const DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS: &str = "DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS";
fn format_licenses(licenses: &[String]) -> String {
licenses.join(", ")
}
@@ -18,6 +20,14 @@ fn verbose_remove_output() -> bool {
std::env::var_os("DEPOT_VERBOSE_REMOVE").is_some()
}
fn should_ignore_sbase_conflicts() -> bool {
std::env::var_os(DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS).is_some()
}
fn should_auto_clear_conflict(owner: &str, path: &str) -> bool {
(owner == "sbase" && should_ignore_sbase_conflicts()) || is_auto_removable_path(path)
}
/// Installed package row from the local package database.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct InstalledPackageRecord {
@@ -213,7 +223,7 @@ pub fn register_package_with_replacement(
if let Ok(owner) = owner_res
&& owner != spec.package.name
{
if is_auto_removable_path(file) {
if should_auto_clear_conflict(&owner, file) {
auto_conflicts.push((file.clone(), owner));
} else {
fatal_conflicts.push((file.clone(), owner));
@@ -241,6 +251,15 @@ pub fn register_package_with_replacement(
)?;
if let Some(rootfs) = &rootfs_opt {
if destdir.join(f).symlink_metadata().is_ok() {
crate::log_info!(
"Auto-cleared DB ownership for path: {} (previously owned by {})",
f,
owner
);
continue;
}
let disk_path = rootfs.join(f);
if disk_path.exists() {
let _ = std::fs::remove_file(&disk_path);
@@ -1056,6 +1075,7 @@ mod tests {
use crate::package::{
Alternatives, Build, BuildFlags, BuildType, Dependencies, PackageInfo, PackageSpec, Source,
};
use crate::test_support::TestEnv;
use std::path::PathBuf;
fn mk_spec(name: &str, version: &str) -> PackageSpec {
@@ -1243,6 +1263,69 @@ mod tests {
assert!(files_b.contains(&"usr/share/perl5/shared.pm".to_string()));
}
#[test]
fn register_package_auto_clears_sbase_conflicts_when_requested() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = tmp.path().join("rootfs");
let db_path = crate::config::Config::for_rootfs(&rootfs).installed_db_path(&rootfs);
std::fs::create_dir_all(rootfs.join("system/binaries")).unwrap();
std::fs::write(rootfs.join("system/binaries/find"), "sbase find").unwrap();
let spec_a = mk_spec("sbase", "1.0");
let dest_a = tmp.path().join("dest_a");
std::fs::create_dir_all(dest_a.join("system/binaries")).unwrap();
std::fs::write(dest_a.join("system/binaries/find"), "sbase find").unwrap();
register_package(&db_path, &spec_a, &dest_a).unwrap();
let spec_b = mk_spec("bfs", "4.1");
let dest_b = tmp.path().join("dest_b");
std::fs::create_dir_all(dest_b.join("system/binaries")).unwrap();
std::fs::write(dest_b.join("system/binaries/find"), "bfs find").unwrap();
let mut env = TestEnv::new();
env.set_var(DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS, "1");
register_package(&db_path, &spec_b, &dest_b).unwrap();
let files_sbase = get_package_files(&db_path, "sbase").unwrap();
assert!(!files_sbase.contains(&"system/binaries/find".to_string()));
let files_bfs = get_package_files(&db_path, "bfs").unwrap();
assert!(files_bfs.contains(&"system/binaries/find".to_string()));
}
#[test]
fn register_package_auto_clear_preserves_new_payload_on_disk() {
let tmp = tempfile::tempdir().unwrap();
let rootfs = tmp.path().join("rootfs");
let db_path = crate::config::Config::for_rootfs(&rootfs).installed_db_path(&rootfs);
std::fs::create_dir_all(rootfs.join("system/binaries")).unwrap();
std::fs::write(rootfs.join("system/binaries/find"), "sbase find").unwrap();
let spec_a = mk_spec("sbase", "1.0");
let dest_a = tmp.path().join("dest_a");
std::fs::create_dir_all(dest_a.join("system/binaries")).unwrap();
std::fs::write(dest_a.join("system/binaries/find"), "sbase find").unwrap();
register_package(&db_path, &spec_a, &dest_a).unwrap();
std::fs::write(rootfs.join("system/binaries/find"), "bfs find").unwrap();
let spec_b = mk_spec("bfs", "4.1");
let dest_b = tmp.path().join("dest_b");
std::fs::create_dir_all(dest_b.join("system/binaries")).unwrap();
std::fs::write(dest_b.join("system/binaries/find"), "bfs find").unwrap();
let mut env = TestEnv::new();
env.set_var(DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS, "1");
register_package(&db_path, &spec_b, &dest_b).unwrap();
assert_eq!(
std::fs::read_to_string(rootfs.join("system/binaries/find")).unwrap(),
"bfs find"
);
let files_sbase = get_package_files(&db_path, "sbase").unwrap();
assert!(!files_sbase.contains(&"system/binaries/find".to_string()));
let files_bfs = get_package_files(&db_path, "bfs").unwrap();
assert!(files_bfs.contains(&"system/binaries/find".to_string()));
}
#[test]
fn get_package_files_missing_package_returns_empty() {
let tmp = tempfile::tempdir().unwrap();
+2
View File
@@ -1,6 +1,7 @@
//! Depot - Not Your Average Package Manager
//! A source-based package manager for Linux
mod bootstrap;
mod build_options;
mod builder;
mod cli;
@@ -24,6 +25,7 @@ mod shell_helpers;
mod signing;
mod source;
mod staging;
mod system_state;
#[cfg(test)]
mod test_support;
mod ui;
+308 -243
View File
@@ -8,9 +8,17 @@ use std::collections::HashSet;
use std::fs::{self, File};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::time::Duration;
use url::Url;
const MAX_MIRROR_RETRIES: usize = 8;
const HTTP_FETCH_TIMEOUT: Duration = Duration::from_secs(120);
const HTTP_FETCH_RETRY_LIMIT: usize = 2;
const MUSL_1_2_6_SNAPSHOT_URL: &str =
"https://git.musl-libc.org/cgit/musl/snapshot/musl-1.2.6.tar.gz";
const MUSL_1_2_6_RELEASE_URL: &str = "https://musl.libc.org/releases/musl-1.2.6.tar.gz";
const MUSL_1_2_6_GENTOO_MIRROR_URL: &str =
"https://tw.archive.ubuntu.com/gentoo/distfiles/9d/musl-1.2.6.tar.gz";
fn scheme_uses_http_transport(scheme: &str) -> bool {
matches!(scheme, "http" | "https")
@@ -32,10 +40,6 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
return Ok(dest_path);
}
crate::log_info!("Fetching: {}", url);
// Parse URL early so we can handle non-HTTP schemes (FTP support)
let parsed_url = Url::parse(&url).with_context(|| format!("Invalid URL: {}", url))?;
let pb = ProgressBar::new(0);
pb.set_style(
ProgressStyle::default_bar()
@@ -46,256 +50,46 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R
.progress_chars("#>-"),
);
// If this is an FTP URL, fetch via suppaftp and skip the HTTP client path.
if parsed_url.scheme() == "ftp" {
// Connect and login (anonymous fallback)
let host = parsed_url.host_str().context("FTP URL missing host")?;
let port = parsed_url.port_or_known_default().unwrap_or(21);
let addr = format!("{}:{}", host, port);
let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str())
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
let user = if parsed_url.username().is_empty() {
"anonymous"
let candidate_urls = archive_fetch_candidates(&url);
let mut attempt_errors = Vec::new();
for (index, candidate_url) in candidate_urls.iter().enumerate() {
if index == 0 {
crate::log_info!("Fetching: {}", candidate_url);
} else {
parsed_url.username()
};
let pass = parsed_url.password().unwrap_or("anonymous@");
ftp_stream
.login(user, pass)
.with_context(|| format!("FTP login failed for {}", host))?;
// Retrieve the path (try with and without leading slash)
let path = parsed_url.path();
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
let mut retrieved = false;
for p in candidates.iter().filter(|s| !s.is_empty()) {
match ftp_stream.retr(
p,
|reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> {
let mut file =
File::create(&dest_path).map_err(suppaftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
loop {
let bytes_read = reader
.read(&mut buffer)
.map_err(suppaftp::FtpError::ConnectionError)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])
.map_err(suppaftp::FtpError::ConnectionError)?;
}
Ok(())
},
) {
Ok(_) => {
retrieved = true;
break;
}
Err(_) => continue,
}
}
ftp_stream.quit().ok();
if !retrieved {
bail!("FTP error fetching {}", url);
}
} else {
if !scheme_uses_http_transport(parsed_url.scheme()) {
bail!(
"Unsupported URL scheme for source fetch: {}",
parsed_url.scheme()
);
crate::log_info!("Primary source failed; trying fallback: {}", candidate_url);
}
// Download with progress bar
// Use a sensible default User-Agent so servers that reject empty/unknown agents (e.g. IANA)
// will accept requests. Include package name/version at compile time.
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
let client = super::build_blocking_client(&ua, None)
.with_context(|| "Failed to build HTTP client")?;
let mut response = client
.get(&url)
.send()
.with_context(|| format!("Failed to fetch: {}", url))?;
// If the server returned a non-success status, read a short body preview and fail early.
// This prevents saving HTML error pages (which then fail checksum) and gives a clearer
// diagnostic to the user.
let status = response.status();
if !status.is_success() {
let mut preview_bytes = Vec::new();
// read up to 1 KiB for a preview (ignore errors while reading preview)
let _ = response.take(1024).read_to_end(&mut preview_bytes);
let preview = String::from_utf8_lossy(&preview_bytes);
bail!(
"HTTP error fetching {}: {}{}",
url,
status,
if preview.trim().is_empty() {
"".to_string()
} else {
format!(" — preview: {}", preview.trim())
}
);
}
let total_size = response.content_length().unwrap_or(0);
pb.set_length(total_size);
let mut file = File::create(&dest_path)
.with_context(|| format!("Failed to create: {}", dest_path.display()))?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = response.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
pb.finish_with_message("Download complete");
}
// Quick validation: ensure the downloaded file looks like the expected
// archive (detect obvious HTML error pages or wrong formats by magic).
// If validation returns an alternate URL (e.g. SourceForge mirror), follow
// and retry a few times.
let mut next_alt = validate_downloaded_archive(&dest_path, &filename, &url)?;
let mut seen_alts: HashSet<String> = HashSet::new();
let mut retries = 0usize;
while let Some(alt) = next_alt {
retries += 1;
if retries > MAX_MIRROR_RETRIES {
bail!(
"Exceeded mirror retry limit ({}) while fetching {}",
MAX_MIRROR_RETRIES,
url
);
}
if !seen_alts.insert(alt.clone()) {
bail!("Mirror retry loop detected for URL: {}", alt);
}
crate::log_info!("Retrying download from mirror: {}", alt);
pb.set_position(0);
pb.set_length(0);
fs::remove_file(&dest_path).ok();
// If mirror URL is FTP -> use suppaftp; otherwise use HTTP retry.
if let Ok(alt_url) = Url::parse(&alt) {
if alt_url.scheme() == "ftp" {
// FTP mirror retrieval
let host = alt_url.host_str().context("FTP mirror URL missing host")?;
let port = alt_url.port_or_known_default().unwrap_or(21);
let addr = format!("{}:{}", host, port);
let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str())
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
let user = if alt_url.username().is_empty() {
"anonymous"
} else {
alt_url.username()
};
let pass = alt_url.password().unwrap_or("anonymous@");
ftp_stream
.login(user, pass)
.with_context(|| format!("FTP login failed for {}", host))?;
let path = alt_url.path();
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
let mut retrieved = false;
for p in candidates.iter().filter(|s| !s.is_empty()) {
match ftp_stream.retr(
p,
|reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> {
let mut file = File::create(&dest_path)
.map_err(suppaftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = reader
.read(&mut buffer)
.map_err(suppaftp::FtpError::ConnectionError)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])
.map_err(suppaftp::FtpError::ConnectionError)?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
Ok(())
},
) {
Ok(_) => {
retrieved = true;
break;
}
Err(_) => continue,
}
}
ftp_stream.quit().ok();
if !retrieved {
bail!("FTP mirror error fetching {}", alt);
}
} else {
// HTTP(S) mirror retry (recreate client for retry)
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
let client = super::build_blocking_client(&ua, None)
.with_context(|| "Failed to build HTTP client")?;
let mut response = client
.get(&alt)
.send()
.with_context(|| format!("Failed to fetch mirror URL: {}", alt))?;
let status = response.status();
if !status.is_success() {
let mut preview_bytes = Vec::new();
let _ = response.take(1024).read_to_end(&mut preview_bytes);
let preview = String::from_utf8_lossy(&preview_bytes);
bail!(
"HTTP error fetching {}: {}{}",
alt,
status,
if preview.trim().is_empty() {
"".to_string()
} else {
format!(" — preview: {}", preview.trim())
}
);
match fetch_archive_from_candidate(candidate_url, &dest_path, &filename, &pb) {
Ok(()) => {
if !verify_checksum(&dest_path, &source.sha256)? {
fs::remove_file(&dest_path)?;
attempt_errors.push(format!(
"{}: checksum verification failed for {}",
candidate_url, filename
));
continue;
}
let mut file = File::create(&dest_path)
.with_context(|| format!("Failed to create: {}", dest_path.display()))?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = response.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
crate::log_info!("Checksum verified: {}", filename);
return Ok(dest_path);
}
Err(err) => {
attempt_errors.push(format!("{}: {err:#}", candidate_url));
fs::remove_file(&dest_path).ok();
}
} else {
// Fallback: unknown/malformed alt URL -> bail
bail!("Malformed mirror URL: {}", alt);
}
pb.finish_with_message("Download complete (mirror)");
next_alt = validate_downloaded_archive(&dest_path, &filename, &alt)?;
}
// Verify checksum
if !verify_checksum(&dest_path, &source.sha256)? {
fs::remove_file(&dest_path)?;
bail!("Checksum verification failed for {}", filename);
}
crate::log_info!("Checksum verified: {}", filename);
Ok(dest_path)
bail!(
"Failed to fetch {} from any candidate URL:\n{}",
filename,
attempt_errors.join("\n")
)
}
/// Verify checksum of a file supporting optional algorithm prefix.
@@ -338,6 +132,238 @@ pub(crate) fn derive_filename_from_url(url: &str) -> String {
format!("source-{}.download", &hex[..12])
}
fn archive_fetch_candidates(primary_url: &str) -> Vec<String> {
let mut candidates = Vec::new();
if matches!(
primary_url,
MUSL_1_2_6_SNAPSHOT_URL | MUSL_1_2_6_RELEASE_URL | MUSL_1_2_6_GENTOO_MIRROR_URL
) {
candidates.push(MUSL_1_2_6_GENTOO_MIRROR_URL.to_string());
}
if !candidates.iter().any(|url| url == primary_url) {
candidates.push(primary_url.to_string());
}
if let Some(mirror_url) = musl_release_mirror_url(primary_url)
&& !candidates.contains(&mirror_url)
{
candidates.push(mirror_url);
}
candidates
}
fn musl_release_mirror_url(url: &str) -> Option<String> {
let parsed = Url::parse(url).ok()?;
if parsed.host_str()? != "git.musl-libc.org" {
return None;
}
let mut segments = parsed.path_segments()?;
match (
segments.next(),
segments.next(),
segments.next(),
segments.next(),
segments.next(),
) {
(Some("cgit"), Some("musl"), Some("snapshot"), Some(filename), None)
if !filename.trim().is_empty() =>
{
Some(format!("https://musl.libc.org/releases/{filename}"))
}
_ => None,
}
}
fn fetch_archive_from_candidate(
candidate_url: &str,
dest_path: &Path,
filename: &str,
pb: &ProgressBar,
) -> Result<()> {
let mut current_url = candidate_url.to_string();
let mut seen_alts: HashSet<String> = HashSet::new();
let mut retries = 0usize;
loop {
download_archive_from_url(&current_url, dest_path, pb)?;
pb.finish_with_message("Download complete");
let Some(next_alt) = validate_downloaded_archive(dest_path, filename, &current_url)? else {
return Ok(());
};
retries += 1;
if retries > MAX_MIRROR_RETRIES {
bail!(
"Exceeded mirror retry limit ({}) while fetching {}",
MAX_MIRROR_RETRIES,
candidate_url
);
}
if !seen_alts.insert(next_alt.clone()) {
bail!("Mirror retry loop detected for URL: {}", next_alt);
}
crate::log_info!("Retrying download from mirror: {}", next_alt);
fs::remove_file(dest_path).ok();
pb.set_position(0);
pb.set_length(0);
current_url = next_alt;
}
}
fn download_archive_from_url(url: &str, dest_path: &Path, pb: &ProgressBar) -> Result<()> {
let parsed_url = Url::parse(url).with_context(|| format!("Invalid URL: {}", url))?;
if parsed_url.scheme() == "ftp" {
return download_ftp_archive(&parsed_url, dest_path, pb);
}
if !scheme_uses_http_transport(parsed_url.scheme()) {
bail!(
"Unsupported URL scheme for source fetch: {}",
parsed_url.scheme()
);
}
download_http_archive(url, dest_path, pb)
}
fn download_ftp_archive(parsed_url: &Url, dest_path: &Path, pb: &ProgressBar) -> Result<()> {
let host = parsed_url.host_str().context("FTP URL missing host")?;
let port = parsed_url.port_or_known_default().unwrap_or(21);
let addr = format!("{}:{}", host, port);
let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str())
.with_context(|| format!("Failed to connect to FTP host: {}", addr))?;
let user = if parsed_url.username().is_empty() {
"anonymous"
} else {
parsed_url.username()
};
let pass = parsed_url.password().unwrap_or("anonymous@");
ftp_stream
.login(user, pass)
.with_context(|| format!("FTP login failed for {}", host))?;
let path = parsed_url.path();
let candidates = [path.to_string(), path.trim_start_matches('/').to_string()];
let mut retrieved = false;
for candidate in candidates.iter().filter(|path| !path.is_empty()) {
match ftp_stream.retr(
candidate,
|reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> {
let mut file =
File::create(dest_path).map_err(suppaftp::FtpError::ConnectionError)?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = reader
.read(&mut buffer)
.map_err(suppaftp::FtpError::ConnectionError)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])
.map_err(suppaftp::FtpError::ConnectionError)?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
Ok(())
},
) {
Ok(_) => {
retrieved = true;
break;
}
Err(_) => continue,
}
}
ftp_stream.quit().ok();
if !retrieved {
bail!("FTP error fetching {}", parsed_url);
}
Ok(())
}
fn download_http_archive(url: &str, dest_path: &Path, pb: &ProgressBar) -> Result<()> {
let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
let mut last_err = None;
for attempt in 1..=HTTP_FETCH_RETRY_LIMIT {
match download_http_archive_once(url, dest_path, pb, &ua) {
Ok(()) => return Ok(()),
Err(err) if attempt < HTTP_FETCH_RETRY_LIMIT && is_transient_http_error(&err) => {
crate::log_info!(
"Fetch attempt {} for {} failed with a transient network error; retrying",
attempt,
url
);
fs::remove_file(dest_path).ok();
pb.set_position(0);
pb.set_length(0);
last_err = Some(err);
}
Err(err) => return Err(err),
}
}
Err(last_err.unwrap_or_else(|| anyhow::anyhow!("Failed to fetch: {}", url)))
}
fn download_http_archive_once(
url: &str,
dest_path: &Path,
pb: &ProgressBar,
ua: &str,
) -> Result<()> {
let client = super::build_blocking_client(ua, Some(HTTP_FETCH_TIMEOUT))
.with_context(|| "Failed to build HTTP client")?;
let mut response = client
.get(url)
.send()
.with_context(|| format!("Failed to fetch: {}", url))?;
let status = response.status();
if !status.is_success() {
let mut preview_bytes = Vec::new();
let _ = response.take(1024).read_to_end(&mut preview_bytes);
let preview = String::from_utf8_lossy(&preview_bytes);
bail!(
"HTTP error fetching {}: {}{}",
url,
status,
if preview.trim().is_empty() {
"".to_string()
} else {
format!(" — preview: {}", preview.trim())
}
);
}
let total_size = response.content_length().unwrap_or(0);
pb.set_length(total_size);
let mut file = File::create(dest_path)
.with_context(|| format!("Failed to create: {}", dest_path.display()))?;
let mut buffer = [0u8; 8192];
let mut downloaded = 0u64;
loop {
let bytes_read = response.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
file.write_all(&buffer[..bytes_read])?;
downloaded += bytes_read as u64;
pb.set_position(downloaded);
}
Ok(())
}
fn is_transient_http_error(err: &anyhow::Error) -> bool {
err.chain().any(|cause| {
cause
.downcast_ref::<reqwest::Error>()
.is_some_and(|inner| inner.is_timeout() || inner.is_connect() || inner.is_request())
})
}
/// Validate downloaded file's magic header to make sure it is the expected
/// archive format (avoids saving HTML pages or other unexpected content).
fn validate_downloaded_archive(
@@ -647,6 +673,45 @@ mod tests {
assert!(name.starts_with("source-") && name.ends_with(".download"));
}
#[test]
fn musl_snapshot_candidates_prefer_known_gentoo_mirror() {
let candidates = archive_fetch_candidates(MUSL_1_2_6_SNAPSHOT_URL);
assert_eq!(
candidates,
vec![
MUSL_1_2_6_GENTOO_MIRROR_URL.to_string(),
MUSL_1_2_6_SNAPSHOT_URL.to_string(),
"https://musl.libc.org/releases/musl-1.2.6.tar.gz".to_string(),
]
);
}
#[test]
fn musl_release_candidates_prefer_known_gentoo_mirror() {
let candidates = archive_fetch_candidates(MUSL_1_2_6_RELEASE_URL);
assert_eq!(
candidates,
vec![
MUSL_1_2_6_GENTOO_MIRROR_URL.to_string(),
MUSL_1_2_6_RELEASE_URL.to_string(),
]
);
}
#[test]
fn musl_release_mirror_only_matches_snapshot_urls() {
assert_eq!(
musl_release_mirror_url(
"https://git.musl-libc.org/cgit/musl/snapshot/musl-1.2.5.tar.gz"
),
Some("https://musl.libc.org/releases/musl-1.2.5.tar.gz".to_string())
);
assert_eq!(
musl_release_mirror_url("https://musl.libc.org/releases/musl-1.2.5.tar.gz"),
None
);
}
#[test]
fn sourceforge_html_no_link_falls_back_to_download_suffix() {
use std::io::Write;
+414 -21
View File
@@ -19,9 +19,12 @@ pub const INTERNAL_DEPOT_DIR: &str = ".depot";
pub const INTERNAL_OUTPUTS_DIR: &str = ".depot/outputs";
fn is_info_dir_index_path(rel_path: &str) -> bool {
matches!(rel_path, "usr/info/dir" | "usr/share/info/dir")
|| rel_path.starts_with("usr/info/dir.")
matches!(
rel_path,
"usr/info/dir" | "usr/share/info/dir" | "system/documentation/info/dir"
) || rel_path.starts_with("usr/info/dir.")
|| rel_path.starts_with("usr/share/info/dir.")
|| rel_path.starts_with("system/documentation/info/dir.")
}
fn is_purged_install_basename(rel_path: &str) -> bool {
@@ -43,6 +46,8 @@ fn is_skipped_install_path(rel_path: &str) -> bool {
|| p == INTERNAL_DEPOT_DIR
|| p.strip_prefix(INTERNAL_DEPOT_DIR)
.is_some_and(|rest| rest.starts_with('/'))
|| p == "destdir"
|| p.starts_with("destdir/")
|| p == "scripts"
|| p.starts_with("scripts/")
|| is_purged_payload_path(p)
@@ -340,12 +345,26 @@ fn move_tree_preserving_layout(src: &Path, dst: &Path) -> Result<()> {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create {}", parent.display()))?;
}
if dst.symlink_metadata().is_ok() {
anyhow::bail!(
"Failed to move {} into {}: destination already exists",
src.display(),
dst.display()
);
match dst.symlink_metadata() {
Ok(dst_metadata)
if duplicate_staged_path_is_equivalent(src, &metadata, dst, &dst_metadata)? =>
{
fs::remove_file(src).with_context(|| {
format!("Failed to remove duplicate staged path {}", src.display())
})?;
return Ok(());
}
Ok(_) => {
anyhow::bail!(
"Failed to move {} into {}: destination already exists",
src.display(),
dst.display()
);
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(err).with_context(|| format!("Failed to inspect {}", dst.display()));
}
}
fs::rename(src, dst).with_context(|| {
format!(
@@ -359,6 +378,59 @@ fn move_tree_preserving_layout(src: &Path, dst: &Path) -> Result<()> {
Ok(())
}
fn duplicate_staged_path_is_equivalent(
src: &Path,
src_metadata: &fs::Metadata,
dst: &Path,
dst_metadata: &fs::Metadata,
) -> Result<bool> {
let src_type = src_metadata.file_type();
let dst_type = dst_metadata.file_type();
if src_type.is_symlink() || dst_type.is_symlink() {
if !(src_type.is_symlink() && dst_type.is_symlink()) {
return Ok(false);
}
let src_target = fs::read_link(src)
.with_context(|| format!("Failed to read symlink {}", src.display()))?;
let dst_target = fs::read_link(dst)
.with_context(|| format!("Failed to read symlink {}", dst.display()))?;
return Ok(src_target == dst_target);
}
if !src_metadata.is_file() || !dst_metadata.is_file() {
return Ok(false);
}
if src_metadata.len() != dst_metadata.len() {
return Ok(false);
}
files_have_same_contents(src, dst)
}
fn files_have_same_contents(left: &Path, right: &Path) -> Result<bool> {
let mut left = fs::File::open(left)
.with_context(|| format!("Failed to open staged file {}", left.display()))?;
let mut right = fs::File::open(right)
.with_context(|| format!("Failed to open staged file {}", right.display()))?;
let mut left_buf = [0u8; 8192];
let mut right_buf = [0u8; 8192];
loop {
let left_read = left.read(&mut left_buf)?;
let right_read = right.read(&mut right_buf)?;
if left_read != right_read {
return Ok(false);
}
if left_read == 0 {
return Ok(true);
}
if left_buf[..left_read] != right_buf[..right_read] {
return Ok(false);
}
}
}
fn split_docs_for_output(
output_destdir: &Path,
docs_destdir: &Path,
@@ -829,9 +901,182 @@ pub fn process(destdir: &Path, spec: &PackageSpec) -> Result<()> {
);
}
let normalized = normalize_lbi_layout(destdir)?;
if normalized > 0 {
crate::log_info!("Normalized {} path(s) into the /system layout", normalized);
}
Ok(())
}
fn normalize_lbi_layout(destdir: &Path) -> Result<usize> {
let mut roots = vec![destdir.to_path_buf()];
let outputs = output_staging_root(destdir);
if outputs.exists() {
let mut output_dirs = fs::read_dir(&outputs)
.with_context(|| format!("Failed to read {}", outputs.display()))?
.collect::<std::result::Result<Vec<_>, _>>()
.with_context(|| {
format!(
"Failed to read output staging entry from {}",
outputs.display()
)
})?;
output_dirs.sort_by_key(|entry| entry.file_name());
for entry in output_dirs {
let path = entry.path();
if path.is_dir() {
roots.push(path);
}
}
}
let mut changed = 0usize;
for root in roots {
changed += normalize_lbi_tree_paths(&root)?;
changed += rewrite_lbi_pkgconfig_files(&root)?;
}
Ok(changed)
}
fn normalize_lbi_tree_paths(root: &Path) -> Result<usize> {
let mappings = [
("usr/share/man", "system/documentation/man-pages"),
("usr/share/info", "system/documentation/info"),
("usr/bin", "system/binaries"),
("usr/sbin", "system/systembinaries"),
("usr/lib64", "system/libraries"),
("usr/lib", "system/libraries"),
("usr/include", "system/headers"),
("usr/share", "system/share"),
("bin", "system/binaries"),
("sbin", "system/systembinaries"),
("lib64", "system/libraries"),
("lib", "system/libraries"),
("include", "system/headers"),
("etc", "system/configuration"),
("var", "system/variable"),
("system/bin", "system/binaries"),
("system/sbin", "system/systembinaries"),
("system/lib64", "system/libraries"),
("system/lib", "system/libraries"),
("system/include", "system/headers"),
("system/share/man", "system/documentation/man-pages"),
("system/share/info", "system/documentation/info"),
];
let mut changed = 0usize;
for (from, to) in mappings {
let src = root.join(from);
if !src.exists() {
continue;
}
let dst = root.join(to);
move_lbi_path(&src, &dst)?;
changed += 1;
}
prune_empty_dirs(root, &["usr", "system"])?;
Ok(changed)
}
fn move_lbi_path(src: &Path, dst: &Path) -> Result<()> {
if src == dst {
return Ok(());
}
if let Some(parent) = dst.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create {}", parent.display()))?;
}
if !dst.exists() {
fs::rename(src, dst)
.with_context(|| format!("Failed to move {} to {}", src.display(), dst.display()))?;
return Ok(());
}
let src_meta = fs::symlink_metadata(src)
.with_context(|| format!("Failed to inspect {}", src.display()))?;
let dst_meta = fs::symlink_metadata(dst)
.with_context(|| format!("Failed to inspect {}", dst.display()))?;
if src_meta.is_dir() && dst_meta.is_dir() {
move_directory_contents(src, dst)?;
return Ok(());
}
anyhow::bail!(
"Refusing to overwrite existing path while normalizing /system layout: {}",
dst.display()
);
}
fn prune_empty_dirs(root: &Path, dirs: &[&str]) -> Result<()> {
for dir in dirs {
let path = root.join(dir);
if path.exists() && path.is_dir() && is_directory_empty(&path)? {
fs::remove_dir(&path)
.with_context(|| format!("Failed to remove empty directory {}", path.display()))?;
}
}
Ok(())
}
fn rewrite_lbi_pkgconfig_files(root: &Path) -> Result<usize> {
let mut changed = 0usize;
if !root.exists() {
return Ok(0);
}
for entry in WalkDir::new(root).follow_links(false) {
let entry = entry.with_context(|| format!("Failed to walk {}", root.display()))?;
if !entry.file_type().is_file() {
continue;
}
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) != Some("pc") {
continue;
}
let Ok(original) = fs::read_to_string(path) else {
continue;
};
let rewritten = rewrite_lbi_pkgconfig_text(&original);
if rewritten != original {
fs::write(path, rewritten)
.with_context(|| format!("Failed to rewrite pkg-config file {}", path.display()))?;
changed += 1;
}
}
Ok(changed)
}
fn rewrite_lbi_pkgconfig_text(input: &str) -> String {
input
.replace("prefix=/usr", "prefix=/system")
.replace("exec_prefix=/usr", "exec_prefix=/system")
.replace("bindir=${prefix}/bin", "bindir=${prefix}/binaries")
.replace("sbindir=${prefix}/sbin", "sbindir=${prefix}/systembinaries")
.replace("libdir=${prefix}/lib64", "libdir=${prefix}/libraries")
.replace("libdir=${prefix}/lib", "libdir=${prefix}/libraries")
.replace(
"includedir=${prefix}/include",
"includedir=${prefix}/headers",
)
.replace(
"mandir=${prefix}/share/man",
"mandir=${prefix}/documentation/man-pages",
)
.replace(
"infodir=${prefix}/share/info",
"infodir=${prefix}/documentation/info",
)
.replace("/usr/sbin", "/system/systembinaries")
.replace("/usr/bin", "/system/binaries")
.replace("/usr/lib64", "/system/libraries")
.replace("/usr/lib", "/system/libraries")
.replace("/usr/include", "/system/headers")
.replace("/usr/share/man", "/system/documentation/man-pages")
.replace("/usr/share/info", "/system/documentation/info")
.replace("/usr/share", "/system/share")
}
/// Copy license files into the staged tree.
///
/// Copies common license file patterns from the source directory root into:
@@ -1539,9 +1784,9 @@ mod tests {
let spec = mk_spec_for_stage_processing();
process(&destdir, &spec).unwrap();
assert!(!destdir.join("usr/lib/libfoo.a").exists());
assert!(!destdir.join("usr/lib/libfoo.la").exists());
assert!(destdir.join("usr/lib/libfoo.so").exists());
assert!(!destdir.join("system/libraries/libfoo.a").exists());
assert!(!destdir.join("system/libraries/libfoo.la").exists());
assert!(destdir.join("system/libraries/libfoo.so").exists());
}
#[test]
@@ -1556,8 +1801,8 @@ mod tests {
spec.build.flags.no_delete_static = true;
process(&destdir, &spec).unwrap();
assert!(destdir.join("usr/lib/libfoo.a").exists());
assert!(!destdir.join("usr/lib/libfoo.la").exists());
assert!(destdir.join("system/libraries/libfoo.a").exists());
assert!(!destdir.join("system/libraries/libfoo.la").exists());
}
#[test]
@@ -1580,18 +1825,18 @@ mod tests {
process(&destdir, &spec).unwrap();
let docs_destdir = output_staging_dir(&destdir, "foo-docs");
assert!(docs_destdir.join("usr/share/doc/foo/README").exists());
assert!(docs_destdir.join("system/share/doc/foo/README").exists());
assert!(
docs_destdir
.join("usr/share/gtk-doc/html/foo/index.html")
.join("system/share/gtk-doc/html/foo/index.html")
.exists()
);
assert!(docs_destdir.join("opt/foo-docs/guide.txt").exists());
assert!(destdir.join("usr/bin/foo").exists());
assert!(!destdir.join("usr/share/doc/foo/README").exists());
assert!(destdir.join("system/binaries/foo").exists());
assert!(!destdir.join("system/share/doc/foo/README").exists());
assert!(
!destdir
.join("usr/share/gtk-doc/html/foo/index.html")
.join("system/share/gtk-doc/html/foo/index.html")
.exists()
);
assert!(!destdir.join("opt/foo-docs/guide.txt").exists());
@@ -1623,9 +1868,129 @@ mod tests {
process(&destdir, &spec).unwrap();
let docs_destdir = output_staging_dir(&destdir, "foo-dev-docs");
assert!(docs_destdir.join("usr/share/doc/foo-dev/README").exists());
assert!(dev_destdir.join("usr/include/foo.h").exists());
assert!(!dev_destdir.join("usr/share/doc/foo-dev/README").exists());
assert!(
docs_destdir
.join("system/share/doc/foo-dev/README")
.exists()
);
assert!(dev_destdir.join("system/headers/foo.h").exists());
assert!(!dev_destdir.join("system/share/doc/foo-dev/README").exists());
}
#[test]
fn process_normalizes_usr_paths_into_system_layout() {
let tmp = tempfile::tempdir().unwrap();
let destdir = tmp.path().join("dest");
std::fs::create_dir_all(destdir.join("usr/bin")).unwrap();
std::fs::create_dir_all(destdir.join("usr/lib/pkgconfig")).unwrap();
std::fs::create_dir_all(destdir.join("usr/include/foo")).unwrap();
std::fs::create_dir_all(destdir.join("usr/share/man/man1")).unwrap();
std::fs::create_dir_all(destdir.join("usr/share/info")).unwrap();
std::fs::write(destdir.join("usr/bin/foo"), "bin").unwrap();
std::fs::write(destdir.join("usr/share/man/man1/foo.1"), "man").unwrap();
std::fs::write(destdir.join("usr/share/info/foo.info"), "info").unwrap();
std::fs::write(
destdir.join("usr/lib/pkgconfig/foo.pc"),
"prefix=/usr\nbindir=${prefix}/bin\nlibdir=${prefix}/lib\nincludedir=${prefix}/include\nmandir=${prefix}/share/man\ninfodir=${prefix}/share/info\n",
)
.unwrap();
let spec = mk_spec_for_stage_processing();
process(&destdir, &spec).unwrap();
assert!(destdir.join("system/binaries/foo").exists());
assert!(destdir.join("system/headers/foo").is_dir());
assert!(
destdir
.join("system/documentation/man-pages/man1/foo.1")
.exists()
);
assert!(destdir.join("system/documentation/info/foo.info").exists());
let pc =
std::fs::read_to_string(destdir.join("system/libraries/pkgconfig/foo.pc")).unwrap();
assert!(pc.contains("prefix=/system"));
assert!(pc.contains("bindir=${prefix}/binaries"));
assert!(pc.contains("libdir=${prefix}/libraries"));
assert!(pc.contains("includedir=${prefix}/headers"));
assert!(pc.contains("mandir=${prefix}/documentation/man-pages"));
assert!(pc.contains("infodir=${prefix}/documentation/info"));
assert!(!destdir.join("usr").exists());
}
#[test]
fn process_normalizes_duplicate_identical_lbi_paths() {
let tmp = tempfile::tempdir().unwrap();
let destdir = tmp.path().join("dest");
std::fs::create_dir_all(destdir.join("system/lib/clang/22/include")).unwrap();
std::fs::create_dir_all(destdir.join("system/libraries/clang/22/include")).unwrap();
std::fs::write(
destdir.join("system/lib/clang/22/include/builtins.h"),
"same header\n",
)
.unwrap();
std::fs::write(
destdir.join("system/libraries/clang/22/include/builtins.h"),
"same header\n",
)
.unwrap();
let spec = mk_spec_for_stage_processing();
process(&destdir, &spec).unwrap();
assert!(
!destdir
.join("system/lib/clang/22/include/builtins.h")
.exists()
);
assert_eq!(
std::fs::read_to_string(destdir.join("system/libraries/clang/22/include/builtins.h"))
.unwrap(),
"same header\n"
);
}
#[test]
fn process_rejects_conflicting_lbi_normalized_paths() {
let tmp = tempfile::tempdir().unwrap();
let destdir = tmp.path().join("dest");
std::fs::create_dir_all(destdir.join("system/lib/clang/22/include")).unwrap();
std::fs::create_dir_all(destdir.join("system/libraries/clang/22/include")).unwrap();
std::fs::write(
destdir.join("system/lib/clang/22/include/builtins.h"),
"left header\n",
)
.unwrap();
std::fs::write(
destdir.join("system/libraries/clang/22/include/builtins.h"),
"right header\n",
)
.unwrap();
let spec = mk_spec_for_stage_processing();
let err = process(&destdir, &spec).unwrap_err();
assert!(
err.to_string().contains("destination already exists"),
"{err:#}"
);
}
#[test]
fn process_normalizes_split_output_usr_paths() {
let tmp = tempfile::tempdir().unwrap();
let destdir = tmp.path().join("dest");
let out = destdir.join(".depot/outputs/foo-devel/usr/lib/pkgconfig");
std::fs::create_dir_all(&out).unwrap();
std::fs::write(out.join("foo.pc"), "prefix=/usr\nlibdir=/usr/lib\n").unwrap();
let spec = mk_spec_for_stage_processing();
process(&destdir, &spec).unwrap();
let pc_path = destdir.join(".depot/outputs/foo-devel/system/libraries/pkgconfig/foo.pc");
assert!(pc_path.exists());
let pc = std::fs::read_to_string(pc_path).unwrap();
assert!(pc.contains("prefix=/system"));
assert!(pc.contains("libdir=/system/libraries"));
}
#[test]
@@ -2428,6 +2793,34 @@ mod tests {
.contains(&".depot/outputs/clang/usr/bin/clang".to_string())
);
}
#[test]
fn generate_manifest_skips_bootstrap_chroot_destdir_mountpoint() {
let tmp = tempfile::tempdir().unwrap();
let destdir = tmp.path().join("dest");
std::fs::create_dir_all(destdir.join("system/binaries")).unwrap();
std::fs::create_dir_all(destdir.join("destdir/system/documentation/xz")).unwrap();
std::fs::write(destdir.join("system/binaries/xz"), "ok").unwrap();
std::fs::write(
destdir.join("destdir/system/documentation/xz/README"),
"stale staging",
)
.unwrap();
let manifest = generate_manifest_with_dirs(&destdir).unwrap();
assert!(manifest.files.contains(&"system/binaries/xz".to_string()));
assert!(
!manifest
.files
.contains(&"destdir/system/documentation/xz/README".to_string())
);
assert!(
!manifest
.directories
.contains(&"destdir/system/documentation/xz".to_string())
);
}
}
/// Manifest containing files and directories for a package
+504
View File
@@ -0,0 +1,504 @@
use crate::config::Config;
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Path, PathBuf};
const STATE_FILENAME: &str = "system.toml";
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub(crate) struct SystemState {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) stage: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) target: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) arch: Option<String>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub(crate) layers: BTreeMap<String, Vec<String>>,
}
pub(crate) fn state_path(config: &Config) -> PathBuf {
config.db_dir.join(STATE_FILENAME)
}
pub(crate) fn load(config: &Config) -> Result<SystemState> {
let path = state_path(config);
if !path.exists() {
return Ok(SystemState::default());
}
let content =
fs::read_to_string(&path).with_context(|| format!("Failed to read {}", path.display()))?;
toml::from_str(&content).with_context(|| format!("Failed to parse {}", path.display()))
}
pub(crate) fn save(config: &Config, state: &SystemState) -> Result<PathBuf> {
let path = state_path(config);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create {}", parent.display()))?;
}
let content = toml::to_string_pretty(state).context("Failed to serialize system state")?;
fs::write(&path, content).with_context(|| format!("Failed to write {}", path.display()))?;
Ok(path)
}
pub(crate) fn set_stage(config: &Config, stage: String) -> Result<SystemState> {
let stage = normalize_token("stage", &stage)?;
let mut state = load(config)?;
state.stage = Some(stage);
save(config, &state)?;
Ok(state)
}
pub(crate) fn add_packages_to_layer(
config: &Config,
layer: String,
packages: &[String],
) -> Result<SystemState> {
let layer = normalize_token("layer", &layer)?;
let mut state = load(config)?;
let existing = state.layers.remove(&layer).unwrap_or_default();
let mut merged = existing.into_iter().collect::<BTreeSet<_>>();
for package in packages {
merged.insert(normalize_token("package", package)?);
}
state.layers.insert(layer, merged.into_iter().collect());
save(config, &state)?;
Ok(state)
}
pub(crate) fn set_layer_packages(
config: &Config,
layer: String,
packages: &[String],
) -> Result<SystemState> {
let layer = normalize_token("layer", &layer)?;
let mut state = load(config)?;
let mut seen = BTreeSet::new();
let mut normalized = Vec::new();
for package in packages {
let package = normalize_token("package", package)?;
if seen.insert(package.clone()) {
normalized.push(package);
}
}
state.layers.insert(layer, normalized);
save(config, &state)?;
Ok(state)
}
pub(crate) fn remove_packages_from_layer(
config: &Config,
layer: String,
packages: &[String],
) -> Result<SystemState> {
let layer = normalize_token("layer", &layer)?;
let remove = packages
.iter()
.map(|package| normalize_token("package", package))
.collect::<Result<BTreeSet<_>>>()?;
let mut state = load(config)?;
if let Some(existing) = state.layers.get_mut(&layer) {
existing.retain(|package| !remove.contains(package));
if existing.is_empty() {
state.layers.remove(&layer);
}
}
save(config, &state)?;
Ok(state)
}
pub(crate) fn init_lbi_layout(
rootfs: &Path,
config: &Config,
target: &str,
arch: Option<&str>,
force: bool,
) -> Result<SystemState> {
let target = normalize_token("target", target)?;
let arch = match arch {
Some(arch) => normalize_token("arch", arch)?,
None => crate::cross::target_arch_from_triple(&target).to_string(),
};
ensure_lbi_layout_paths(rootfs)?;
write_lbi_build_config(rootfs, &target, &arch, force)?;
let mut state = load(config)?;
state.target = Some(target);
state.arch = Some(arch);
state.stage.get_or_insert_with(|| "layout".to_string());
save(config, &state)?;
Ok(state)
}
pub(crate) fn ensure_lbi_layout_paths(rootfs: &Path) -> Result<()> {
create_lbi_directories(rootfs)?;
create_lbi_links(rootfs)?;
Ok(())
}
fn normalize_token(kind: &str, value: &str) -> Result<String> {
let trimmed = value.trim();
if trimmed.is_empty() {
bail!("{kind} must not be empty");
}
if trimmed.contains('/') || trimmed.contains('\0') {
bail!("{kind} must not contain '/' or NUL bytes: {trimmed}");
}
Ok(trimmed.to_string())
}
fn create_lbi_directories(rootfs: &Path) -> Result<()> {
let dirs = [
"system",
"system/configuration",
"system/binaries",
"system/systembinaries",
"system/libraries",
"system/headers",
"system/share",
"system/documentation",
"system/documentation/man-pages",
"system/documentation/info",
"system/tools",
"system/variable",
"system/variable/lib",
"system/users",
"system/charlie",
"system/devices",
"system/devices/pts",
"system/devices/shm",
"system/processes",
"system/run",
"system/system",
"system/temporary",
"usr",
];
for dir in dirs {
let path = rootfs.join(dir);
fs::create_dir_all(&path)
.with_context(|| format!("Failed to create {}", path.display()))?;
}
Ok(())
}
fn create_lbi_links(rootfs: &Path) -> Result<()> {
let links = [
("etc", "system/configuration"),
("bin", "system/binaries"),
("sbin", "system/systembinaries"),
("lib", "system/libraries"),
("var", "system/variable"),
("home", "system/users"),
("root", "system/charlie"),
("dev", "system/devices"),
("proc", "system/processes"),
("run", "system/run"),
("sys", "system/system"),
("usr/bin", "../system/binaries"),
("usr/sbin", "../system/systembinaries"),
("usr/lib", "../system/libraries"),
("usr/include", "../system/headers"),
("usr/share", "../system/share"),
("system/lib", "libraries"),
];
for (link, target) in links {
ensure_relative_symlink(rootfs, Path::new(link), Path::new(target))?;
}
Ok(())
}
#[cfg(unix)]
fn ensure_relative_symlink(rootfs: &Path, link: &Path, target: &Path) -> Result<()> {
use std::os::unix::fs as unix_fs;
let link_path = rootfs.join(link);
if let Ok(metadata) = fs::symlink_metadata(&link_path) {
if !metadata.file_type().is_symlink() {
if metadata.is_dir() {
let target_path = link_path
.parent()
.unwrap_or(rootfs)
.join(target)
.components()
.collect::<PathBuf>();
merge_dir_contents(&link_path, &target_path).with_context(|| {
format!(
"Failed to merge existing directory {} into {}",
link_path.display(),
target_path.display()
)
})?;
fs::remove_dir(&link_path)
.with_context(|| format!("Failed to remove {}", link_path.display()))?;
} else {
bail!(
"Refusing to replace non-directory path while creating LBI layout: {}",
link_path.display()
);
}
} else {
let existing = fs::read_link(&link_path)
.with_context(|| format!("Failed to read symlink {}", link_path.display()))?;
if existing == target {
return Ok(());
}
fs::remove_file(&link_path)
.with_context(|| format!("Failed to replace symlink {}", link_path.display()))?;
}
}
if let Some(parent) = link_path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create {}", parent.display()))?;
}
unix_fs::symlink(target, &link_path)
.with_context(|| format!("Failed to create symlink {}", link_path.display()))?;
Ok(())
}
fn merge_dir_contents(src: &Path, dest: &Path) -> Result<()> {
fs::create_dir_all(dest).with_context(|| format!("Failed to create {}", dest.display()))?;
let mut entries = fs::read_dir(src)
.with_context(|| format!("Failed to read {}", src.display()))?
.collect::<std::result::Result<Vec<_>, _>>()
.with_context(|| format!("Failed to read entry from {}", src.display()))?;
entries.sort_by_key(|entry| entry.file_name());
for entry in entries {
let source_path = entry.path();
let dest_path = dest.join(entry.file_name());
if dest_path.exists() {
let source_type = entry
.file_type()
.with_context(|| format!("Failed to inspect {}", source_path.display()))?;
let dest_type = fs::symlink_metadata(&dest_path)
.with_context(|| format!("Failed to inspect {}", dest_path.display()))?
.file_type();
if source_type.is_dir() && dest_type.is_dir() {
merge_dir_contents(&source_path, &dest_path)?;
fs::remove_dir(&source_path)
.with_context(|| format!("Failed to remove {}", source_path.display()))?;
continue;
}
bail!(
"Refusing to overwrite existing path while merging LBI directory: {}",
dest_path.display()
);
}
fs::rename(&source_path, &dest_path).with_context(|| {
format!(
"Failed to move {} to {}",
source_path.display(),
dest_path.display()
)
})?;
}
Ok(())
}
#[cfg(not(unix))]
fn ensure_relative_symlink(_rootfs: &Path, _link: &Path, _target: &Path) -> Result<()> {
bail!("LBI layout initialization requires Unix symlink support")
}
fn write_lbi_build_config(rootfs: &Path, target: &str, arch: &str, force: bool) -> Result<PathBuf> {
let path = rootfs.join("etc/depot.d/build.toml");
if path.exists() && !force {
bail!(
"{} already exists; re-run with --force to replace it",
path.display()
);
}
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("Failed to create {}", parent.display()))?;
}
let content = lbi_build_config_toml(target, arch);
fs::write(&path, content).with_context(|| format!("Failed to write {}", path.display()))?;
Ok(path)
}
fn lbi_build_config_toml(target: &str, arch: &str) -> String {
let makeflags = format!(
"-j{}",
std::thread::available_parallelism()
.map(|parallelism| parallelism.get())
.unwrap_or(1)
);
format!(
r#"# Generated by `depot system init-lbi`.
# These defaults mirror the Linux by Intent /system layout.
[flags]
prefix = "/system"
bindir = "/system/binaries"
sbindir = "/system/systembinaries"
libdir = "/system/libraries"
libexecdir = "/system/libraries"
sysconfdir = "/system/configuration"
localstatedir = "/system/variable"
sharedstatedir = "/system/variable/lib"
includedir = "/system/headers"
datarootdir = "/system/share"
datadir = "/system/share"
mandir = "/system/documentation/man-pages"
infodir = "/system/documentation/info"
cc = "clang"
cxx = "clang++"
ar = "llvm-ar"
ld = "ld.lld"
carch = "{arch}"
chost = "{target}"
target = "{target}"
cflags = ["-O2", "-pipe"]
cxxflags = ["-O2", "-pipe"]
ldflags = ["-Wl,-rpath,/system/libraries"]
makeflags = "{makeflags}"
"#,
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn layer_add_sorts_and_deduplicates_packages() {
let tmp = tempfile::tempdir().unwrap();
let config = Config::for_rootfs(tmp.path());
let state = add_packages_to_layer(
&config,
"base".to_string(),
&["zlib".to_string(), "musl".to_string(), "zlib".to_string()],
)
.unwrap();
assert_eq!(state.layers["base"], vec!["musl", "zlib"]);
}
#[test]
fn set_layer_packages_replaces_only_named_layer() {
let tmp = tempfile::tempdir().unwrap();
let config = Config::for_rootfs(tmp.path());
add_packages_to_layer(&config, "custom".to_string(), &["kept".to_string()]).unwrap();
add_packages_to_layer(
&config,
"base".to_string(),
&["old".to_string(), "zlib".to_string()],
)
.unwrap();
let state = set_layer_packages(
&config,
"base".to_string(),
&["zlib".to_string(), "musl".to_string(), "zlib".to_string()],
)
.unwrap();
assert_eq!(state.layers["base"], vec!["zlib", "musl"]);
assert_eq!(state.layers["custom"], vec!["kept"]);
}
#[test]
fn stage_is_persisted() {
let tmp = tempfile::tempdir().unwrap();
let config = Config::for_rootfs(tmp.path());
set_stage(&config, "cross-tools".to_string()).unwrap();
let loaded = load(&config).unwrap();
assert_eq!(loaded.stage.as_deref(), Some("cross-tools"));
}
#[cfg(unix)]
#[test]
fn init_lbi_creates_layout_and_build_defaults() {
let tmp = tempfile::tempdir().unwrap();
let config = Config::for_rootfs(tmp.path());
let state = init_lbi_layout(
tmp.path(),
&config,
"x86_64-unknown-linux-musl",
None,
false,
)
.unwrap();
assert_eq!(state.stage.as_deref(), Some("layout"));
assert_eq!(state.arch.as_deref(), Some("x86_64"));
assert!(tmp.path().join("system/binaries").is_dir());
assert!(tmp.path().join("system/devices/pts").is_dir());
assert!(tmp.path().join("system/devices/shm").is_dir());
assert_eq!(
fs::read_link(tmp.path().join("usr/bin")).unwrap(),
PathBuf::from("../system/binaries")
);
assert_eq!(
fs::read_link(tmp.path().join("dev")).unwrap(),
PathBuf::from("system/devices")
);
assert_eq!(
fs::read_link(tmp.path().join("proc")).unwrap(),
PathBuf::from("system/processes")
);
assert_eq!(
fs::read_link(tmp.path().join("sys")).unwrap(),
PathBuf::from("system/system")
);
assert_eq!(
fs::read_link(tmp.path().join("run")).unwrap(),
PathBuf::from("system/run")
);
let build_toml = fs::read_to_string(tmp.path().join("etc/depot.d/build.toml")).unwrap();
assert!(build_toml.contains("prefix = \"/system\""));
assert!(build_toml.contains("chost = \"x86_64-unknown-linux-musl\""));
let makeflags_line = build_toml
.lines()
.find(|line| line.trim_start().starts_with("makeflags = \"-j"))
.expect("expected makeflags default in generated build.toml");
let jobs = makeflags_line
.trim()
.trim_start_matches("makeflags = \"-j")
.trim_end_matches('"')
.parse::<usize>()
.expect("expected numeric makeflags job count");
assert!(jobs >= 1);
}
#[cfg(unix)]
#[test]
fn ensure_lbi_layout_paths_reconciles_existing_usr_include_directory() {
let tmp = tempfile::tempdir().unwrap();
fs::create_dir_all(tmp.path().join("usr/include")).unwrap();
fs::write(tmp.path().join("usr/include/marker.h"), "/* marker */").unwrap();
ensure_lbi_layout_paths(tmp.path()).unwrap();
assert_eq!(
fs::read_link(tmp.path().join("usr/include")).unwrap(),
PathBuf::from("../system/headers")
);
assert!(tmp.path().join("system/headers/marker.h").exists());
}
#[cfg(unix)]
#[test]
fn init_lbi_migrates_existing_var_contents_before_linking() {
let tmp = tempfile::tempdir().unwrap();
fs::create_dir_all(tmp.path().join("var/lib/depot")).unwrap();
fs::write(tmp.path().join("var/lib/depot/lock"), "").unwrap();
let config = Config::for_rootfs(tmp.path());
init_lbi_layout(
tmp.path(),
&config,
"x86_64-unknown-linux-musl",
None,
false,
)
.unwrap();
assert_eq!(
fs::read_link(tmp.path().join("var")).unwrap(),
PathBuf::from("system/variable")
);
assert!(tmp.path().join("system/variable/lib/depot/lock").exists());
}
}
+10
View File
@@ -50,6 +50,16 @@ pub fn success(message: impl AsRef<str>) {
println!("{} {}", label(Stream::Stdout, "OK", "32"), message.as_ref());
}
pub fn merge_package(layer: &str, package: &str) {
println!(
"{} {} {} into layer {}",
paint(Stream::Stdout, ">>>", "32;1"),
paint(Stream::Stdout, "merging package", "36;1"),
paint(Stream::Stdout, package, "32;1"),
layer
);
}
pub fn warn(message: impl AsRef<str>) {
eprintln!(
"{} {}",