diff --git a/Cargo.lock b/Cargo.lock index c780e53..7af35fa 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -418,7 +418,7 @@ checksum = "807800ff3288b621186fe0a8f3392c4652068257302709c24efd918c3dffcdc2" [[package]] name = "depot" -version = "0.32.0" +version = "0.32.1" dependencies = [ "anyhow", "ar", diff --git a/Cargo.toml b/Cargo.toml index d0b2971..14730ab 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "depot" -version = "0.32.0" +version = "0.32.1" edition = "2024" [lints.rust] diff --git a/meson.build b/meson.build index dece09b..e17a72e 100644 --- a/meson.build +++ b/meson.build @@ -1,6 +1,6 @@ project( 'depot', - version: '0.32.0', + version: '0.32.1', meson_version: '>=0.60.0', default_options: ['buildtype=release'], ) diff --git a/src/cli.rs b/src/cli.rs index 86ee9c7..b489586 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -314,6 +314,8 @@ pub enum InternalCommands { #[arg(long, default_value = "/usr")] prefix: String, }, + #[command(hide = true)] + Clone { repo: String, dest: Option }, } #[derive(Debug, Clone, Args)] diff --git a/src/commands.rs b/src/commands.rs index 08f80c8..6400953 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -640,6 +640,36 @@ fn run_internal_command(command: InternalCommands) -> Result<()> { &env_vars, ) } + InternalCommands::Clone { repo, dest } => { + let (base, rev) = crate::source::split_git_url(&repo).with_context(|| { + format!("Unsupported repository URL for internal clone: {}", repo) + })?; + let dest = if let Some(dest) = dest { + dest + } else { + let cwd = + std::env::current_dir().context("Failed to determine current directory")?; + cwd.join(crate::source::git_default_checkout_dir_name(&base)) + }; + if dest.exists() { + anyhow::bail!("Clone destination already exists: {}", dest.display()); + } + let cache_root = + tempfile::tempdir().context("Failed to create temporary git cache for clone")?; + let label = dest + .file_name() + .and_then(|name| name.to_str()) + .filter(|name| !name.is_empty()) + .unwrap_or("clone"); + crate::source::git_checkout( + &base, + &rev, + &dest, + &cache_root.path().join("git"), + label, + &[], + ) + } } } @@ -5974,6 +6004,8 @@ mod tests { UpdateArgs, }; use crate::test_support::TestEnv; + use git2::{Oid, Repository}; + use std::path::Path; use std::sync::Mutex; static ASSUME_YES_TEST_LOCK: Mutex<()> = Mutex::new(()); @@ -6071,6 +6103,74 @@ mod tests { } } + fn commit_git_file(repo: &Repository, workdir: &Path, rel: &str, data: &str) -> Oid { + let full_path = workdir.join(rel); + if let Some(parent) = full_path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&full_path, data).unwrap(); + + let mut index = repo.index().unwrap(); + index.add_path(Path::new(rel)).unwrap(); + let tree_id = index.write_tree().unwrap(); + let tree = repo.find_tree(tree_id).unwrap(); + let sig = git2::Signature::now("depot-test", "depot@example.test").unwrap(); + let mut parents = Vec::new(); + if let Ok(head) = repo.head() + && let Some(oid) = head.target() + { + parents.push(repo.find_commit(oid).unwrap()); + } + let parent_refs: Vec<&git2::Commit<'_>> = parents.iter().collect(); + + repo.commit(Some("HEAD"), &sig, &sig, "test", &tree, &parent_refs) + .unwrap() + } + + fn make_remote_git_repo() -> (tempfile::TempDir, String, Oid) { + let tmp = tempfile::tempdir().unwrap(); + let remote_dir = tmp.path().join("origin.git"); + let workdir = tmp.path().join("work"); + + Repository::init_bare(&remote_dir).unwrap(); + let repo = Repository::init(&workdir).unwrap(); + let tagged = commit_git_file(&repo, &workdir, "README", "tagged\n"); + let tag_target = repo.find_object(tagged, None).unwrap(); + repo.tag_lightweight("v1.0.0", &tag_target, false).unwrap(); + + let branch_ref = repo.head().unwrap().name().unwrap().to_string(); + let mut remote = repo.remote("origin", remote_dir.to_str().unwrap()).unwrap(); + let push_specs = [ + format!("{branch_ref}:{branch_ref}"), + "refs/tags/v1.0.0:refs/tags/v1.0.0".to_string(), + ]; + let push_spec_refs: Vec<&String> = push_specs.iter().collect(); + remote.push(&push_spec_refs, None).unwrap(); + + let remote_url = url::Url::from_file_path(&remote_dir).unwrap().to_string(); + (tmp, remote_url, tagged) + } + + #[test] + fn run_internal_clone_checks_out_git_revision() { + let (_tmp, remote_url, tagged) = make_remote_git_repo(); + let clone_root = tempfile::tempdir().unwrap(); + let dest = clone_root.path().join("cloned-src"); + + run_internal_command(InternalCommands::Clone { + repo: format!("{remote_url}#v1.0.0"), + dest: Some(dest.clone()), + }) + .unwrap(); + + let repo = Repository::open(&dest).unwrap(); + assert_eq!(repo.head().unwrap().target().unwrap(), tagged); + assert_eq!( + std::fs::read_to_string(dest.join("README")).unwrap(), + "tagged\n" + ); + } + fn register_installed_test_package( config: &config::Config, rootfs: &Path, diff --git a/src/package/interactive.rs b/src/package/interactive.rs index a7bff95..48c7fed 100644 --- a/src/package/interactive.rs +++ b/src/package/interactive.rs @@ -183,7 +183,7 @@ fn prompt_manual_sources() -> Result> { } else { prompt_repeating_list( "Manual source URL", - "Supports http(s), ftp, and file:// URLs", + "Supports http(s), ftp, file://, and git URLs like ...repo.git#rev", )? }; diff --git a/src/package/spec.rs b/src/package/spec.rs index 4e70e2e..5db92c6 100755 --- a/src/package/spec.rs +++ b/src/package/spec.rs @@ -3973,17 +3973,18 @@ pub struct ManualSource { /// Multiple filenames in the spec directory (local manual source mode). #[serde(default)] pub files: Vec, - /// Remote URL to fetch (remote manual source mode). + /// Remote URL to fetch or clone (remote manual source mode). #[serde(default)] pub url: Option, - /// Multiple remote URLs to fetch (remote manual source mode). + /// Multiple remote URLs to fetch or clone (remote manual source mode). #[serde(default)] pub urls: Vec, /// Checksum (optional, use "skip" to bypass verification). #[serde(default)] pub sha256: Option, /// Destination path relative to build work directory. - /// Defaults to `file` for local mode or a derived filename for URL mode. + /// Defaults to `file` for local mode, a derived filename for archive URLs, + /// or the repository directory name for git URLs. #[serde(default)] pub dest: Option, } diff --git a/src/shell_helpers.rs b/src/shell_helpers.rs index 7564eb0..c36c3e9 100644 --- a/src/shell_helpers.rs +++ b/src/shell_helpers.rs @@ -14,6 +14,7 @@ const DEPOT_HAUL_HELPER_ENV: &str = "DEPOT_HAUL_HELPER"; const DEPOT_SUBDESTDIR_HELPER_ENV: &str = "DEPOT_SUBDESTDIR_HELPER"; const DEPOT_PYTHON_BUILD_HELPER_ENV: &str = "DEPOT_PYTHON_BUILD_HELPER"; const DEPOT_PYTHON_INSTALL_HELPER_ENV: &str = "DEPOT_PYTHON_INSTALL_HELPER"; +const DEPOT_CLONE_HELPER_ENV: &str = "DEPOT_CLONE_HELPER"; const DEPOT_EXECUTABLE_ENV: &str = "DEPOT_EXECUTABLE"; /// Ephemeral helper command directory to prepend to PATH while running scripts. @@ -26,6 +27,7 @@ pub struct ShellHelpers { subdestdir_path: PathBuf, python_build_path: PathBuf, python_install_path: PathBuf, + clone_path: PathBuf, } impl ShellHelpers { @@ -123,6 +125,24 @@ impl ShellHelpers { })?; } + let clone_path = bin_dir.join("depot_clone"); + fs::write(&clone_path, CLONE_SCRIPT).with_context(|| { + format!( + "Failed to write shell helper command: {}", + clone_path.display() + ) + })?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&clone_path) + .with_context(|| format!("Failed to stat helper: {}", clone_path.display()))? + .permissions(); + perms.set_mode(0o755); + fs::set_permissions(&clone_path, perms) + .with_context(|| format!("Failed to chmod helper: {}", clone_path.display()))?; + } + let path_value = crate::runtime_env::prepend_helper_to_safe_path(&bin_dir); Ok(Self { @@ -134,6 +154,7 @@ impl ShellHelpers { subdestdir_path, python_build_path, python_install_path, + clone_path, }) } @@ -166,6 +187,11 @@ impl ShellHelpers { DEPOT_PYTHON_INSTALL_HELPER_ENV, self.python_install_path.to_string_lossy().into_owned(), ); + set_env_var( + env_vars, + DEPOT_CLONE_HELPER_ENV, + self.clone_path.to_string_lossy().into_owned(), + ); set_env_var( env_vars, DEPOT_EXECUTABLE_ENV, @@ -178,7 +204,7 @@ impl ShellHelpers { /// through `/bin/sh`, avoiding direct execution from mounts that may be `noexec`. pub fn wrap_shell_command(command: &str) -> String { format!( - "haul() {{ /bin/sh \"${{{DEPOT_HAUL_HELPER_ENV}:?}}\" \"$@\"; }}\nsubdestdir() {{ /bin/sh \"${{{DEPOT_SUBDESTDIR_HELPER_ENV}:?}}\" \"$@\"; }}\npython_build() {{ /bin/sh \"${{{DEPOT_PYTHON_BUILD_HELPER_ENV}:?}}\" \"$@\"; }}\npython_install() {{ /bin/sh \"${{{DEPOT_PYTHON_INSTALL_HELPER_ENV}:?}}\" \"$@\"; }}\n{command}" + "haul() {{ /bin/sh \"${{{DEPOT_HAUL_HELPER_ENV}:?}}\" \"$@\"; }}\nsubdestdir() {{ /bin/sh \"${{{DEPOT_SUBDESTDIR_HELPER_ENV}:?}}\" \"$@\"; }}\npython_build() {{ /bin/sh \"${{{DEPOT_PYTHON_BUILD_HELPER_ENV}:?}}\" \"$@\"; }}\npython_install() {{ /bin/sh \"${{{DEPOT_PYTHON_INSTALL_HELPER_ENV}:?}}\" \"$@\"; }}\ndepot_clone() {{ /bin/sh \"${{{DEPOT_CLONE_HELPER_ENV}:?}}\" \"$@\"; }}\n{command}" ) } @@ -435,6 +461,19 @@ fail() { exec "$DEPOT_EXECUTABLE" internal python-install "$@" "#; +const CLONE_SCRIPT: &str = r#"#!/bin/sh +set -eu + +fail() { + echo "depot_clone: $*" >&2 + exit 1 +} + +[ "${DEPOT_EXECUTABLE:-}" != "" ] || fail "DEPOT_EXECUTABLE is not set" + +exec "$DEPOT_EXECUTABLE" internal clone "$@" +"#; + #[cfg(test)] mod tests { use super::{INTERNAL_DEPOT_DIR, ShellHelpers, shell_ident_suffix, wrap_shell_command}; @@ -517,14 +556,16 @@ mod tests { envs.iter() .any(|(key, _)| key == "DEPOT_PYTHON_INSTALL_HELPER") ); + assert!(envs.iter().any(|(key, _)| key == "DEPOT_CLONE_HELPER")); assert!(envs.iter().any(|(key, _)| key == "DEPOT_EXECUTABLE")); } #[test] fn wrap_shell_command_exposes_python_helpers() { - let wrapped = wrap_shell_command("python_build\npython_install"); + let wrapped = wrap_shell_command("python_build\npython_install\ndepot_clone foo"); assert!(wrapped.contains("python_build()")); assert!(wrapped.contains("python_install()")); + assert!(wrapped.contains("depot_clone()")); } #[test] diff --git a/src/source/git.rs b/src/source/git.rs index 03f7ea2..7f0d342 100755 --- a/src/source/git.rs +++ b/src/source/git.rs @@ -11,6 +11,7 @@ use sha2::{Digest, Sha256}; use std::fs; use std::io::{self, IsTerminal, Write}; use std::path::Path; +use url::Url; /// Checkout a repository URL at a specific revision into `checkout_dir`. /// @@ -33,6 +34,14 @@ pub fn checkout( git_cache_dir.display() ) })?; + if let Some(parent) = checkout_dir.parent() { + fs::create_dir_all(parent).with_context(|| { + format!( + "Failed to create parent directory for checkout: {}", + parent.display() + ) + })?; + } let mirror_dir = git_cache_dir.join(mirror_key(url)); ensure_mirror(url, &mirror_dir, pkgname, rev, cherry_pick_revs)?; @@ -76,6 +85,51 @@ pub fn checkout( Ok(()) } +/// Prime the git mirror cache for a repository/revision without creating a checkout. +pub fn prime_cache( + url: &str, + rev: &str, + git_cache_dir: &Path, + pkgname: &str, + cherry_pick_revs: &[String], +) -> Result<()> { + crate::interrupts::install().context("Failed to enable Ctrl-C handling for git operations")?; + fs::create_dir_all(git_cache_dir).with_context(|| { + format!( + "Failed to create git cache dir: {}", + git_cache_dir.display() + ) + })?; + let mirror_dir = git_cache_dir.join(mirror_key(url)); + ensure_mirror(url, &mirror_dir, pkgname, rev, cherry_pick_revs) +} + +/// Derive a default checkout directory name from a git repository URL. +pub fn default_checkout_dir_name(url: &str) -> String { + let without_fragment = url.split('#').next().unwrap_or(url).trim_end_matches('/'); + let last_segment = Url::parse(without_fragment) + .ok() + .and_then(|parsed| { + parsed + .path_segments()? + .rfind(|segment| !segment.is_empty()) + .map(str::to_string) + }) + .or_else(|| { + without_fragment + .rsplit(['/', ':']) + .find(|segment| !segment.is_empty()) + .map(str::to_string) + }) + .unwrap_or_else(|| "repo".to_string()); + + last_segment + .strip_suffix(".git") + .filter(|name| !name.is_empty()) + .unwrap_or(&last_segment) + .to_string() +} + fn apply_cherry_picks(repo: &Repository, cherry_pick_revs: &[String]) -> Result<()> { if cherry_pick_revs.is_empty() { return Ok(()); diff --git a/src/source/mod.rs b/src/source/mod.rs index d311bcc..0089b7b 100755 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -6,6 +6,10 @@ mod git; pub mod hooks; pub(crate) use git::authenticated_remote_callbacks; +pub(crate) use git::{ + checkout as git_checkout, default_checkout_dir_name as git_default_checkout_dir_name, + prime_cache as git_prime_cache, +}; use crate::package::PackageSpec; use anyhow::{Context, Result, bail}; @@ -56,7 +60,8 @@ fn manual_url_entries(manual: &crate::package::ManualSource) -> Vec { /// /// Local entries are checked for existence and optional checksum correctness. /// Remote entries are fetched into the manual-source cache so build-time source -/// preparation can reuse the verified result later. +/// preparation can reuse the verified result later. Git manual sources prime +/// their mirror cache and validate revision reachability. pub fn preflight_manual_sources(spec: &PackageSpec, cache_dir: &Path) -> Result<()> { if spec.manual_sources.is_empty() { return Ok(()); @@ -97,6 +102,24 @@ pub fn preflight_manual_sources(spec: &PackageSpec, cache_dir: &Path) -> Result< if !url_entries.is_empty() { for raw_url in url_entries { let expanded_url = expand_manual_source_value(spec, &raw_url); + if let Some((base, rev)) = split_git_url(&expanded_url) { + if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip") { + bail!( + "Manual git source {} cannot use checksum {}; pin the desired revision in the URL fragment instead", + expanded_url, + expected_hash + ); + } + git_prime_cache( + &base, + &rev, + &cache_dir.join("manual").join("git"), + &spec.package.name, + &[], + )?; + continue; + } + let parsed = Url::parse(&expanded_url) .with_context(|| format!("Invalid URL: {}", expanded_url))?; @@ -192,6 +215,19 @@ pub fn copy_manual_sources(spec: &PackageSpec, cache_dir: &Path, build_dir: &Pat if !url_entries.is_empty() { for raw_url in url_entries { let expanded_url = expand_manual_source_value(spec, &raw_url); + if let Some((base, rev)) = split_git_url(&expanded_url) { + checkout_manual_git_source( + spec, + manual, + build_dir, + cache_dir, + &expanded_url, + &base, + &rev, + )?; + continue; + } + let parsed = Url::parse(&expanded_url) .with_context(|| format!("Invalid URL: {}", expanded_url))?; @@ -255,11 +291,7 @@ fn copy_manual_source_file( manual: &crate::package::ManualSource, default_dest: &str, ) -> Result<()> { - let dest_name = if let Some(dest) = manual.dest.as_ref() { - expand_manual_source_value(spec, dest) - } else { - default_dest.to_string() - }; + let dest_name = manual_source_dest_name(spec, manual, default_dest); let dest_path = build_dir.join(&dest_name); if let Some(parent) = dest_path.parent() { @@ -277,6 +309,69 @@ fn copy_manual_source_file( Ok(()) } +fn manual_source_dest_name( + spec: &PackageSpec, + manual: &crate::package::ManualSource, + default_dest: &str, +) -> String { + if let Some(dest) = manual.dest.as_ref() { + expand_manual_source_value(spec, dest) + } else { + default_dest.to_string() + } +} + +fn remove_existing_path(path: &Path) -> Result<()> { + if !path.exists() && fs::symlink_metadata(path).is_err() { + return Ok(()); + } + let meta = + fs::symlink_metadata(path).with_context(|| format!("Failed to stat {}", path.display()))?; + if meta.file_type().is_dir() && !meta.file_type().is_symlink() { + fs::remove_dir_all(path) + .with_context(|| format!("Failed to remove directory {}", path.display()))?; + } else { + fs::remove_file(path) + .with_context(|| format!("Failed to remove file {}", path.display()))?; + } + Ok(()) +} + +fn checkout_manual_git_source( + spec: &PackageSpec, + manual: &crate::package::ManualSource, + build_dir: &Path, + cache_dir: &Path, + expanded_url: &str, + base: &str, + rev: &str, +) -> Result<()> { + if let Some(expected_hash) = manual.sha256.as_ref().filter(|h| *h != "skip") { + bail!( + "Manual git source {} cannot use checksum {}; pin the desired revision in the URL fragment instead", + expanded_url, + expected_hash + ); + } + + let default_dest = git_default_checkout_dir_name(base); + let dest_name = manual_source_dest_name(spec, manual, &default_dest); + let dest_path = build_dir.join(&dest_name); + if let Some(parent) = dest_path.parent() { + fs::create_dir_all(parent)?; + } + remove_existing_path(&dest_path)?; + crate::log_info!(" {} -> {}", expanded_url, dest_path.display()); + git_checkout( + base, + rev, + &dest_path, + &cache_dir.join("manual").join("git"), + &spec.package.name, + &[], + ) +} + /// Verify a file against an `expected` checksum string. /// /// Formats accepted: `sha256:HEX`, `sha512:HEX`, `sha1:HEX`, `md5:HEX`, @@ -554,7 +649,7 @@ fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<()> { Ok(()) } -fn split_git_url(url: &str) -> Option<(String, String)> { +pub(crate) fn split_git_url(url: &str) -> Option<(String, String)> { // Check for explicit revision with # if let Some((base, rev)) = url.split_once('#') { // Ignore fragment for obvious archives. @@ -997,6 +1092,49 @@ mod tests { ); } + #[test] + fn preflight_manual_sources_accepts_git_url() { + let (_tmp, remote_url, _tagged, hashed) = make_remote_git_repo(); + let spec = mk_spec_with_manuals( + PathBuf::from("."), + vec![ManualSource { + file: None, + files: Vec::new(), + url: Some(format!("{remote_url}#{hashed}")), + urls: Vec::new(), + sha256: None, + dest: None, + }], + ); + let cache_dir = tempfile::tempdir().unwrap(); + + preflight_manual_sources(&spec, cache_dir.path()).unwrap(); + } + + #[test] + fn copy_manual_sources_git_url_mode_checks_out_repository() { + let (_tmp, remote_url, _tagged, hashed) = make_remote_git_repo(); + let spec = mk_spec_with_manuals( + PathBuf::from("."), + vec![ManualSource { + file: None, + files: Vec::new(), + url: Some(format!("{remote_url}#{hashed}")), + urls: Vec::new(), + sha256: None, + dest: None, + }], + ); + let cache_dir = tempfile::tempdir().unwrap(); + let build_dir = tempfile::tempdir().unwrap(); + + copy_manual_sources(&spec, cache_dir.path(), build_dir.path()).unwrap(); + assert_eq!( + std::fs::read_to_string(build_dir.path().join("origin/README")).unwrap(), + "hashed\n" + ); + } + #[test] fn copy_manual_sources_multi_files_in_one_block() { let tmp = tempfile::tempdir().unwrap();