feat: update depot and zip versions to 0.30.1 and 8.3.1 respectively, enhance shell helper functions for safer file operations

Cargo.lock

@@ -418,7 +418,7 @@ checksum = "807800ff3288b621186fe0a8f3392c4652068257302709c24efd918c3dffcdc2"
 
 [[package]]
 name = "depot"
-version = "0.30.0"
+version = "0.30.1"
 dependencies = [
  "anyhow",
  "ar",
@@ -3090,9 +3090,9 @@ dependencies = [
 
 [[package]]
 name = "zip"
-version = "8.3.0"
+version = "8.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a243cfad17427fc077f529da5a95abe4e94fd2bfdb601611870a6557cc67657"
+checksum = "5c546feb4481b0fbafb4ef0d79b6204fc41c6f9884b1b73b1d73f82442fc0845"
 dependencies = [
  "aes",
  "bzip2",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "depot"
-version = "0.30.0"
+version = "0.30.1"
 edition = "2024"
 
 [lints.rust]
@@ -27,7 +27,7 @@ toml = "1.0.6"
 url = "2.5.8"
 walkdir = "2.5.0"
 xz2 = "0.1.7"
-zip = "8.3.0"
+zip = "8.3.1"
 zstd = { version = "0.13.3", features = ["zstdmt"] }
 inquire = "0.9.4"
 md5 = "0.8.0"

meson.build

@@ -1,6 +1,6 @@
 project(
   'depot',
-  version: '0.30.0',
+  version: '0.30.1',
   meson_version: '>=0.60.0',
   default_options: ['buildtype=release'],
 )

src/shell_helpers.rs

@@ -242,9 +242,69 @@ tmp_matches=$(mktemp "${TMPDIR:-/tmp}/depot-haul.XXXXXX")
 trap 'rm -f "$tmp_matches"' EXIT HUP INT TERM
 : > "$tmp_matches"
 
+safe_move() {
+    from=$1
+    to=$2
+
+    mkdir -p "$(dirname "$to")"
+
+    if mv "$from" "$to" 2>/dev/null; then
+        return 0
+    fi
+
+    if [ -d "$from" ] && [ ! -L "$from" ]; then
+        cp -a "$from" "$to" && rm -rf "$from"
+    elif [ -L "$from" ]; then
+        cp -P "$from" "$to" && rm -f "$from"
+    else
+        cp -p "$from" "$to" 2>/dev/null || cp "$from" "$to"
+        rm -f "$from"
+    fi
+}
+
+cleanup_empty_dirs() {
+    dir=$1
+    stop_at=$2
+
+    while [ "$dir" != "$stop_at" ] && [ "$dir" != "/" ] && [ "$dir" != "." ]; do
+        if [ -d "$dir" ] && [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+            rmdir "$dir" 2>/dev/null || break
+            dir=$(dirname "$dir")
+        else
+            break
+        fi
+    done
+}
+
+scan_root_for_pattern() {
+    root=$1
+    pattern=$2
+
+    [ -d "$root" ] || return 0
+
+    find "$root" -mindepth 1 -print | LC_ALL=C sort |
+    while IFS= read -r abs_path; do
+        rel_path=${abs_path#"$root"/}
+        [ "$rel_path" != "$abs_path" ] || continue
+
+        case "$root:$rel_path" in
+            "$src_root":.depot|"$src_root":.depot/*)
+                continue
+                ;;
+        esac
+
+        case "$rel_path" in
+            $pattern)
+                printf '%s\t%s\n' "$root" "$rel_path" >> "$tmp_matches"
+                ;;
+        esac
+    done
+}
+
 collect_matches_for_pattern() {
     pattern=$1
     before_count=$(wc -l < "$tmp_matches" | tr -d '[:space:]')
+
     case "$pattern" in
         ""|/|/*|../*|*/../*|..)
             fail "unsafe pattern: $pattern"
@@ -258,17 +318,12 @@ collect_matches_for_pattern() {
         fail "DESTDIR does not exist: $src_root"
     fi
 
-    find "$src_root" -mindepth 1 \
+    scan_root_for_pattern "$src_root" "$pattern"
-        ! -path "$DEPOT_OUTPUTS_DIR" ! -path "$DEPOT_OUTPUTS_DIR/*" \
+
-        -print | LC_ALL=C sort |
+    for candidate_root in "$DEPOT_OUTPUTS_DIR"/*; do
-    while IFS= read -r abs_path; do
+        [ -d "$candidate_root" ] || continue
-        rel_path=${abs_path#"$src_root"/}
+        [ "$candidate_root" = "$out_root" ] && continue
-        [ "$rel_path" != "$abs_path" ] || continue
+        scan_root_for_pattern "$candidate_root" "$pattern"
-        case "$rel_path" in
-            $pattern)
-                printf '%s\n' "$rel_path" >> "$tmp_matches"
-                ;;
-        esac
     done
 
     after_count=$(wc -l < "$tmp_matches" | tr -d '[:space:]')
@@ -281,9 +336,9 @@ for pattern in "$@"; do
     collect_matches_for_pattern "$pattern"
 done
 
-LC_ALL=C sort -u "$tmp_matches" | while IFS= read -r rel_path; do
+LC_ALL=C sort -u "$tmp_matches" | while IFS='	' read -r match_root rel_path; do
     [ -n "$rel_path" ] || continue
-    src_path=$src_root/$rel_path
+    src_path=$match_root/$rel_path
 
     if [ ! -e "$src_path" ] && [ ! -L "$src_path" ]; then
         # Path may already have been moved because an ancestor directory matched.
@@ -291,14 +346,29 @@ LC_ALL=C sort -u "$tmp_matches" | while IFS= read -r rel_path; do
     fi
 
     dst_path=$out_root/$rel_path
-    dst_parent=$(dirname "$dst_path")
+    if [ "$src_path" = "$dst_path" ]; then
-    mkdir -p "$dst_parent"
+        continue
-
-    if [ -e "$dst_path" ] || [ -L "$dst_path" ]; then
-        fail "destination already exists: $dst_path"
     fi
 
-    mv "$src_path" "$dst_path"
+    if [ -d "$src_path" ] && [ ! -L "$src_path" ] && [ -d "$dst_path" ]; then
+        for child in "$src_path"/* "$src_path"/.[!.]* "$src_path"/..?*; do
+            [ -e "$child" ] || [ -L "$child" ] || continue
+            child_name=$(basename "$child")
+            child_dst=$dst_path/$child_name
+            if [ -e "$child_dst" ] || [ -L "$child_dst" ]; then
+                fail "destination already exists: $child_dst"
+            fi
+            safe_move "$child" "$child_dst" || fail "failed to move: $child -> $child_dst"
+        done
+        rmdir "$src_path" 2>/dev/null || true
+    else
+        if [ -e "$dst_path" ] || [ -L "$dst_path" ]; then
+            fail "destination already exists: $dst_path"
+        fi
+        safe_move "$src_path" "$dst_path" || fail "failed to move: $src_path -> $dst_path"
+    fi
+
+    cleanup_empty_dirs "$(dirname "$src_path")" "$match_root"
 done
 
 # Clean empty directories left behind in the primary staging tree, but never
@@ -360,8 +430,28 @@ exec "$DEPOT_EXECUTABLE" internal python-install "$@"
 #[cfg(test)]
 mod tests {
     use super::{INTERNAL_DEPOT_DIR, ShellHelpers, shell_ident_suffix, wrap_shell_command};
+    use std::path::Path;
+    use std::process::Command;
     use tempfile::tempdir;
 
+    fn run_haul(helpers: &ShellHelpers, destdir: &Path, args: &[&str]) {
+        let mut envs = Vec::new();
+        helpers.apply_to_env_vars(&mut envs);
+        envs.push(("DESTDIR".into(), destdir.to_string_lossy().into_owned()));
+
+        let mut cmd = Command::new("sh");
+        cmd.arg(&helpers.haul_path);
+        for arg in args {
+            cmd.arg(arg);
+        }
+        for (key, value) in &envs {
+            cmd.env(key, value);
+        }
+
+        let status = cmd.status().unwrap();
+        assert!(status.success(), "haul failed for args: {args:?}");
+    }
+
     #[test]
     fn shell_ident_suffix_normalizes_package_names() {
         assert_eq!(shell_ident_suffix("clang"), "CLANG");
@@ -416,4 +506,86 @@ mod tests {
         assert!(wrapped.contains("python_build()"));
         assert!(wrapped.contains("python_install()"));
     }
+
+    #[test]
+    fn haul_can_move_paths_from_existing_output_staging() {
+        let temp = tempdir().unwrap();
+        let destdir = temp.path().join("dest");
+        std::fs::create_dir_all(destdir.join("usr/share/vulkan/implicit_layer.d")).unwrap();
+        std::fs::write(
+            destdir.join("usr/share/vulkan/implicit_layer.d/mesa.json"),
+            "mesa",
+        )
+        .unwrap();
+
+        let helpers = ShellHelpers::new(&destdir).unwrap();
+
+        run_haul(
+            &helpers,
+            &destdir,
+            &["vulkan-a", "usr/share/vulkan/implicit_layer.d"],
+        );
+        run_haul(
+            &helpers,
+            &destdir,
+            &["vulkan-b", "usr/share/vulkan/implicit_layer.d"],
+        );
+
+        assert!(
+            !destdir
+                .join(".depot/outputs/vulkan-a/usr/share/vulkan/implicit_layer.d")
+                .exists()
+        );
+        assert!(
+            destdir
+                .join(".depot/outputs/vulkan-b/usr/share/vulkan/implicit_layer.d/mesa.json")
+                .exists()
+        );
+    }
+
+    #[test]
+    fn haul_merges_directory_into_existing_output_dir() {
+        let temp = tempdir().unwrap();
+        let destdir = temp.path().join("dest");
+        std::fs::create_dir_all(destdir.join("usr/share/vulkan/explicit_layer.d")).unwrap();
+        std::fs::write(
+            destdir.join("usr/share/vulkan/explicit_layer.d/new-layer.json"),
+            "new",
+        )
+        .unwrap();
+        std::fs::create_dir_all(
+            destdir.join(".depot/outputs/vulkan-layers/usr/share/vulkan/explicit_layer.d"),
+        )
+        .unwrap();
+        std::fs::write(
+            destdir.join(
+                ".depot/outputs/vulkan-layers/usr/share/vulkan/explicit_layer.d/existing.json",
+            ),
+            "existing",
+        )
+        .unwrap();
+
+        let helpers = ShellHelpers::new(&destdir).unwrap();
+        run_haul(
+            &helpers,
+            &destdir,
+            &["vulkan-layers", "usr/share/vulkan/explicit_layer.d"],
+        );
+
+        assert!(!destdir.join("usr/share/vulkan/explicit_layer.d").exists());
+        assert!(
+            destdir
+                .join(
+                    ".depot/outputs/vulkan-layers/usr/share/vulkan/explicit_layer.d/existing.json"
+                )
+                .exists()
+        );
+        assert!(
+            destdir
+                .join(
+                    ".depot/outputs/vulkan-layers/usr/share/vulkan/explicit_layer.d/new-layer.json"
+                )
+                .exists()
+        );
+    }
 }