Files
depot/src/shell_helpers.rs
T

700 lines
22 KiB
Rust

//! Shell helper commands exposed to package build/post-install scripts.
use crate::builder::{EnvVars, set_env_var};
use anyhow::{Context, Result};
use std::fs;
use std::path::{Path, PathBuf};
use tempfile::TempDir;
/// Internal staging namespace reserved inside `DESTDIR`.
pub const INTERNAL_DEPOT_DIR: &str = ".depot";
/// Internal split-output staging root inside `DESTDIR`.
pub const INTERNAL_OUTPUTS_DIR: &str = ".depot/outputs";
const DEPOT_HAUL_HELPER_ENV: &str = "DEPOT_HAUL_HELPER";
const DEPOT_SUBDESTDIR_HELPER_ENV: &str = "DEPOT_SUBDESTDIR_HELPER";
const DEPOT_PYTHON_BUILD_HELPER_ENV: &str = "DEPOT_PYTHON_BUILD_HELPER";
const DEPOT_PYTHON_INSTALL_HELPER_ENV: &str = "DEPOT_PYTHON_INSTALL_HELPER";
const DEPOT_CLONE_HELPER_ENV: &str = "DEPOT_CLONE_HELPER";
const DEPOT_EXECUTABLE_ENV: &str = "DEPOT_EXECUTABLE";
/// Ephemeral helper command directory to prepend to PATH while running scripts.
pub struct ShellHelpers {
_tempdir: TempDir,
path_value: String,
outputs_dir: PathBuf,
depot_executable: PathBuf,
haul_path: PathBuf,
subdestdir_path: PathBuf,
python_build_path: PathBuf,
python_install_path: PathBuf,
clone_path: PathBuf,
}
impl ShellHelpers {
/// Create helper commands for a given staging tree (`DESTDIR`).
pub fn new(destdir: &Path) -> Result<Self> {
let helper_root = destdir.join(INTERNAL_DEPOT_DIR).join("helpers");
fs::create_dir_all(&helper_root).with_context(|| {
format!(
"Failed to create shell helper root dir: {}",
helper_root.display()
)
})?;
let tempdir =
tempfile::tempdir_in(&helper_root).context("Failed to create shell helper tempdir")?;
let bin_dir = tempdir.path().join("bin");
fs::create_dir_all(&bin_dir)
.with_context(|| format!("Failed to create helper bin dir: {}", bin_dir.display()))?;
let depot_executable = std::env::current_exe()
.context("Failed to locate depot executable for shell helpers")?;
let haul_path = bin_dir.join("haul");
fs::write(&haul_path, HAUL_SCRIPT).with_context(|| {
format!(
"Failed to write shell helper command: {}",
haul_path.display()
)
})?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(&haul_path)
.with_context(|| format!("Failed to stat helper: {}", haul_path.display()))?
.permissions();
perms.set_mode(0o755);
fs::set_permissions(&haul_path, perms)
.with_context(|| format!("Failed to chmod helper: {}", haul_path.display()))?;
}
let subdestdir_path = bin_dir.join("subdestdir");
fs::write(&subdestdir_path, SUBDESTDIR_SCRIPT).with_context(|| {
format!(
"Failed to write shell helper command: {}",
subdestdir_path.display()
)
})?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(&subdestdir_path)
.with_context(|| format!("Failed to stat helper: {}", subdestdir_path.display()))?
.permissions();
perms.set_mode(0o755);
fs::set_permissions(&subdestdir_path, perms).with_context(|| {
format!("Failed to chmod helper: {}", subdestdir_path.display())
})?;
}
let python_build_path = bin_dir.join("python_build");
fs::write(&python_build_path, PYTHON_BUILD_SCRIPT).with_context(|| {
format!(
"Failed to write shell helper command: {}",
python_build_path.display()
)
})?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(&python_build_path)
.with_context(|| format!("Failed to stat helper: {}", python_build_path.display()))?
.permissions();
perms.set_mode(0o755);
fs::set_permissions(&python_build_path, perms).with_context(|| {
format!("Failed to chmod helper: {}", python_build_path.display())
})?;
}
let python_install_path = bin_dir.join("python_install");
fs::write(&python_install_path, PYTHON_INSTALL_SCRIPT).with_context(|| {
format!(
"Failed to write shell helper command: {}",
python_install_path.display()
)
})?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(&python_install_path)
.with_context(|| {
format!("Failed to stat helper: {}", python_install_path.display())
})?
.permissions();
perms.set_mode(0o755);
fs::set_permissions(&python_install_path, perms).with_context(|| {
format!("Failed to chmod helper: {}", python_install_path.display())
})?;
}
let clone_path = bin_dir.join("depot_clone");
fs::write(&clone_path, CLONE_SCRIPT).with_context(|| {
format!(
"Failed to write shell helper command: {}",
clone_path.display()
)
})?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(&clone_path)
.with_context(|| format!("Failed to stat helper: {}", clone_path.display()))?
.permissions();
perms.set_mode(0o755);
fs::set_permissions(&clone_path, perms)
.with_context(|| format!("Failed to chmod helper: {}", clone_path.display()))?;
}
let path_value = crate::runtime_env::prepend_helper_to_safe_path(&bin_dir);
Ok(Self {
_tempdir: tempdir,
path_value,
outputs_dir: destdir.join(INTERNAL_OUTPUTS_DIR),
depot_executable,
haul_path,
subdestdir_path,
python_build_path,
python_install_path,
clone_path,
})
}
/// Apply helper-related variables to an environment vector used with `prepare_command`.
pub fn apply_to_env_vars(&self, env_vars: &mut EnvVars) {
set_env_var(env_vars, "PATH", self.path_value.clone());
set_env_var(
env_vars,
"DEPOT_OUTPUTS_DIR",
self.outputs_dir.to_string_lossy().into_owned(),
);
set_env_var(env_vars, "DEPOT_INTERNAL_DIR", INTERNAL_DEPOT_DIR);
set_env_var(
env_vars,
DEPOT_HAUL_HELPER_ENV,
self.haul_path.to_string_lossy().into_owned(),
);
set_env_var(
env_vars,
DEPOT_SUBDESTDIR_HELPER_ENV,
self.subdestdir_path.to_string_lossy().into_owned(),
);
set_env_var(
env_vars,
DEPOT_PYTHON_BUILD_HELPER_ENV,
self.python_build_path.to_string_lossy().into_owned(),
);
set_env_var(
env_vars,
DEPOT_PYTHON_INSTALL_HELPER_ENV,
self.python_install_path.to_string_lossy().into_owned(),
);
set_env_var(
env_vars,
DEPOT_CLONE_HELPER_ENV,
self.clone_path.to_string_lossy().into_owned(),
);
set_env_var(
env_vars,
DEPOT_EXECUTABLE_ENV,
self.depot_executable.to_string_lossy().into_owned(),
);
}
}
/// Wrap a shell command with helper functions that invoke the helper scripts
/// through `/bin/sh`, avoiding direct execution from mounts that may be `noexec`.
pub fn wrap_shell_command(command: &str) -> String {
format!(
"haul() {{ /bin/sh \"${{{DEPOT_HAUL_HELPER_ENV}:?}}\" \"$@\"; }}\nsubdestdir() {{ /bin/sh \"${{{DEPOT_SUBDESTDIR_HELPER_ENV}:?}}\" \"$@\"; }}\npython_build() {{ /bin/sh \"${{{DEPOT_PYTHON_BUILD_HELPER_ENV}:?}}\" \"$@\"; }}\npython_install() {{ /bin/sh \"${{{DEPOT_PYTHON_INSTALL_HELPER_ENV}:?}}\" \"$@\"; }}\ndepot_clone() {{ /bin/sh \"${{{DEPOT_CLONE_HELPER_ENV}:?}}\" \"$@\"; }}\n{command}"
)
}
/// Convert a package name into a safe shell identifier suffix.
pub fn shell_ident_suffix(pkg_name: &str) -> String {
let mut out = String::with_capacity(pkg_name.len().max(1));
for ch in pkg_name.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch.to_ascii_uppercase());
} else {
out.push('_');
}
}
if out.is_empty() {
out.push('_');
}
if out.as_bytes().first().is_some_and(|b| b.is_ascii_digit()) {
out.insert(0, '_');
}
out
}
const HAUL_SCRIPT: &str = r#"#!/bin/sh
set -eu
usage() {
echo "Usage: haul <output-package> <path-pattern> [path-pattern ...]" >&2
exit 2
}
fail() {
echo "haul: $*" >&2
exit 1
}
[ "$#" -ge 2 ] || usage
out_pkg=$1
shift
case "$out_pkg" in
""|.|..|*/*) fail "invalid output package name: $out_pkg" ;;
esac
[ "${DESTDIR:-}" != "" ] || fail "DESTDIR is not set"
[ "${DEPOT_OUTPUTS_DIR:-}" != "" ] || fail "DEPOT_OUTPUTS_DIR is not set"
src_root=$DESTDIR
primary_root=${DEPOT_PRIMARY_DESTDIR:-$DESTDIR}
out_root=$DEPOT_OUTPUTS_DIR/$out_pkg
case "$src_root" in
*/) src_root=${src_root%/} ;;
esac
case "$primary_root" in
*/) primary_root=${primary_root%/} ;;
esac
case "$out_root" in
*/) out_root=${out_root%/} ;;
esac
mkdir -p "$out_root"
tmp_matches=$(mktemp "${TMPDIR:-/tmp}/depot-haul.XXXXXX")
trap 'rm -f "$tmp_matches"' EXIT HUP INT TERM
: > "$tmp_matches"
safe_move() {
from=$1
to=$2
mkdir -p "$(dirname "$to")"
if mv "$from" "$to" 2>/dev/null; then
return 0
fi
if [ -d "$from" ] && [ ! -L "$from" ]; then
cp -a "$from" "$to" && rm -rf "$from"
elif [ -L "$from" ]; then
cp -P "$from" "$to" && rm -f "$from"
else
cp -p "$from" "$to" 2>/dev/null || cp "$from" "$to"
rm -f "$from"
fi
}
cleanup_empty_dirs() {
dir=$1
stop_at=$2
while [ "$dir" != "$stop_at" ] && [ "$dir" != "/" ] && [ "$dir" != "." ]; do
if [ -d "$dir" ] && [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
rmdir "$dir" 2>/dev/null || break
dir=$(dirname "$dir")
else
break
fi
done
}
scan_root_for_pattern() {
root=$1
pattern=$2
[ -d "$root" ] || return 0
find "$root" -mindepth 1 -print | LC_ALL=C sort |
while IFS= read -r abs_path; do
rel_path=${abs_path#"$root"/}
[ "$rel_path" != "$abs_path" ] || continue
case "$root:$rel_path" in
"$src_root":.depot|"$src_root":.depot/*)
continue
;;
esac
case "$rel_path" in
$pattern)
printf '%s\t%s\n' "$root" "$rel_path" >> "$tmp_matches"
;;
esac
done
}
collect_matches_for_pattern() {
pattern=$1
before_count=$(wc -l < "$tmp_matches" | tr -d '[:space:]')
case "$pattern" in
""|/|/*|../*|*/../*|..)
fail "unsafe pattern: $pattern"
;;
.depot|.depot/*)
fail "refusing to haul internal staging paths: $pattern"
;;
esac
if [ ! -d "$src_root" ]; then
fail "DESTDIR does not exist: $src_root"
fi
scan_root_for_pattern "$src_root" "$pattern"
if [ "$primary_root" != "$src_root" ]; then
scan_root_for_pattern "$primary_root" "$pattern"
fi
for candidate_root in "$DEPOT_OUTPUTS_DIR"/*; do
[ -d "$candidate_root" ] || continue
[ "$candidate_root" = "$out_root" ] && continue
scan_root_for_pattern "$candidate_root" "$pattern"
done
after_count=$(wc -l < "$tmp_matches" | tr -d '[:space:]')
if [ "$before_count" = "$after_count" ]; then
fail "no matches for pattern: $pattern"
fi
}
for pattern in "$@"; do
collect_matches_for_pattern "$pattern"
done
LC_ALL=C sort -u "$tmp_matches" | while IFS=' ' read -r match_root rel_path; do
[ -n "$rel_path" ] || continue
src_path=$match_root/$rel_path
if [ ! -e "$src_path" ] && [ ! -L "$src_path" ]; then
# Path may already have been moved because an ancestor directory matched.
continue
fi
dst_path=$out_root/$rel_path
if [ "$src_path" = "$dst_path" ]; then
continue
fi
if [ -d "$src_path" ] && [ ! -L "$src_path" ] && [ -d "$dst_path" ]; then
for child in "$src_path"/* "$src_path"/.[!.]* "$src_path"/..?*; do
[ -e "$child" ] || [ -L "$child" ] || continue
child_name=$(basename "$child")
child_dst=$dst_path/$child_name
if [ -e "$child_dst" ] || [ -L "$child_dst" ]; then
fail "destination already exists: $child_dst"
fi
safe_move "$child" "$child_dst" || fail "failed to move: $child -> $child_dst"
done
rmdir "$src_path" 2>/dev/null || true
else
if [ -e "$dst_path" ] || [ -L "$dst_path" ]; then
fail "destination already exists: $dst_path"
fi
safe_move "$src_path" "$dst_path" || fail "failed to move: $src_path -> $dst_path"
fi
cleanup_empty_dirs "$(dirname "$src_path")" "$match_root"
done
# Clean empty directories left behind in the primary staging tree, but never
# touch the internal depot namespace.
find "$src_root" -depth -type d \
! -path "$src_root" \
! -path "$src_root/.depot" ! -path "$src_root/.depot/*" \
-empty -exec rmdir {} + 2>/dev/null || true
"#;
const SUBDESTDIR_SCRIPT: &str = r#"#!/bin/sh
set -eu
fail() {
echo "subdestdir: $*" >&2
exit 1
}
[ "$#" -eq 1 ] || fail "usage: subdestdir <output-package>"
[ "${DEPOT_OUTPUTS_DIR:-}" != "" ] || fail "DEPOT_OUTPUTS_DIR is not set"
pkg=$1
case "$pkg" in
""|.|..|*/*) fail "invalid output package name: $pkg" ;;
esac
path=$DEPOT_OUTPUTS_DIR/$pkg
mkdir -p "$path"
printf '%s\n' "$path"
"#;
const PYTHON_BUILD_SCRIPT: &str = r#"#!/bin/sh
set -eu
fail() {
echo "python_build: $*" >&2
exit 1
}
[ "${DEPOT_EXECUTABLE:-}" != "" ] || fail "DEPOT_EXECUTABLE is not set"
exec "$DEPOT_EXECUTABLE" internal python-build "$@"
"#;
const PYTHON_INSTALL_SCRIPT: &str = r#"#!/bin/sh
set -eu
fail() {
echo "python_install: $*" >&2
exit 1
}
[ "${DEPOT_EXECUTABLE:-}" != "" ] || fail "DEPOT_EXECUTABLE is not set"
[ "${DESTDIR:-}" != "" ] || fail "DESTDIR is not set"
exec "$DEPOT_EXECUTABLE" internal python-install "$@"
"#;
const CLONE_SCRIPT: &str = r#"#!/bin/sh
set -eu
fail() {
echo "depot_clone: $*" >&2
exit 1
}
[ "${DEPOT_EXECUTABLE:-}" != "" ] || fail "DEPOT_EXECUTABLE is not set"
exec "$DEPOT_EXECUTABLE" internal clone "$@"
"#;
#[cfg(test)]
mod tests {
use super::{INTERNAL_DEPOT_DIR, ShellHelpers, shell_ident_suffix, wrap_shell_command};
use std::path::Path;
use std::process::Command;
use tempfile::tempdir;
fn run_haul(helpers: &ShellHelpers, destdir: &Path, args: &[&str]) {
run_haul_with_env(helpers, destdir, &[], args);
}
fn run_haul_with_env(
helpers: &ShellHelpers,
destdir: &Path,
extra_envs: &[(&str, &Path)],
args: &[&str],
) {
let mut envs = Vec::new();
helpers.apply_to_env_vars(&mut envs);
envs.push(("DESTDIR".into(), destdir.to_string_lossy().into_owned()));
for (key, value) in extra_envs {
envs.push(((*key).into(), value.to_string_lossy().into_owned()));
}
let mut cmd = Command::new("sh");
cmd.arg(&helpers.haul_path);
for arg in args {
cmd.arg(arg);
}
for (key, value) in &envs {
cmd.env(key, value);
}
let status = cmd.status().unwrap();
assert!(status.success(), "haul failed for args: {args:?}");
}
#[test]
fn shell_ident_suffix_normalizes_package_names() {
assert_eq!(shell_ident_suffix("clang"), "CLANG");
assert_eq!(shell_ident_suffix("llvm-libgcc"), "LLVM_LIBGCC");
assert_eq!(shell_ident_suffix("3foo"), "_3FOO");
assert_eq!(shell_ident_suffix("foo.bar+baz"), "FOO_BAR_BAZ");
}
#[test]
fn shell_helpers_use_destdir_internal_helper_dir() {
let destdir = tempdir().unwrap();
let helpers = ShellHelpers::new(destdir.path()).unwrap();
let mut envs = Vec::new();
helpers.apply_to_env_vars(&mut envs);
let path = envs
.iter()
.find(|(key, _)| key == "PATH")
.map(|(_, value)| value)
.unwrap();
let helper_prefix = destdir
.path()
.join(INTERNAL_DEPOT_DIR)
.join("helpers")
.to_string_lossy()
.into_owned();
assert!(path.starts_with(&helper_prefix));
}
#[test]
fn shell_helpers_export_python_helper_paths() {
let destdir = tempdir().unwrap();
let helpers = ShellHelpers::new(destdir.path()).unwrap();
let mut envs = Vec::new();
helpers.apply_to_env_vars(&mut envs);
assert!(
envs.iter()
.any(|(key, _)| key == "DEPOT_PYTHON_BUILD_HELPER")
);
assert!(
envs.iter()
.any(|(key, _)| key == "DEPOT_PYTHON_INSTALL_HELPER")
);
assert!(envs.iter().any(|(key, _)| key == "DEPOT_CLONE_HELPER"));
assert!(envs.iter().any(|(key, _)| key == "DEPOT_EXECUTABLE"));
}
#[test]
fn wrap_shell_command_exposes_python_helpers() {
let wrapped = wrap_shell_command("python_build\npython_install\ndepot_clone foo");
assert!(wrapped.contains("python_build()"));
assert!(wrapped.contains("python_install()"));
assert!(wrapped.contains("depot_clone()"));
}
#[test]
fn haul_can_move_paths_from_existing_output_staging() {
let temp = tempdir().unwrap();
let destdir = temp.path().join("dest");
std::fs::create_dir_all(destdir.join("usr/share/vulkan/implicit_layer.d")).unwrap();
std::fs::write(
destdir.join("usr/share/vulkan/implicit_layer.d/mesa.json"),
"mesa",
)
.unwrap();
let helpers = ShellHelpers::new(&destdir).unwrap();
run_haul(
&helpers,
&destdir,
&["vulkan-a", "usr/share/vulkan/implicit_layer.d"],
);
run_haul(
&helpers,
&destdir,
&["vulkan-b", "usr/share/vulkan/implicit_layer.d"],
);
assert!(
!destdir
.join(".depot/outputs/vulkan-a/usr/share/vulkan/implicit_layer.d")
.exists()
);
assert!(
destdir
.join(".depot/outputs/vulkan-b/usr/share/vulkan/implicit_layer.d/mesa.json")
.exists()
);
}
#[test]
fn haul_merges_directory_into_existing_output_dir() {
let temp = tempdir().unwrap();
let destdir = temp.path().join("dest");
std::fs::create_dir_all(destdir.join("usr/share/vulkan/explicit_layer.d")).unwrap();
std::fs::write(
destdir.join("usr/share/vulkan/explicit_layer.d/new-layer.json"),
"new",
)
.unwrap();
std::fs::create_dir_all(
destdir.join(".depot/outputs/vulkan-layers/usr/share/vulkan/explicit_layer.d"),
)
.unwrap();
std::fs::write(
destdir.join(
".depot/outputs/vulkan-layers/usr/share/vulkan/explicit_layer.d/existing.json",
),
"existing",
)
.unwrap();
let helpers = ShellHelpers::new(&destdir).unwrap();
run_haul(
&helpers,
&destdir,
&["vulkan-layers", "usr/share/vulkan/explicit_layer.d"],
);
assert!(!destdir.join("usr/share/vulkan/explicit_layer.d").exists());
assert!(
destdir
.join(
".depot/outputs/vulkan-layers/usr/share/vulkan/explicit_layer.d/existing.json"
)
.exists()
);
assert!(
destdir
.join(
".depot/outputs/vulkan-layers/usr/share/vulkan/explicit_layer.d/new-layer.json"
)
.exists()
);
}
#[test]
fn haul_scans_primary_staging_when_running_in_split_output_destdir() {
let temp = tempdir().unwrap();
let primary_destdir = temp.path().join("dest");
std::fs::create_dir_all(primary_destdir.join("usr/lib")).unwrap();
std::fs::write(
primary_destdir.join("usr/lib/libRusticlOpenCL.so.1.0.0"),
"rusticl",
)
.unwrap();
std::os::unix::fs::symlink(
"libRusticlOpenCL.so.1.0.0",
primary_destdir.join("usr/lib/libRusticlOpenCL.so.1"),
)
.unwrap();
std::os::unix::fs::symlink(
"libRusticlOpenCL.so.1",
primary_destdir.join("usr/lib/libRusticlOpenCL.so"),
)
.unwrap();
let helpers = ShellHelpers::new(&primary_destdir).unwrap();
let split_destdir = primary_destdir.join(".depot/outputs/opencl-mesa");
std::fs::create_dir_all(&split_destdir).unwrap();
run_haul_with_env(
&helpers,
&split_destdir,
&[("DEPOT_PRIMARY_DESTDIR", &primary_destdir)],
&["opencl-mesa", "usr/lib/libRusticlOpenCL*"],
);
assert!(
!primary_destdir
.join("usr/lib/libRusticlOpenCL.so.1.0.0")
.exists()
);
assert!(
split_destdir
.join("usr/lib/libRusticlOpenCL.so.1.0.0")
.exists()
);
let link_meta =
std::fs::symlink_metadata(split_destdir.join("usr/lib/libRusticlOpenCL.so")).unwrap();
assert!(link_meta.file_type().is_symlink());
}
}