diff --git a/README.md b/README.md index 4ed5106..ef19b34 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ depot install zlib-1.2.11-1-x86_64.depot.pkg.tar.zst - `list`: List all installed packages. - `repo owns `: Query binary repo metadata for path ownership. - `repo create [DIR]`: Create a repository database from a directory of packages. +- `repo index [DIR] [--subdir ...]`: Create/update `depot-index.tsv` at a source repo root for fast source lookup. - `config`: Show current configuration and overrides. ## Package Specification diff --git a/src/cli.rs b/src/cli.rs index 27bb514..9176337 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -142,6 +142,15 @@ pub enum RepoCommands { /// Update only one source repo by name name: Option, }, + /// Create/update a source index at the root of a source repo + Index { + /// Source repository root directory + #[arg(default_value = ".")] + dir: PathBuf, + /// Optional subdirectory to scan (repeatable, e.g. --subdir core --subdir extra) + #[arg(long = "subdir")] + subdirs: Vec, + }, /// List configured source and binary repos List, /// Add or update a repo entry in /etc/depot.d/repos.toml diff --git a/src/commands.rs b/src/commands.rs index e4946d5..b50f704 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -1670,6 +1670,28 @@ pub fn run(cli: Cli) -> Result<()> { } } } + RepoCommands::Index { dir, subdirs } => { + let cfg = config::Config::for_rootfs(&cli.rootfs); + let mut repo_lock = locking::open_lock(&cfg)?; + let repo_lock_path = locking::lock_path(&cfg); + let _repo_lock_guard = + locking::try_write(&mut repo_lock, &repo_lock_path, "repo index")?; + let stats = index::create_source_repo_index(&dir, &subdirs).with_context(|| { + format!("Failed to create source index for {}", dir.display()) + })?; + ui::success(format!( + "Wrote source index: {}", + stats.index_path.display() + )); + ui::info(format!( + "Indexed {} spec(s) from {} TOML file(s): package rows={} provides rows={} ignored_toml={}", + stats.specs_indexed, + stats.toml_files_scanned, + stats.package_rows, + stats.provides_rows, + stats.ignored_toml_files + )); + } RepoCommands::List => { let config = config::Config::for_rootfs(&cli.rootfs); let repo_lock = locking::open_lock(&config)?; diff --git a/src/index.rs b/src/index.rs index 37b4939..9d28f79 100755 --- a/src/index.rs +++ b/src/index.rs @@ -3,9 +3,35 @@ //! Caches package name -> spec path and provides -> spec path mappings. use crate::package::PackageSpec; +use anyhow::{Context, Result}; use std::collections::HashMap; +use std::ffi::OsStr; use std::fs; -use std::path::PathBuf; +use std::path::{Component, Path, PathBuf}; +use walkdir::WalkDir; + +/// Filename for the source-repo index written at a repo root. +pub const SOURCE_REPO_INDEX_FILENAME: &str = "depot-index.tsv"; +const SOURCE_REPO_INDEX_HEADER: &str = "depot-source-index-v1"; +const SOURCE_REPO_INDEX_KIND_PACKAGE: &str = "P"; +const SOURCE_REPO_INDEX_KIND_PROVIDES: &str = "V"; + +/// Statistics for generating a source repository index file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceRepoIndexStats { + /// Absolute path to the written index file. + pub index_path: PathBuf, + /// Number of `.toml` files discovered under the selected scan roots. + pub toml_files_scanned: usize, + /// Number of discovered TOML files that parsed as valid package specs. + pub specs_indexed: usize, + /// Number of package-name rows written to the index. + pub package_rows: usize, + /// Number of provides rows written to the index. + pub provides_rows: usize, + /// Number of discovered TOML files ignored because they were not package specs. + pub ignored_toml_files: usize, +} /// Cached package index for O(1) lookups #[derive(Debug, Default)] @@ -24,6 +50,126 @@ pub struct SourceSearchHit { pub provides: Vec, } +/// Return the source index file path for `repo_root`. +pub fn source_repo_index_path(repo_root: &Path) -> PathBuf { + repo_root.join(SOURCE_REPO_INDEX_FILENAME) +} + +/// Create/update the source-repo package index at `repo_root`. +/// +/// The file format is line-based TSV with deterministic ordering: +/// - Header: `depot-source-index-v1` +/// - Package name row: `P` +/// - Provides row: `V` +pub fn create_source_repo_index( + repo_root: &Path, + subdirs: &[String], +) -> Result { + let repo_root = repo_root + .canonicalize() + .with_context(|| format!("Failed to resolve repo root {}", repo_root.display()))?; + if !repo_root.is_dir() { + anyhow::bail!("Repo root is not a directory: {}", repo_root.display()); + } + + let scan_roots = resolve_scan_roots(&repo_root, subdirs)?; + let mut spec_rows: Vec<(String, String, Vec)> = Vec::new(); + let mut toml_files_scanned = 0usize; + let mut ignored_toml_files = 0usize; + + for spec_path in scan_toml_files(&scan_roots)? { + toml_files_scanned += 1; + let spec = match PackageSpec::from_file(&spec_path) { + Ok(spec) => spec, + Err(_) => { + ignored_toml_files += 1; + continue; + } + }; + + let rel = spec_path.strip_prefix(&repo_root).with_context(|| { + format!( + "Failed to compute relative path for {} from {}", + spec_path.display(), + repo_root.display() + ) + })?; + let rel = rel.to_string_lossy().replace('\\', "/"); + spec_rows.push((spec.package.name.clone(), rel, spec.alternatives.provides)); + } + + let mut rows: Vec<(String, String, String)> = Vec::new(); + for (name, rel, provides) in &spec_rows { + rows.push(( + SOURCE_REPO_INDEX_KIND_PACKAGE.to_string(), + name.clone(), + rel.clone(), + )); + for provided in provides { + rows.push(( + SOURCE_REPO_INDEX_KIND_PROVIDES.to_string(), + provided.clone(), + rel.clone(), + )); + } + } + rows.sort_by(|a, b| { + a.0.cmp(&b.0) + .then_with(|| a.1.cmp(&b.1)) + .then_with(|| a.2.cmp(&b.2)) + }); + + let mut out = String::new(); + out.push_str(SOURCE_REPO_INDEX_HEADER); + out.push('\n'); + for (kind, name, rel) in &rows { + if name.contains('\n') || name.contains('\r') || name.contains('\t') { + anyhow::bail!( + "Index field contains unsupported control character: {}", + name + ); + } + if rel.contains('\n') || rel.contains('\r') || rel.contains('\t') { + anyhow::bail!("Index path contains unsupported control character: {}", rel); + } + out.push_str(kind); + out.push('\t'); + out.push_str(name); + out.push('\t'); + out.push_str(rel); + out.push('\n'); + } + + let index_path = source_repo_index_path(&repo_root); + let tmp_path = index_path.with_extension("tsv.tmp"); + fs::write(&tmp_path, out).with_context(|| format!("Failed to write {}", tmp_path.display()))?; + fs::rename(&tmp_path, &index_path).with_context(|| { + format!( + "Failed to replace index {} from {}", + index_path.display(), + tmp_path.display() + ) + })?; + + let package_rows = rows + .iter() + .filter(|(kind, _, _)| kind == SOURCE_REPO_INDEX_KIND_PACKAGE) + .count(); + let provides_rows = rows + .iter() + .filter(|(kind, _, _)| kind == SOURCE_REPO_INDEX_KIND_PROVIDES) + .count(); + + Ok(SourceRepoIndexStats { + index_path, + toml_files_scanned, + specs_indexed: spec_rows.len(), + package_rows, + provides_rows, + ignored_toml_files, + }) +} + impl PackageIndex { /// Build index by scanning packages/*/*.toml and configured repo dir. /// @@ -35,62 +181,11 @@ impl PackageIndex { let mut index = Self::default(); let packages_dir = PathBuf::from("packages"); - // Scan local packages//*.toml - if let Ok(entries) = fs::read_dir(&packages_dir) { - for entry in entries.flatten() { - if !entry.file_type().map(|t| t.is_dir()).unwrap_or(false) { - continue; - } - let dir = entry.path(); + index.scan_spec_tree(&packages_dir); - if let Ok(files) = fs::read_dir(&dir) { - for file in files.flatten() { - let path = file.path(); - if path.extension().map(|e| e == "toml").unwrap_or(false) - && let Ok(spec) = PackageSpec::from_file(&path) - { - index - .by_name - .insert(spec.package.name.clone(), path.clone()); - for provided in &spec.alternatives.provides { - index - .by_provides - .entry(provided.clone()) - .or_default() - .push(path.clone()); - } - } - } - } - } - } - - // Also scan system mirrors under provided repo_dir (or default) let sys_dir = repo_dir.unwrap_or_else(|| PathBuf::from("/usr/src/depot")); - if sys_dir.exists() { - for entry in walkdir::WalkDir::new(&sys_dir) - .min_depth(1) - .max_depth(5) - .into_iter() - .flatten() - { - let path = entry.path().to_path_buf(); - if path.extension().map(|e| e == "toml").unwrap_or(false) - && let Ok(spec) = PackageSpec::from_file(&path) - { - index - .by_name - .insert(spec.package.name.clone(), path.clone()); - for provided in &spec.alternatives.provides { - index - .by_provides - .entry(provided.clone()) - .or_default() - .push(path.clone()); - } - } - } + index.scan_repo_store(&sys_dir); } crate::log_info!( @@ -102,6 +197,150 @@ impl PackageIndex { index } + fn scan_repo_store(&mut self, sys_dir: &Path) { + let direct_index = source_repo_index_path(sys_dir); + if direct_index.exists() || sys_dir.join(".git").is_dir() { + self.scan_repo_root(sys_dir); + return; + } + + let mut child_dirs = Vec::new(); + if let Ok(entries) = fs::read_dir(sys_dir) { + for entry in entries.flatten() { + let Ok(ft) = entry.file_type() else { + continue; + }; + if ft.is_dir() { + child_dirs.push(entry.path()); + } + } + } + + if child_dirs.is_empty() { + self.scan_spec_tree(sys_dir); + return; + } + + child_dirs.sort(); + for child in child_dirs { + self.scan_repo_root(&child); + } + } + + fn scan_repo_root(&mut self, repo_root: &Path) { + let index_path = source_repo_index_path(repo_root); + if index_path.exists() { + match self.load_repo_index(repo_root, &index_path) { + Ok(_) => return, + Err(err) => { + crate::log_warn!( + "Failed to read source index {} (falling back to TOML scan): {}", + index_path.display(), + err + ); + } + } + } + self.scan_spec_tree(repo_root); + } + + fn load_repo_index(&mut self, repo_root: &Path, index_path: &Path) -> Result<()> { + let content = fs::read_to_string(index_path) + .with_context(|| format!("Failed to read {}", index_path.display()))?; + let mut lines = content.lines(); + let header = lines + .next() + .ok_or_else(|| anyhow::anyhow!("Missing source index header"))?; + if header.trim() != SOURCE_REPO_INDEX_HEADER { + anyhow::bail!( + "Unsupported source index header '{}' in {}", + header, + index_path.display() + ); + } + + for (idx, raw) in lines.enumerate() { + let line_no = idx + 2; + let line = raw.trim_end_matches('\r'); + if line.is_empty() || line.starts_with('#') { + continue; + } + + let mut parts = line.splitn(3, '\t'); + let kind = parts.next().unwrap_or_default(); + let name = parts.next().ok_or_else(|| { + anyhow::anyhow!("Malformed line {} in {}", line_no, index_path.display()) + })?; + let rel = parts.next().ok_or_else(|| { + anyhow::anyhow!("Malformed line {} in {}", line_no, index_path.display()) + })?; + if name.is_empty() || rel.is_empty() { + anyhow::bail!("Malformed line {} in {}", line_no, index_path.display()); + } + let path = repo_root.join(rel); + match kind { + SOURCE_REPO_INDEX_KIND_PACKAGE => { + self.by_name.insert(name.to_string(), path); + } + SOURCE_REPO_INDEX_KIND_PROVIDES => { + self.by_provides + .entry(name.to_string()) + .or_default() + .push(path); + } + _ => { + anyhow::bail!( + "Unknown source index row type '{}' on line {} in {}", + kind, + line_no, + index_path.display() + ); + } + } + } + + Ok(()) + } + + fn scan_spec_tree(&mut self, root: &Path) { + if !root.exists() { + return; + } + + let mut paths = Vec::new(); + for entry in WalkDir::new(root) + .follow_links(false) + .into_iter() + .filter_entry(|entry| entry.file_name() != OsStr::new(".git")) + .flatten() + { + if !entry.file_type().is_file() { + continue; + } + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) == Some("toml") { + paths.push(path.to_path_buf()); + } + } + paths.sort(); + + for path in paths { + if let Ok(spec) = PackageSpec::from_file(&path) { + self.add_spec(&spec, path); + } + } + } + + fn add_spec(&mut self, spec: &PackageSpec, path: PathBuf) { + self.by_name.insert(spec.package.name.clone(), path.clone()); + for provided in &spec.alternatives.provides { + self.by_provides + .entry(provided.clone()) + .or_default() + .push(path.clone()); + } + } + /// Find a spec by package name or provides pub fn find(&self, name: &str) -> Option { // First try by name @@ -163,3 +402,146 @@ impl PackageIndex { hits } } + +fn resolve_scan_roots(repo_root: &Path, subdirs: &[String]) -> Result> { + if subdirs.is_empty() { + return Ok(vec![repo_root.to_path_buf()]); + } + + let mut out = Vec::new(); + for subdir in subdirs { + let trimmed = subdir.trim(); + if trimmed.is_empty() { + anyhow::bail!("Subdirectory entries cannot be empty"); + } + let rel = Path::new(trimmed); + if rel.is_absolute() || rel.components().any(|c| c == Component::ParentDir) { + anyhow::bail!( + "Subdirectory '{}' must be a relative path without '..'", + trimmed + ); + } + let abs = repo_root.join(rel); + if !abs.is_dir() { + anyhow::bail!("Subdirectory not found: {}", abs.display()); + } + out.push(abs); + } + out.sort(); + out.dedup(); + Ok(out) +} + +fn scan_toml_files(scan_roots: &[PathBuf]) -> Result> { + let mut paths = Vec::new(); + for root in scan_roots { + for entry in WalkDir::new(root) + .follow_links(false) + .into_iter() + .filter_entry(|entry| entry.file_name() != OsStr::new(".git")) + { + let entry = entry.with_context(|| { + format!("Failed walking repository tree under {}", root.display()) + })?; + if !entry.file_type().is_file() { + continue; + } + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) == Some("toml") { + paths.push(path.to_path_buf()); + } + } + } + paths.sort(); + paths.dedup(); + Ok(paths) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write_meta_spec(path: &Path, name: &str, provides: &[&str]) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + let provides_line = if provides.is_empty() { + String::new() + } else { + let quoted = provides + .iter() + .map(|p| format!("\"{p}\"")) + .collect::>() + .join(", "); + format!("[alternatives]\nprovides = [{quoted}]\n\n") + }; + let content = format!( + "[package]\nname = \"{name}\"\nversion = \"1.0.0\"\ndescription = \"test\"\nhomepage = \"https://example.com\"\nlicense = \"MIT\"\n\n{provides_line}[build]\ntype = \"meta\"\n" + ); + std::fs::write(path, content).unwrap(); + } + + #[test] + fn create_source_repo_index_and_load_it() { + let tmp = tempfile::tempdir().unwrap(); + let repo_root = tmp.path().join("depot"); + let spec_path = repo_root.join("core/hello/hello.toml"); + write_meta_spec(&spec_path, "hello", &["sh"]); + + let stats = create_source_repo_index(&repo_root, &["core".to_string()]).unwrap(); + assert_eq!(stats.specs_indexed, 1); + assert_eq!(stats.package_rows, 1); + assert_eq!(stats.provides_rows, 1); + assert!(stats.index_path.exists()); + + let index = PackageIndex::build_with_repo_dir(Some(repo_root.clone())); + let hit = index.find("hello").expect("package name should resolve"); + assert!(hit.ends_with(Path::new("core/hello/hello.toml"))); + let providers = index.find_providers("sh"); + assert_eq!(providers.len(), 1); + assert!(providers[0].ends_with(Path::new("core/hello/hello.toml"))); + } + + #[test] + fn build_with_repo_dir_falls_back_when_index_is_missing() { + let tmp = tempfile::tempdir().unwrap(); + let repo_store = tmp.path().join("repo-store"); + let repo_root = repo_store.join("vertex"); + let spec_path = repo_root.join("core/base/base.toml"); + write_meta_spec(&spec_path, "base", &[]); + + let index = PackageIndex::build_with_repo_dir(Some(repo_store)); + let hit = index + .find("base") + .expect("package should be found by fallback TOML scanning"); + assert_eq!(hit, spec_path.canonicalize().unwrap()); + } + + #[test] + fn malformed_index_file_falls_back_to_toml_scan() { + let tmp = tempfile::tempdir().unwrap(); + let repo_root = tmp.path().join("depot"); + let spec_path = repo_root.join("core/curl/curl.toml"); + write_meta_spec(&spec_path, "curl", &["libcurl"]); + + std::fs::create_dir_all(&repo_root).unwrap(); + std::fs::write(source_repo_index_path(&repo_root), "invalid-header\n").unwrap(); + + let index = PackageIndex::build_with_repo_dir(Some(repo_root)); + let hit = index + .find("curl") + .expect("fallback should still find package spec"); + assert_eq!(hit, spec_path.canonicalize().unwrap()); + } + + #[test] + fn create_source_repo_index_rejects_parent_subdir() { + let tmp = tempfile::tempdir().unwrap(); + let err = create_source_repo_index(tmp.path(), &["../core".to_string()]) + .expect_err("unsafe subdir should be rejected"); + assert!( + err.to_string() + .contains("must be a relative path without '..'") + ); + } +}