repo: add source repo root index command and fast TSV index loading

- add `depot repo index [DIR] [--subdir ...]` to generate `depot-index.tsv`
- use deterministic, git-friendly TSV format with package/provides rows
- teach source lookup to prefer repo index files and fall back to TOML scanning
- add index generation/loading/fallback tests
- document new command in README

README.md

@@ -55,6 +55,7 @@ depot install zlib-1.2.11-1-x86_64.depot.pkg.tar.zst
 - `list`: List all installed packages.
 - `repo owns <PATH>`: Query binary repo metadata for path ownership.
 - `repo create [DIR]`: Create a repository database from a directory of packages.
+- `repo index [DIR] [--subdir <NAME> ...]`: Create/update `depot-index.tsv` at a source repo root for fast source lookup.
 - `config`: Show current configuration and overrides.
 
 ## Package Specification

src/cli.rs

@@ -142,6 +142,15 @@ pub enum RepoCommands {
         /// Update only one source repo by name
         name: Option<String>,
     },
+    /// Create/update a source index at the root of a source repo
+    Index {
+        /// Source repository root directory
+        #[arg(default_value = ".")]
+        dir: PathBuf,
+        /// Optional subdirectory to scan (repeatable, e.g. --subdir core --subdir extra)
+        #[arg(long = "subdir")]
+        subdirs: Vec<String>,
+    },
     /// List configured source and binary repos
     List,
     /// Add or update a repo entry in /etc/depot.d/repos.toml

src/commands.rs

@@ -1670,6 +1670,28 @@ pub fn run(cli: Cli) -> Result<()> {
                     }
                 }
             }
+            RepoCommands::Index { dir, subdirs } => {
+                let cfg = config::Config::for_rootfs(&cli.rootfs);
+                let mut repo_lock = locking::open_lock(&cfg)?;
+                let repo_lock_path = locking::lock_path(&cfg);
+                let _repo_lock_guard =
+                    locking::try_write(&mut repo_lock, &repo_lock_path, "repo index")?;
+                let stats = index::create_source_repo_index(&dir, &subdirs).with_context(|| {
+                    format!("Failed to create source index for {}", dir.display())
+                })?;
+                ui::success(format!(
+                    "Wrote source index: {}",
+                    stats.index_path.display()
+                ));
+                ui::info(format!(
+                    "Indexed {} spec(s) from {} TOML file(s): package rows={} provides rows={} ignored_toml={}",
+                    stats.specs_indexed,
+                    stats.toml_files_scanned,
+                    stats.package_rows,
+                    stats.provides_rows,
+                    stats.ignored_toml_files
+                ));
+            }
             RepoCommands::List => {
                 let config = config::Config::for_rootfs(&cli.rootfs);
                 let repo_lock = locking::open_lock(&config)?;

src/index.rs

@@ -3,9 +3,35 @@
 //! Caches package name -> spec path and provides -> spec path mappings.
 
 use crate::package::PackageSpec;
+use anyhow::{Context, Result};
 use std::collections::HashMap;
+use std::ffi::OsStr;
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Component, Path, PathBuf};
+use walkdir::WalkDir;
+
+/// Filename for the source-repo index written at a repo root.
+pub const SOURCE_REPO_INDEX_FILENAME: &str = "depot-index.tsv";
+const SOURCE_REPO_INDEX_HEADER: &str = "depot-source-index-v1";
+const SOURCE_REPO_INDEX_KIND_PACKAGE: &str = "P";
+const SOURCE_REPO_INDEX_KIND_PROVIDES: &str = "V";
+
+/// Statistics for generating a source repository index file.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SourceRepoIndexStats {
+    /// Absolute path to the written index file.
+    pub index_path: PathBuf,
+    /// Number of `.toml` files discovered under the selected scan roots.
+    pub toml_files_scanned: usize,
+    /// Number of discovered TOML files that parsed as valid package specs.
+    pub specs_indexed: usize,
+    /// Number of package-name rows written to the index.
+    pub package_rows: usize,
+    /// Number of provides rows written to the index.
+    pub provides_rows: usize,
+    /// Number of discovered TOML files ignored because they were not package specs.
+    pub ignored_toml_files: usize,
+}
 
 /// Cached package index for O(1) lookups
 #[derive(Debug, Default)]
@@ -24,6 +50,126 @@ pub struct SourceSearchHit {
     pub provides: Vec<String>,
 }
 
+/// Return the source index file path for `repo_root`.
+pub fn source_repo_index_path(repo_root: &Path) -> PathBuf {
+    repo_root.join(SOURCE_REPO_INDEX_FILENAME)
+}
+
+/// Create/update the source-repo package index at `repo_root`.
+///
+/// The file format is line-based TSV with deterministic ordering:
+/// - Header: `depot-source-index-v1`
+/// - Package name row: `P<TAB><name><TAB><relative-spec-path>`
+/// - Provides row: `V<TAB><feature><TAB><relative-spec-path>`
+pub fn create_source_repo_index(
+    repo_root: &Path,
+    subdirs: &[String],
+) -> Result<SourceRepoIndexStats> {
+    let repo_root = repo_root
+        .canonicalize()
+        .with_context(|| format!("Failed to resolve repo root {}", repo_root.display()))?;
+    if !repo_root.is_dir() {
+        anyhow::bail!("Repo root is not a directory: {}", repo_root.display());
+    }
+
+    let scan_roots = resolve_scan_roots(&repo_root, subdirs)?;
+    let mut spec_rows: Vec<(String, String, Vec<String>)> = Vec::new();
+    let mut toml_files_scanned = 0usize;
+    let mut ignored_toml_files = 0usize;
+
+    for spec_path in scan_toml_files(&scan_roots)? {
+        toml_files_scanned += 1;
+        let spec = match PackageSpec::from_file(&spec_path) {
+            Ok(spec) => spec,
+            Err(_) => {
+                ignored_toml_files += 1;
+                continue;
+            }
+        };
+
+        let rel = spec_path.strip_prefix(&repo_root).with_context(|| {
+            format!(
+                "Failed to compute relative path for {} from {}",
+                spec_path.display(),
+                repo_root.display()
+            )
+        })?;
+        let rel = rel.to_string_lossy().replace('\\', "/");
+        spec_rows.push((spec.package.name.clone(), rel, spec.alternatives.provides));
+    }
+
+    let mut rows: Vec<(String, String, String)> = Vec::new();
+    for (name, rel, provides) in &spec_rows {
+        rows.push((
+            SOURCE_REPO_INDEX_KIND_PACKAGE.to_string(),
+            name.clone(),
+            rel.clone(),
+        ));
+        for provided in provides {
+            rows.push((
+                SOURCE_REPO_INDEX_KIND_PROVIDES.to_string(),
+                provided.clone(),
+                rel.clone(),
+            ));
+        }
+    }
+    rows.sort_by(|a, b| {
+        a.0.cmp(&b.0)
+            .then_with(|| a.1.cmp(&b.1))
+            .then_with(|| a.2.cmp(&b.2))
+    });
+
+    let mut out = String::new();
+    out.push_str(SOURCE_REPO_INDEX_HEADER);
+    out.push('\n');
+    for (kind, name, rel) in &rows {
+        if name.contains('\n') || name.contains('\r') || name.contains('\t') {
+            anyhow::bail!(
+                "Index field contains unsupported control character: {}",
+                name
+            );
+        }
+        if rel.contains('\n') || rel.contains('\r') || rel.contains('\t') {
+            anyhow::bail!("Index path contains unsupported control character: {}", rel);
+        }
+        out.push_str(kind);
+        out.push('\t');
+        out.push_str(name);
+        out.push('\t');
+        out.push_str(rel);
+        out.push('\n');
+    }
+
+    let index_path = source_repo_index_path(&repo_root);
+    let tmp_path = index_path.with_extension("tsv.tmp");
+    fs::write(&tmp_path, out).with_context(|| format!("Failed to write {}", tmp_path.display()))?;
+    fs::rename(&tmp_path, &index_path).with_context(|| {
+        format!(
+            "Failed to replace index {} from {}",
+            index_path.display(),
+            tmp_path.display()
+        )
+    })?;
+
+    let package_rows = rows
+        .iter()
+        .filter(|(kind, _, _)| kind == SOURCE_REPO_INDEX_KIND_PACKAGE)
+        .count();
+    let provides_rows = rows
+        .iter()
+        .filter(|(kind, _, _)| kind == SOURCE_REPO_INDEX_KIND_PROVIDES)
+        .count();
+
+    Ok(SourceRepoIndexStats {
+        index_path,
+        toml_files_scanned,
+        specs_indexed: spec_rows.len(),
+        package_rows,
+        provides_rows,
+        ignored_toml_files,
+    })
+}
+
 impl PackageIndex {
     /// Build index by scanning packages/*/*.toml and configured repo dir.
     ///
@@ -35,62 +181,11 @@ impl PackageIndex {
         let mut index = Self::default();
         let packages_dir = PathBuf::from("packages");
 
-        // Scan local packages/<pkg>/*.toml
+        index.scan_spec_tree(&packages_dir);
-        if let Ok(entries) = fs::read_dir(&packages_dir) {
-            for entry in entries.flatten() {
-                if !entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
-                    continue;
-                }
-                let dir = entry.path();
 
-                if let Ok(files) = fs::read_dir(&dir) {
-                    for file in files.flatten() {
-                        let path = file.path();
-                        if path.extension().map(|e| e == "toml").unwrap_or(false)
-                            && let Ok(spec) = PackageSpec::from_file(&path)
-                        {
-                            index
-                                .by_name
-                                .insert(spec.package.name.clone(), path.clone());
-                            for provided in &spec.alternatives.provides {
-                                index
-                                    .by_provides
-                                    .entry(provided.clone())
-                                    .or_default()
-                                    .push(path.clone());
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        // Also scan system mirrors under provided repo_dir (or default)
         let sys_dir = repo_dir.unwrap_or_else(|| PathBuf::from("/usr/src/depot"));
-
         if sys_dir.exists() {
-            for entry in walkdir::WalkDir::new(&sys_dir)
+            index.scan_repo_store(&sys_dir);
-                .min_depth(1)
-                .max_depth(5)
-                .into_iter()
-                .flatten()
-            {
-                let path = entry.path().to_path_buf();
-                if path.extension().map(|e| e == "toml").unwrap_or(false)
-                    && let Ok(spec) = PackageSpec::from_file(&path)
-                {
-                    index
-                        .by_name
-                        .insert(spec.package.name.clone(), path.clone());
-                    for provided in &spec.alternatives.provides {
-                        index
-                            .by_provides
-                            .entry(provided.clone())
-                            .or_default()
-                            .push(path.clone());
-                    }
-                }
-            }
         }
 
         crate::log_info!(
@@ -102,6 +197,150 @@ impl PackageIndex {
         index
     }
 
+    fn scan_repo_store(&mut self, sys_dir: &Path) {
+        let direct_index = source_repo_index_path(sys_dir);
+        if direct_index.exists() || sys_dir.join(".git").is_dir() {
+            self.scan_repo_root(sys_dir);
+            return;
+        }
+
+        let mut child_dirs = Vec::new();
+        if let Ok(entries) = fs::read_dir(sys_dir) {
+            for entry in entries.flatten() {
+                let Ok(ft) = entry.file_type() else {
+                    continue;
+                };
+                if ft.is_dir() {
+                    child_dirs.push(entry.path());
+                }
+            }
+        }
+
+        if child_dirs.is_empty() {
+            self.scan_spec_tree(sys_dir);
+            return;
+        }
+
+        child_dirs.sort();
+        for child in child_dirs {
+            self.scan_repo_root(&child);
+        }
+    }
+
+    fn scan_repo_root(&mut self, repo_root: &Path) {
+        let index_path = source_repo_index_path(repo_root);
+        if index_path.exists() {
+            match self.load_repo_index(repo_root, &index_path) {
+                Ok(_) => return,
+                Err(err) => {
+                    crate::log_warn!(
+                        "Failed to read source index {} (falling back to TOML scan): {}",
+                        index_path.display(),
+                        err
+                    );
+                }
+            }
+        }
+        self.scan_spec_tree(repo_root);
+    }
+
+    fn load_repo_index(&mut self, repo_root: &Path, index_path: &Path) -> Result<()> {
+        let content = fs::read_to_string(index_path)
+            .with_context(|| format!("Failed to read {}", index_path.display()))?;
+        let mut lines = content.lines();
+        let header = lines
+            .next()
+            .ok_or_else(|| anyhow::anyhow!("Missing source index header"))?;
+        if header.trim() != SOURCE_REPO_INDEX_HEADER {
+            anyhow::bail!(
+                "Unsupported source index header '{}' in {}",
+                header,
+                index_path.display()
+            );
+        }
+
+        for (idx, raw) in lines.enumerate() {
+            let line_no = idx + 2;
+            let line = raw.trim_end_matches('\r');
+            if line.is_empty() || line.starts_with('#') {
+                continue;
+            }
+
+            let mut parts = line.splitn(3, '\t');
+            let kind = parts.next().unwrap_or_default();
+            let name = parts.next().ok_or_else(|| {
+                anyhow::anyhow!("Malformed line {} in {}", line_no, index_path.display())
+            })?;
+            let rel = parts.next().ok_or_else(|| {
+                anyhow::anyhow!("Malformed line {} in {}", line_no, index_path.display())
+            })?;
+            if name.is_empty() || rel.is_empty() {
+                anyhow::bail!("Malformed line {} in {}", line_no, index_path.display());
+            }
+            let path = repo_root.join(rel);
+            match kind {
+                SOURCE_REPO_INDEX_KIND_PACKAGE => {
+                    self.by_name.insert(name.to_string(), path);
+                }
+                SOURCE_REPO_INDEX_KIND_PROVIDES => {
+                    self.by_provides
+                        .entry(name.to_string())
+                        .or_default()
+                        .push(path);
+                }
+                _ => {
+                    anyhow::bail!(
+                        "Unknown source index row type '{}' on line {} in {}",
+                        kind,
+                        line_no,
+                        index_path.display()
+                    );
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn scan_spec_tree(&mut self, root: &Path) {
+        if !root.exists() {
+            return;
+        }
+
+        let mut paths = Vec::new();
+        for entry in WalkDir::new(root)
+            .follow_links(false)
+            .into_iter()
+            .filter_entry(|entry| entry.file_name() != OsStr::new(".git"))
+            .flatten()
+        {
+            if !entry.file_type().is_file() {
+                continue;
+            }
+            let path = entry.path();
+            if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
+                paths.push(path.to_path_buf());
+            }
+        }
+        paths.sort();
+
+        for path in paths {
+            if let Ok(spec) = PackageSpec::from_file(&path) {
+                self.add_spec(&spec, path);
+            }
+        }
+    }
+
+    fn add_spec(&mut self, spec: &PackageSpec, path: PathBuf) {
+        self.by_name.insert(spec.package.name.clone(), path.clone());
+        for provided in &spec.alternatives.provides {
+            self.by_provides
+                .entry(provided.clone())
+                .or_default()
+                .push(path.clone());
+        }
+    }
+
     /// Find a spec by package name or provides
     pub fn find(&self, name: &str) -> Option<PathBuf> {
         // First try by name
@@ -163,3 +402,146 @@ impl PackageIndex {
         hits
     }
 }
+
+fn resolve_scan_roots(repo_root: &Path, subdirs: &[String]) -> Result<Vec<PathBuf>> {
+    if subdirs.is_empty() {
+        return Ok(vec![repo_root.to_path_buf()]);
+    }
+
+    let mut out = Vec::new();
+    for subdir in subdirs {
+        let trimmed = subdir.trim();
+        if trimmed.is_empty() {
+            anyhow::bail!("Subdirectory entries cannot be empty");
+        }
+        let rel = Path::new(trimmed);
+        if rel.is_absolute() || rel.components().any(|c| c == Component::ParentDir) {
+            anyhow::bail!(
+                "Subdirectory '{}' must be a relative path without '..'",
+                trimmed
+            );
+        }
+        let abs = repo_root.join(rel);
+        if !abs.is_dir() {
+            anyhow::bail!("Subdirectory not found: {}", abs.display());
+        }
+        out.push(abs);
+    }
+    out.sort();
+    out.dedup();
+    Ok(out)
+}
+
+fn scan_toml_files(scan_roots: &[PathBuf]) -> Result<Vec<PathBuf>> {
+    let mut paths = Vec::new();
+    for root in scan_roots {
+        for entry in WalkDir::new(root)
+            .follow_links(false)
+            .into_iter()
+            .filter_entry(|entry| entry.file_name() != OsStr::new(".git"))
+        {
+            let entry = entry.with_context(|| {
+                format!("Failed walking repository tree under {}", root.display())
+            })?;
+            if !entry.file_type().is_file() {
+                continue;
+            }
+            let path = entry.path();
+            if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
+                paths.push(path.to_path_buf());
+            }
+        }
+    }
+    paths.sort();
+    paths.dedup();
+    Ok(paths)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn write_meta_spec(path: &Path, name: &str, provides: &[&str]) {
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent).unwrap();
+        }
+        let provides_line = if provides.is_empty() {
+            String::new()
+        } else {
+            let quoted = provides
+                .iter()
+                .map(|p| format!("\"{p}\""))
+                .collect::<Vec<_>>()
+                .join(", ");
+            format!("[alternatives]\nprovides = [{quoted}]\n\n")
+        };
+        let content = format!(
+            "[package]\nname = \"{name}\"\nversion = \"1.0.0\"\ndescription = \"test\"\nhomepage = \"https://example.com\"\nlicense = \"MIT\"\n\n{provides_line}[build]\ntype = \"meta\"\n"
+        );
+        std::fs::write(path, content).unwrap();
+    }
+
+    #[test]
+    fn create_source_repo_index_and_load_it() {
+        let tmp = tempfile::tempdir().unwrap();
+        let repo_root = tmp.path().join("depot");
+        let spec_path = repo_root.join("core/hello/hello.toml");
+        write_meta_spec(&spec_path, "hello", &["sh"]);
+
+        let stats = create_source_repo_index(&repo_root, &["core".to_string()]).unwrap();
+        assert_eq!(stats.specs_indexed, 1);
+        assert_eq!(stats.package_rows, 1);
+        assert_eq!(stats.provides_rows, 1);
+        assert!(stats.index_path.exists());
+
+        let index = PackageIndex::build_with_repo_dir(Some(repo_root.clone()));
+        let hit = index.find("hello").expect("package name should resolve");
+        assert!(hit.ends_with(Path::new("core/hello/hello.toml")));
+        let providers = index.find_providers("sh");
+        assert_eq!(providers.len(), 1);
+        assert!(providers[0].ends_with(Path::new("core/hello/hello.toml")));
+    }
+
+    #[test]
+    fn build_with_repo_dir_falls_back_when_index_is_missing() {
+        let tmp = tempfile::tempdir().unwrap();
+        let repo_store = tmp.path().join("repo-store");
+        let repo_root = repo_store.join("vertex");
+        let spec_path = repo_root.join("core/base/base.toml");
+        write_meta_spec(&spec_path, "base", &[]);
+
+        let index = PackageIndex::build_with_repo_dir(Some(repo_store));
+        let hit = index
+            .find("base")
+            .expect("package should be found by fallback TOML scanning");
+        assert_eq!(hit, spec_path.canonicalize().unwrap());
+    }
+
+    #[test]
+    fn malformed_index_file_falls_back_to_toml_scan() {
+        let tmp = tempfile::tempdir().unwrap();
+        let repo_root = tmp.path().join("depot");
+        let spec_path = repo_root.join("core/curl/curl.toml");
+        write_meta_spec(&spec_path, "curl", &["libcurl"]);
+
+        std::fs::create_dir_all(&repo_root).unwrap();
+        std::fs::write(source_repo_index_path(&repo_root), "invalid-header\n").unwrap();
+
+        let index = PackageIndex::build_with_repo_dir(Some(repo_root));
+        let hit = index
+            .find("curl")
+            .expect("fallback should still find package spec");
+        assert_eq!(hit, spec_path.canonicalize().unwrap());
+    }
+
+    #[test]
+    fn create_source_repo_index_rejects_parent_subdir() {
+        let tmp = tempfile::tempdir().unwrap();
+        let err = create_source_repo_index(tmp.path(), &["../core".to_string()])
+            .expect_err("unsafe subdir should be rejected");
+        assert!(
+            err.to_string()
+                .contains("must be a relative path without '..'")
+        );
+    }
+}