repo: add source repo root index command and fast TSV index loading

- add `depot repo index [DIR] [--subdir ...]` to generate `depot-index.tsv`
- use deterministic, git-friendly TSV format with package/provides rows
- teach source lookup to prefer repo index files and fall back to TOML scanning
- add index generation/loading/fallback tests
- document new command in README
This commit is contained in:
2026-02-28 15:04:24 -06:00
parent 421ca362ac
commit 1f22d22887
4 changed files with 468 additions and 54 deletions
+1
View File
@@ -55,6 +55,7 @@ depot install zlib-1.2.11-1-x86_64.depot.pkg.tar.zst
- `list`: List all installed packages.
- `repo owns <PATH>`: Query binary repo metadata for path ownership.
- `repo create [DIR]`: Create a repository database from a directory of packages.
- `repo index [DIR] [--subdir <NAME> ...]`: Create/update `depot-index.tsv` at a source repo root for fast source lookup.
- `config`: Show current configuration and overrides.
## Package Specification
+9
View File
@@ -142,6 +142,15 @@ pub enum RepoCommands {
/// Update only one source repo by name
name: Option<String>,
},
/// Create/update a source index at the root of a source repo
Index {
/// Source repository root directory
#[arg(default_value = ".")]
dir: PathBuf,
/// Optional subdirectory to scan (repeatable, e.g. --subdir core --subdir extra)
#[arg(long = "subdir")]
subdirs: Vec<String>,
},
/// List configured source and binary repos
List,
/// Add or update a repo entry in /etc/depot.d/repos.toml
+22
View File
@@ -1670,6 +1670,28 @@ pub fn run(cli: Cli) -> Result<()> {
}
}
}
RepoCommands::Index { dir, subdirs } => {
let cfg = config::Config::for_rootfs(&cli.rootfs);
let mut repo_lock = locking::open_lock(&cfg)?;
let repo_lock_path = locking::lock_path(&cfg);
let _repo_lock_guard =
locking::try_write(&mut repo_lock, &repo_lock_path, "repo index")?;
let stats = index::create_source_repo_index(&dir, &subdirs).with_context(|| {
format!("Failed to create source index for {}", dir.display())
})?;
ui::success(format!(
"Wrote source index: {}",
stats.index_path.display()
));
ui::info(format!(
"Indexed {} spec(s) from {} TOML file(s): package rows={} provides rows={} ignored_toml={}",
stats.specs_indexed,
stats.toml_files_scanned,
stats.package_rows,
stats.provides_rows,
stats.ignored_toml_files
));
}
RepoCommands::List => {
let config = config::Config::for_rootfs(&cli.rootfs);
let repo_lock = locking::open_lock(&config)?;
+436 -54
View File
@@ -3,9 +3,35 @@
//! Caches package name -> spec path and provides -> spec path mappings.
use crate::package::PackageSpec;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs;
use std::path::PathBuf;
use std::path::{Component, Path, PathBuf};
use walkdir::WalkDir;
/// Filename for the source-repo index written at a repo root.
pub const SOURCE_REPO_INDEX_FILENAME: &str = "depot-index.tsv";
const SOURCE_REPO_INDEX_HEADER: &str = "depot-source-index-v1";
const SOURCE_REPO_INDEX_KIND_PACKAGE: &str = "P";
const SOURCE_REPO_INDEX_KIND_PROVIDES: &str = "V";
/// Statistics for generating a source repository index file.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourceRepoIndexStats {
/// Absolute path to the written index file.
pub index_path: PathBuf,
/// Number of `.toml` files discovered under the selected scan roots.
pub toml_files_scanned: usize,
/// Number of discovered TOML files that parsed as valid package specs.
pub specs_indexed: usize,
/// Number of package-name rows written to the index.
pub package_rows: usize,
/// Number of provides rows written to the index.
pub provides_rows: usize,
/// Number of discovered TOML files ignored because they were not package specs.
pub ignored_toml_files: usize,
}
/// Cached package index for O(1) lookups
#[derive(Debug, Default)]
@@ -24,6 +50,126 @@ pub struct SourceSearchHit {
pub provides: Vec<String>,
}
/// Return the source index file path for `repo_root`.
pub fn source_repo_index_path(repo_root: &Path) -> PathBuf {
repo_root.join(SOURCE_REPO_INDEX_FILENAME)
}
/// Create/update the source-repo package index at `repo_root`.
///
/// The file format is line-based TSV with deterministic ordering:
/// - Header: `depot-source-index-v1`
/// - Package name row: `P<TAB><name><TAB><relative-spec-path>`
/// - Provides row: `V<TAB><feature><TAB><relative-spec-path>`
pub fn create_source_repo_index(
repo_root: &Path,
subdirs: &[String],
) -> Result<SourceRepoIndexStats> {
let repo_root = repo_root
.canonicalize()
.with_context(|| format!("Failed to resolve repo root {}", repo_root.display()))?;
if !repo_root.is_dir() {
anyhow::bail!("Repo root is not a directory: {}", repo_root.display());
}
let scan_roots = resolve_scan_roots(&repo_root, subdirs)?;
let mut spec_rows: Vec<(String, String, Vec<String>)> = Vec::new();
let mut toml_files_scanned = 0usize;
let mut ignored_toml_files = 0usize;
for spec_path in scan_toml_files(&scan_roots)? {
toml_files_scanned += 1;
let spec = match PackageSpec::from_file(&spec_path) {
Ok(spec) => spec,
Err(_) => {
ignored_toml_files += 1;
continue;
}
};
let rel = spec_path.strip_prefix(&repo_root).with_context(|| {
format!(
"Failed to compute relative path for {} from {}",
spec_path.display(),
repo_root.display()
)
})?;
let rel = rel.to_string_lossy().replace('\\', "/");
spec_rows.push((spec.package.name.clone(), rel, spec.alternatives.provides));
}
let mut rows: Vec<(String, String, String)> = Vec::new();
for (name, rel, provides) in &spec_rows {
rows.push((
SOURCE_REPO_INDEX_KIND_PACKAGE.to_string(),
name.clone(),
rel.clone(),
));
for provided in provides {
rows.push((
SOURCE_REPO_INDEX_KIND_PROVIDES.to_string(),
provided.clone(),
rel.clone(),
));
}
}
rows.sort_by(|a, b| {
a.0.cmp(&b.0)
.then_with(|| a.1.cmp(&b.1))
.then_with(|| a.2.cmp(&b.2))
});
let mut out = String::new();
out.push_str(SOURCE_REPO_INDEX_HEADER);
out.push('\n');
for (kind, name, rel) in &rows {
if name.contains('\n') || name.contains('\r') || name.contains('\t') {
anyhow::bail!(
"Index field contains unsupported control character: {}",
name
);
}
if rel.contains('\n') || rel.contains('\r') || rel.contains('\t') {
anyhow::bail!("Index path contains unsupported control character: {}", rel);
}
out.push_str(kind);
out.push('\t');
out.push_str(name);
out.push('\t');
out.push_str(rel);
out.push('\n');
}
let index_path = source_repo_index_path(&repo_root);
let tmp_path = index_path.with_extension("tsv.tmp");
fs::write(&tmp_path, out).with_context(|| format!("Failed to write {}", tmp_path.display()))?;
fs::rename(&tmp_path, &index_path).with_context(|| {
format!(
"Failed to replace index {} from {}",
index_path.display(),
tmp_path.display()
)
})?;
let package_rows = rows
.iter()
.filter(|(kind, _, _)| kind == SOURCE_REPO_INDEX_KIND_PACKAGE)
.count();
let provides_rows = rows
.iter()
.filter(|(kind, _, _)| kind == SOURCE_REPO_INDEX_KIND_PROVIDES)
.count();
Ok(SourceRepoIndexStats {
index_path,
toml_files_scanned,
specs_indexed: spec_rows.len(),
package_rows,
provides_rows,
ignored_toml_files,
})
}
impl PackageIndex {
/// Build index by scanning packages/*/*.toml and configured repo dir.
///
@@ -35,62 +181,11 @@ impl PackageIndex {
let mut index = Self::default();
let packages_dir = PathBuf::from("packages");
// Scan local packages/<pkg>/*.toml
if let Ok(entries) = fs::read_dir(&packages_dir) {
for entry in entries.flatten() {
if !entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
continue;
}
let dir = entry.path();
index.scan_spec_tree(&packages_dir);
if let Ok(files) = fs::read_dir(&dir) {
for file in files.flatten() {
let path = file.path();
if path.extension().map(|e| e == "toml").unwrap_or(false)
&& let Ok(spec) = PackageSpec::from_file(&path)
{
index
.by_name
.insert(spec.package.name.clone(), path.clone());
for provided in &spec.alternatives.provides {
index
.by_provides
.entry(provided.clone())
.or_default()
.push(path.clone());
}
}
}
}
}
}
// Also scan system mirrors under provided repo_dir (or default)
let sys_dir = repo_dir.unwrap_or_else(|| PathBuf::from("/usr/src/depot"));
if sys_dir.exists() {
for entry in walkdir::WalkDir::new(&sys_dir)
.min_depth(1)
.max_depth(5)
.into_iter()
.flatten()
{
let path = entry.path().to_path_buf();
if path.extension().map(|e| e == "toml").unwrap_or(false)
&& let Ok(spec) = PackageSpec::from_file(&path)
{
index
.by_name
.insert(spec.package.name.clone(), path.clone());
for provided in &spec.alternatives.provides {
index
.by_provides
.entry(provided.clone())
.or_default()
.push(path.clone());
}
}
}
index.scan_repo_store(&sys_dir);
}
crate::log_info!(
@@ -102,6 +197,150 @@ impl PackageIndex {
index
}
fn scan_repo_store(&mut self, sys_dir: &Path) {
let direct_index = source_repo_index_path(sys_dir);
if direct_index.exists() || sys_dir.join(".git").is_dir() {
self.scan_repo_root(sys_dir);
return;
}
let mut child_dirs = Vec::new();
if let Ok(entries) = fs::read_dir(sys_dir) {
for entry in entries.flatten() {
let Ok(ft) = entry.file_type() else {
continue;
};
if ft.is_dir() {
child_dirs.push(entry.path());
}
}
}
if child_dirs.is_empty() {
self.scan_spec_tree(sys_dir);
return;
}
child_dirs.sort();
for child in child_dirs {
self.scan_repo_root(&child);
}
}
fn scan_repo_root(&mut self, repo_root: &Path) {
let index_path = source_repo_index_path(repo_root);
if index_path.exists() {
match self.load_repo_index(repo_root, &index_path) {
Ok(_) => return,
Err(err) => {
crate::log_warn!(
"Failed to read source index {} (falling back to TOML scan): {}",
index_path.display(),
err
);
}
}
}
self.scan_spec_tree(repo_root);
}
fn load_repo_index(&mut self, repo_root: &Path, index_path: &Path) -> Result<()> {
let content = fs::read_to_string(index_path)
.with_context(|| format!("Failed to read {}", index_path.display()))?;
let mut lines = content.lines();
let header = lines
.next()
.ok_or_else(|| anyhow::anyhow!("Missing source index header"))?;
if header.trim() != SOURCE_REPO_INDEX_HEADER {
anyhow::bail!(
"Unsupported source index header '{}' in {}",
header,
index_path.display()
);
}
for (idx, raw) in lines.enumerate() {
let line_no = idx + 2;
let line = raw.trim_end_matches('\r');
if line.is_empty() || line.starts_with('#') {
continue;
}
let mut parts = line.splitn(3, '\t');
let kind = parts.next().unwrap_or_default();
let name = parts.next().ok_or_else(|| {
anyhow::anyhow!("Malformed line {} in {}", line_no, index_path.display())
})?;
let rel = parts.next().ok_or_else(|| {
anyhow::anyhow!("Malformed line {} in {}", line_no, index_path.display())
})?;
if name.is_empty() || rel.is_empty() {
anyhow::bail!("Malformed line {} in {}", line_no, index_path.display());
}
let path = repo_root.join(rel);
match kind {
SOURCE_REPO_INDEX_KIND_PACKAGE => {
self.by_name.insert(name.to_string(), path);
}
SOURCE_REPO_INDEX_KIND_PROVIDES => {
self.by_provides
.entry(name.to_string())
.or_default()
.push(path);
}
_ => {
anyhow::bail!(
"Unknown source index row type '{}' on line {} in {}",
kind,
line_no,
index_path.display()
);
}
}
}
Ok(())
}
fn scan_spec_tree(&mut self, root: &Path) {
if !root.exists() {
return;
}
let mut paths = Vec::new();
for entry in WalkDir::new(root)
.follow_links(false)
.into_iter()
.filter_entry(|entry| entry.file_name() != OsStr::new(".git"))
.flatten()
{
if !entry.file_type().is_file() {
continue;
}
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
paths.push(path.to_path_buf());
}
}
paths.sort();
for path in paths {
if let Ok(spec) = PackageSpec::from_file(&path) {
self.add_spec(&spec, path);
}
}
}
fn add_spec(&mut self, spec: &PackageSpec, path: PathBuf) {
self.by_name.insert(spec.package.name.clone(), path.clone());
for provided in &spec.alternatives.provides {
self.by_provides
.entry(provided.clone())
.or_default()
.push(path.clone());
}
}
/// Find a spec by package name or provides
pub fn find(&self, name: &str) -> Option<PathBuf> {
// First try by name
@@ -163,3 +402,146 @@ impl PackageIndex {
hits
}
}
fn resolve_scan_roots(repo_root: &Path, subdirs: &[String]) -> Result<Vec<PathBuf>> {
if subdirs.is_empty() {
return Ok(vec![repo_root.to_path_buf()]);
}
let mut out = Vec::new();
for subdir in subdirs {
let trimmed = subdir.trim();
if trimmed.is_empty() {
anyhow::bail!("Subdirectory entries cannot be empty");
}
let rel = Path::new(trimmed);
if rel.is_absolute() || rel.components().any(|c| c == Component::ParentDir) {
anyhow::bail!(
"Subdirectory '{}' must be a relative path without '..'",
trimmed
);
}
let abs = repo_root.join(rel);
if !abs.is_dir() {
anyhow::bail!("Subdirectory not found: {}", abs.display());
}
out.push(abs);
}
out.sort();
out.dedup();
Ok(out)
}
fn scan_toml_files(scan_roots: &[PathBuf]) -> Result<Vec<PathBuf>> {
let mut paths = Vec::new();
for root in scan_roots {
for entry in WalkDir::new(root)
.follow_links(false)
.into_iter()
.filter_entry(|entry| entry.file_name() != OsStr::new(".git"))
{
let entry = entry.with_context(|| {
format!("Failed walking repository tree under {}", root.display())
})?;
if !entry.file_type().is_file() {
continue;
}
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
paths.push(path.to_path_buf());
}
}
}
paths.sort();
paths.dedup();
Ok(paths)
}
#[cfg(test)]
mod tests {
use super::*;
fn write_meta_spec(path: &Path, name: &str, provides: &[&str]) {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).unwrap();
}
let provides_line = if provides.is_empty() {
String::new()
} else {
let quoted = provides
.iter()
.map(|p| format!("\"{p}\""))
.collect::<Vec<_>>()
.join(", ");
format!("[alternatives]\nprovides = [{quoted}]\n\n")
};
let content = format!(
"[package]\nname = \"{name}\"\nversion = \"1.0.0\"\ndescription = \"test\"\nhomepage = \"https://example.com\"\nlicense = \"MIT\"\n\n{provides_line}[build]\ntype = \"meta\"\n"
);
std::fs::write(path, content).unwrap();
}
#[test]
fn create_source_repo_index_and_load_it() {
let tmp = tempfile::tempdir().unwrap();
let repo_root = tmp.path().join("depot");
let spec_path = repo_root.join("core/hello/hello.toml");
write_meta_spec(&spec_path, "hello", &["sh"]);
let stats = create_source_repo_index(&repo_root, &["core".to_string()]).unwrap();
assert_eq!(stats.specs_indexed, 1);
assert_eq!(stats.package_rows, 1);
assert_eq!(stats.provides_rows, 1);
assert!(stats.index_path.exists());
let index = PackageIndex::build_with_repo_dir(Some(repo_root.clone()));
let hit = index.find("hello").expect("package name should resolve");
assert!(hit.ends_with(Path::new("core/hello/hello.toml")));
let providers = index.find_providers("sh");
assert_eq!(providers.len(), 1);
assert!(providers[0].ends_with(Path::new("core/hello/hello.toml")));
}
#[test]
fn build_with_repo_dir_falls_back_when_index_is_missing() {
let tmp = tempfile::tempdir().unwrap();
let repo_store = tmp.path().join("repo-store");
let repo_root = repo_store.join("vertex");
let spec_path = repo_root.join("core/base/base.toml");
write_meta_spec(&spec_path, "base", &[]);
let index = PackageIndex::build_with_repo_dir(Some(repo_store));
let hit = index
.find("base")
.expect("package should be found by fallback TOML scanning");
assert_eq!(hit, spec_path.canonicalize().unwrap());
}
#[test]
fn malformed_index_file_falls_back_to_toml_scan() {
let tmp = tempfile::tempdir().unwrap();
let repo_root = tmp.path().join("depot");
let spec_path = repo_root.join("core/curl/curl.toml");
write_meta_spec(&spec_path, "curl", &["libcurl"]);
std::fs::create_dir_all(&repo_root).unwrap();
std::fs::write(source_repo_index_path(&repo_root), "invalid-header\n").unwrap();
let index = PackageIndex::build_with_repo_dir(Some(repo_root));
let hit = index
.find("curl")
.expect("fallback should still find package spec");
assert_eq!(hit, spec_path.canonicalize().unwrap());
}
#[test]
fn create_source_repo_index_rejects_parent_subdir() {
let tmp = tempfile::tempdir().unwrap();
let err = create_source_repo_index(tmp.path(), &["../core".to_string()])
.expect_err("unsafe subdir should be rejected");
assert!(
err.to_string()
.contains("must be a relative path without '..'")
);
}
}