From fbb8e83a0013a9def684c04a1138eae0a8389626 Mon Sep 17 00:00:00 2001 From: SFG545 Date: Wed, 25 Feb 2026 20:59:10 -0600 Subject: [PATCH] feat: add manual source handling in interactive prompts and support for file URLs --- src/db/repo.rs | 260 +++++++++++++++++++++++--------- src/package/interactive.rs | 301 +++++++++++++++++++++++++++++++------ 2 files changed, 451 insertions(+), 110 deletions(-) diff --git a/src/db/repo.rs b/src/db/repo.rs index 18f2934..977dd1b 100644 --- a/src/db/repo.rs +++ b/src/db/repo.rs @@ -349,6 +349,51 @@ fn join_repo_url(base: &str, rel: &str) -> Result { .to_string()) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FileUrlCopyOutcome { + NotFileUrl, + Copied, + Missing, +} + +fn copy_file_url_to_path(url: &str, dest: &Path) -> Result { + let parsed = match url::Url::parse(url) { + Ok(parsed) => parsed, + Err(_) => return Ok(FileUrlCopyOutcome::NotFileUrl), + }; + if parsed.scheme() != "file" { + return Ok(FileUrlCopyOutcome::NotFileUrl); + } + + let src = parsed + .to_file_path() + .map_err(|_| anyhow::anyhow!("Invalid file:// URL: {}", url))?; + if !src.exists() { + return Ok(FileUrlCopyOutcome::Missing); + } + if !src.is_file() { + anyhow::bail!("file:// URL is not a file: {}", src.display()); + } + + fs::copy(&src, dest) + .with_context(|| format!("Failed to copy {} to {}", src.display(), dest.display()))?; + Ok(FileUrlCopyOutcome::Copied) +} + +fn normalize_git_mirror_url(url: &str) -> Result { + let parsed = match url::Url::parse(url) { + Ok(parsed) => parsed, + Err(_) => return Ok(url.to_string()), + }; + if parsed.scheme() != "file" { + return Ok(url.to_string()); + } + let path = parsed + .to_file_path() + .map_err(|_| anyhow::anyhow!("Invalid file:// mirror URL: {}", url))?; + Ok(path.to_string_lossy().into_owned()) +} + fn decompress_zstd_file(src: &Path, dst: &Path) -> Result<()> { let mut input = fs::File::open(src).with_context(|| format!("Failed to open {}", src.display()))?; @@ -413,61 +458,97 @@ pub fn fetch_binary_repo_db( let client = reqwest::blocking::Client::builder() .build() .context("Failed to build HTTP client for binary repo fetch")?; - let resp = client - .get(&repo_db_url) - .send() - .with_context(|| format!("Failed to fetch {}", repo_db_url))?; - - if !resp.status().is_success() { - if repo_db_sqlite.exists() { - crate::log_warn!( - "Failed to refresh binary repo '{}' (HTTP {}), using cached DB", - repo_name, - resp.status() - ); - return Ok(repo_db_sqlite); + match copy_file_url_to_path(&repo_db_url, &tmp_zst)? { + FileUrlCopyOutcome::Copied => {} + FileUrlCopyOutcome::Missing => { + if repo_db_sqlite.exists() { + crate::log_warn!( + "Failed to refresh binary repo '{}' (missing local file), using cached DB: {}", + repo_name, + repo_db_url + ); + return Ok(repo_db_sqlite); + } + anyhow::bail!("Failed to fetch {}: local file not found", repo_db_url); + } + FileUrlCopyOutcome::NotFileUrl => { + let resp = client + .get(&repo_db_url) + .send() + .with_context(|| format!("Failed to fetch {}", repo_db_url))?; + + if !resp.status().is_success() { + if repo_db_sqlite.exists() { + crate::log_warn!( + "Failed to refresh binary repo '{}' (HTTP {}), using cached DB", + repo_name, + resp.status() + ); + return Ok(repo_db_sqlite); + } + anyhow::bail!("Failed to fetch {}: HTTP {}", repo_db_url, resp.status()); + } + + let mut resp = resp; + let mut out = fs::File::create(&tmp_zst) + .with_context(|| format!("Failed to create {}", tmp_zst.display()))?; + std::io::copy(&mut resp, &mut out) + .with_context(|| format!("Failed to save {}", tmp_zst.display()))?; + out.flush() + .with_context(|| format!("Failed to flush {}", tmp_zst.display()))?; } - anyhow::bail!("Failed to fetch {}: HTTP {}", repo_db_url, resp.status()); } - let mut resp = resp; - let mut out = fs::File::create(&tmp_zst) - .with_context(|| format!("Failed to create {}", tmp_zst.display()))?; - std::io::copy(&mut resp, &mut out) - .with_context(|| format!("Failed to save {}", tmp_zst.display()))?; - out.flush() - .with_context(|| format!("Failed to flush {}", tmp_zst.display()))?; - - let sig_resp = client - .get(&repo_sig_url) - .send() - .with_context(|| format!("Failed to fetch {}", repo_sig_url))?; - let sig_downloaded = if sig_resp.status().is_success() { - let mut sig_resp = sig_resp; - let mut sig_out = fs::File::create(&tmp_sig) - .with_context(|| format!("Failed to create {}", tmp_sig.display()))?; - std::io::copy(&mut sig_resp, &mut sig_out) - .with_context(|| format!("Failed to save {}", tmp_sig.display()))?; - sig_out - .flush() - .with_context(|| format!("Failed to flush {}", tmp_sig.display()))?; - true - } else { - if !repo.allow_unsigned { - anyhow::bail!( - "Failed to fetch detached signature for binary repo '{}' (HTTP {}): {}", + let sig_downloaded = match copy_file_url_to_path(&repo_sig_url, &tmp_sig)? { + FileUrlCopyOutcome::Copied => true, + FileUrlCopyOutcome::Missing => { + if !repo.allow_unsigned { + anyhow::bail!( + "Failed to fetch detached signature for binary repo '{}' (local file not found): {}", + repo_name, + repo_sig_url + ); + } + crate::log_warn!( + "Binary repo '{}' has no detached signature (missing local file) for {}; allow_unsigned=true so continuing", repo_name, - sig_resp.status(), - repo_sig_url + repo_db_url ); + false + } + FileUrlCopyOutcome::NotFileUrl => { + let sig_resp = client + .get(&repo_sig_url) + .send() + .with_context(|| format!("Failed to fetch {}", repo_sig_url))?; + if sig_resp.status().is_success() { + let mut sig_resp = sig_resp; + let mut sig_out = fs::File::create(&tmp_sig) + .with_context(|| format!("Failed to create {}", tmp_sig.display()))?; + std::io::copy(&mut sig_resp, &mut sig_out) + .with_context(|| format!("Failed to save {}", tmp_sig.display()))?; + sig_out + .flush() + .with_context(|| format!("Failed to flush {}", tmp_sig.display()))?; + true + } else { + if !repo.allow_unsigned { + anyhow::bail!( + "Failed to fetch detached signature for binary repo '{}' (HTTP {}): {}", + repo_name, + sig_resp.status(), + repo_sig_url + ); + } + crate::log_warn!( + "Binary repo '{}' has no detached signature (HTTP {}) for {}; allow_unsigned=true so continuing", + repo_name, + sig_resp.status(), + repo_db_url + ); + false + } } - crate::log_warn!( - "Binary repo '{}' has no detached signature (HTTP {}) for {}; allow_unsigned=true so continuing", - repo_name, - sig_resp.status(), - repo_db_url - ); - false }; if sig_downloaded { @@ -928,23 +1009,31 @@ pub fn fetch_binary_package_archive( let pkg_url = join_repo_url(base_url, &rec.filename)?; crate::log_info!("Fetching binary package: {}", pkg_url); - let client = reqwest::blocking::Client::builder() - .build() - .context("Failed to build HTTP client for binary package fetch")?; - let mut resp = client - .get(&pkg_url) - .send() - .with_context(|| format!("Failed to fetch {}", pkg_url))?; - if !resp.status().is_success() { - anyhow::bail!("Failed to fetch {}: HTTP {}", pkg_url, resp.status()); - } + match copy_file_url_to_path(&pkg_url, &tmp_path)? { + FileUrlCopyOutcome::Copied => {} + FileUrlCopyOutcome::Missing => { + anyhow::bail!("Failed to fetch {}: local file not found", pkg_url); + } + FileUrlCopyOutcome::NotFileUrl => { + let client = reqwest::blocking::Client::builder() + .build() + .context("Failed to build HTTP client for binary package fetch")?; + let mut resp = client + .get(&pkg_url) + .send() + .with_context(|| format!("Failed to fetch {}", pkg_url))?; + if !resp.status().is_success() { + anyhow::bail!("Failed to fetch {}: HTTP {}", pkg_url, resp.status()); + } - let mut out = fs::File::create(&tmp_path) - .with_context(|| format!("Failed to create {}", tmp_path.display()))?; - std::io::copy(&mut resp, &mut out) - .with_context(|| format!("Failed to save {}", tmp_path.display()))?; - out.flush() - .with_context(|| format!("Failed to flush {}", tmp_path.display()))?; + let mut out = fs::File::create(&tmp_path) + .with_context(|| format!("Failed to create {}", tmp_path.display()))?; + std::io::copy(&mut resp, &mut out) + .with_context(|| format!("Failed to save {}", tmp_path.display()))?; + out.flush() + .with_context(|| format!("Failed to flush {}", tmp_path.display()))?; + } + } verify_binary_package_record_checksums(&tmp_path, rec).with_context(|| { format!( @@ -978,6 +1067,7 @@ pub fn sync_mirrors( for (name, url) in mirrors { let target = base.join(name); + let git_url = normalize_git_mirror_url(url)?; if !target.exists() { crate::log_info!("Cloning mirror '{}' -> {}", name, target.display()); @@ -994,7 +1084,7 @@ pub fn sync_mirrors( let mut builder = RepoBuilder::new(); builder.fetch_options(fo); builder - .clone(url, &target) + .clone(&git_url, &target) .with_context(|| format!("Failed to clone {}", url))?; } else { crate::log_info!("Updating mirror '{}' in {}", name, target.display()); @@ -1013,7 +1103,7 @@ pub fn sync_mirrors( // Fetch from origin let mut remote = repo .find_remote("origin") - .or_else(|_| repo.remote_anonymous(url))?; + .or_else(|_| repo.remote_anonymous(&git_url))?; remote .fetch(&["refs/heads/*:refs/remotes/origin/*"], Some(&mut fo), None) .with_context(|| format!("Failed to fetch updates for {}", url))?; @@ -1376,4 +1466,40 @@ license = ["MIT", "Apache-2.0"] verify_binary_package_record_checksums(&pkg, &rec).unwrap(); } + + #[test] + fn test_copy_file_url_to_path_supports_file_scheme() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("repo.db.zst"); + let dst = tmp.path().join("copy.zst"); + fs::write(&src, b"repo-db").unwrap(); + + let url = format!("file://{}", src.display()); + let outcome = copy_file_url_to_path(&url, &dst).unwrap(); + assert_eq!(outcome, FileUrlCopyOutcome::Copied); + assert_eq!(fs::read(&dst).unwrap(), b"repo-db"); + } + + #[test] + fn test_copy_file_url_to_path_reports_missing_file() { + let tmp = tempfile::tempdir().unwrap(); + let missing = tmp.path().join("missing.db.zst"); + let dst = tmp.path().join("copy.zst"); + + let url = format!("file://{}", missing.display()); + let outcome = copy_file_url_to_path(&url, &dst).unwrap(); + assert_eq!(outcome, FileUrlCopyOutcome::Missing); + assert!(!dst.exists()); + } + + #[test] + fn test_normalize_git_mirror_url_converts_file_scheme() { + let tmp = tempfile::tempdir().unwrap(); + let repo_dir = tmp.path().join("repo.git"); + fs::create_dir_all(&repo_dir).unwrap(); + + let url = format!("file://{}", repo_dir.display()); + let normalized = normalize_git_mirror_url(&url).unwrap(); + assert_eq!(normalized, repo_dir.to_string_lossy()); + } } diff --git a/src/package/interactive.rs b/src/package/interactive.rs index 875acf8..bb04686 100644 --- a/src/package/interactive.rs +++ b/src/package/interactive.rs @@ -1,5 +1,6 @@ use crate::package::{ - Alternatives, Build, BuildFlags, BuildType, Dependencies, PackageInfo, PackageSpec, Source, + Alternatives, Build, BuildFlags, BuildType, Dependencies, ManualSource, PackageInfo, + PackageSpec, Source, }; use anyhow::{Context, Result}; use inquire::{Confirm, Select, Text}; @@ -146,6 +147,99 @@ fn prompt_optional_text(prompt: &str, help: &str) -> Result { .to_string()) } +fn prompt_manual_sources() -> Result> { + let mut out = Vec::new(); + loop { + let add = Confirm::new("Add a manual source?") + .with_help_message( + "Use for local files (keys, patches, configs) or direct URLs copied into the build dir", + ) + .with_default(out.is_empty()) + .prompt()?; + if !add { + break; + } + + let modes = vec!["Local file(s)", "Remote URL(s)"]; + let mode = Select::new("Manual source mode:", modes) + .with_help_message("Local files are resolved relative to the spec directory") + .prompt()?; + + let mut manual = ManualSource { + file: None, + files: Vec::new(), + url: None, + urls: Vec::new(), + sha256: None, + dest: None, + }; + + let entries = if mode == "Local file(s)" { + prompt_repeating_list( + "Manual source file", + "Path relative to spec dir, e.g. depot.pub, keys/repo.pub", + )? + } else { + prompt_repeating_list( + "Manual source URL", + "Supports http(s), ftp, and file:// URLs", + )? + }; + + if entries.is_empty() { + crate::log_warn!("Manual source block skipped (no entries provided)."); + continue; + } + + if mode == "Local file(s)" { + if entries.len() == 1 { + manual.file = Some(entries[0].clone()); + } else { + manual.files = entries; + } + } else if entries.len() == 1 { + manual.url = Some(entries[0].clone()); + } else { + manual.urls = entries; + } + + let single_entry = matches!( + ( + &manual.file, + manual.files.len(), + &manual.url, + manual.urls.len() + ), + (Some(_), 0, None, 0) | (None, 0, Some(_), 0) + ); + + if single_entry { + let checksum = prompt_optional_text( + "Manual source checksum (optional):", + "sha256:/sha512:/md5:, raw SHA256 hex, or 'skip' (empty omits field)", + )?; + if !checksum.is_empty() { + manual.sha256 = Some(checksum); + } + + let dest = prompt_optional_text( + "Manual source destination (optional):", + "Path relative to build work dir (default derives from file/url name)", + )?; + if !dest.is_empty() { + manual.dest = Some(dest); + } + } else { + crate::log_info!( + "Multiple entries in one manual_sources block cannot use a shared checksum or dest; skipping those prompts." + ); + } + + out.push(manual); + } + Ok(out) +} + fn parse_license_list(input: &str) -> Vec { input .split(',') @@ -282,58 +376,79 @@ pub fn create_interactive() -> Result { }; let mut sources = Vec::new(); + let mut manual_sources = Vec::new(); if !is_metapackage { let source_url = Text::new("Source URL:") - .with_help_message("URL to the source tarball or git repository") + .with_help_message( + "URL to the source tarball or git repository (supports file://; leave empty if you will add manual_sources later)", + ) .with_default(gnu_source_default.as_str()) .prompt()?; if source_url.trim().is_empty() { - anyhow::bail!("Source URL cannot be empty"); - } - - // Attempt to compute SHA256 automatically when online — use as the default if available. - let checksum_source_url = expand_known_package_vars(&source_url, &name, &version); - let computed_sha_default = match compute_sha256_for_url(&checksum_source_url) { - Ok(hex) => { - // Use raw hex as the default so pressing Enter accepts it - hex + crate::log_warn!( + "No source URL provided. Generated spec will omit [source]; add manual_sources or source entries before building (unless this becomes a metapackage)." + ); + if Confirm::new("Add manual sources now?") + .with_help_message("Useful for keyrings/patch-only/custom packages") + .with_default(true) + .prompt()? + { + manual_sources = prompt_manual_sources()?; } - Err(_) => "skip".to_string(), - }; + } else { + // Attempt to compute SHA256 automatically when online — use as the default if available. + let checksum_source_url = expand_known_package_vars(&source_url, &name, &version); + let computed_sha_default = match compute_sha256_for_url(&checksum_source_url) { + Ok(hex) => { + // Use raw hex as the default so pressing Enter accepts it + hex + } + Err(_) => "skip".to_string(), + }; - let source_sha256 = Text::new("Source checksum:") - .with_help_message( - "Accepts sha256:, sha512:, md5:, or raw SHA256 hex (use 'skip' to bypass)", - ) - .with_default(computed_sha_default.as_str()) - .prompt()?; + let source_sha256 = Text::new("Source checksum:") + .with_help_message( + "Accepts sha256:, sha512:, md5:, or raw SHA256 hex (use 'skip' to bypass)", + ) + .with_default(computed_sha_default.as_str()) + .prompt()?; - let extract_dir = Text::new("Extract Directory:") - .with_help_message("Directory created after extraction (supports $name, $version)") - .with_default("$name-$version") - .prompt()?; + let extract_dir = Text::new("Extract Directory:") + .with_help_message("Directory created after extraction (supports $name, $version)") + .with_default("$name-$version") + .prompt()?; - let mut source = Source { - url: source_url, - sha256: source_sha256, - extract_dir, - patches: Vec::new(), - post_extract: Vec::new(), - }; + let mut source = Source { + url: source_url, + sha256: source_sha256, + extract_dir, + patches: Vec::new(), + post_extract: Vec::new(), + }; - if show_advanced { - source.patches = prompt_repeating_list( - "Patch path/URL", - "Patch file path (relative to spec dir) or direct URL", - )?; - source.post_extract = prompt_repeating_list( - "post_extract command", - "Runs in extracted source directory using sh -c", - )?; + if show_advanced { + source.patches = prompt_repeating_list( + "Patch path/URL", + "Patch file path (relative to spec dir) or direct URL", + )?; + source.post_extract = prompt_repeating_list( + "post_extract command", + "Runs in extracted source directory using sh -c", + )?; + if Confirm::new("Add manual sources?") + .with_help_message( + "Optional local files or URLs copied into the build dir before source fetching", + ) + .with_default(false) + .prompt()? + { + manual_sources = prompt_manual_sources()?; + } + } + + sources.push(source); } - - sources.push(source); } else { crate::log_info!( "Metapackage selected: skipping source/build prompts. Define runtime dependencies to pull in." @@ -537,7 +652,7 @@ pub fn create_interactive() -> Result { }, packages: Vec::new(), alternatives: Alternatives::default(), - manual_sources: Vec::new(), + manual_sources, source: sources, build: Build { build_type, flags }, dependencies: Dependencies { @@ -603,6 +718,47 @@ pub fn spec_to_minimal_toml(spec: &PackageSpec) -> anyhow::Result { root.insert("packages".into(), Value::Array(arr)); } + // manual sources + if !spec.manual_sources.is_empty() { + let mut arr = Vec::new(); + for m in &spec.manual_sources { + let mut mt = Table::new(); + if let Some(file) = m.file.as_ref().filter(|s| !s.trim().is_empty()) { + mt.insert("file".into(), Value::String(file.clone())); + } + if !m.files.is_empty() { + mt.insert( + "files".into(), + Value::Array(m.files.iter().map(|s| Value::String(s.clone())).collect()), + ); + } + if let Some(url) = m.url.as_ref().filter(|s| !s.trim().is_empty()) { + mt.insert("url".into(), Value::String(url.clone())); + } + if !m.urls.is_empty() { + mt.insert( + "urls".into(), + Value::Array(m.urls.iter().map(|s| Value::String(s.clone())).collect()), + ); + } + if let Some(sha256) = m.sha256.as_ref().filter(|s| { + let trimmed = s.trim(); + !trimmed.is_empty() && !trimmed.eq_ignore_ascii_case("skip") + }) { + mt.insert("sha256".into(), Value::String(sha256.clone())); + } + if let Some(dest) = m.dest.as_ref().filter(|s| !s.trim().is_empty()) { + mt.insert("dest".into(), Value::String(dest.clone())); + } + if !mt.is_empty() { + arr.push(Value::Table(mt)); + } + } + if !arr.is_empty() { + root.insert("manual_sources".into(), Value::Array(arr)); + } + } + // sources if !spec.source.is_empty() { let mut arr = Vec::new(); @@ -1112,7 +1268,9 @@ pub fn spec_to_minimal_toml(spec: &PackageSpec) -> anyhow::Result { #[cfg(test)] mod tests { use super::*; - use crate::package::{BuildFlags, BuildType, Dependencies, PackageInfo, PackageSpec, Source}; + use crate::package::{ + BuildFlags, BuildType, Dependencies, ManualSource, PackageInfo, PackageSpec, Source, + }; #[test] fn spec_to_minimal_toml_omits_defaults() { @@ -1440,6 +1598,63 @@ mod tests { assert!(val.get("source").is_none()); } + #[test] + fn spec_to_minimal_toml_includes_manual_sources() { + let spec = PackageSpec { + package: PackageInfo { + name: "vertex-keyring".into(), + version: "1.0.0".into(), + revision: 1, + description: "keyring".into(), + homepage: "https://www.vertexlinux.net".into(), + license: vec!["MIT".into()], + }, + packages: Vec::new(), + alternatives: Alternatives::default(), + manual_sources: vec![ + ManualSource { + file: Some("vertex.pub".into()), + files: Vec::new(), + url: None, + urls: Vec::new(), + sha256: None, + dest: Some("usr/share/depot/keys/public/vertex.pub".into()), + }, + ManualSource { + file: None, + files: Vec::new(), + url: Some("file:///tmp/vertex.minisig".into()), + urls: Vec::new(), + sha256: Some("skip".into()), + dest: Some("usr/share/depot/keys/sign/vertex.minisig".into()), + }, + ], + source: Vec::new(), + build: Build { + build_type: BuildType::Custom, + flags: BuildFlags::default(), + }, + dependencies: Dependencies::default(), + package_alternatives: Default::default(), + package_dependencies: Default::default(), + spec_dir: PathBuf::from("."), + }; + + let toml = spec_to_minimal_toml(&spec).unwrap(); + assert!(toml.contains("[[manual_sources]]")); + assert!(toml.contains("file = \"vertex.pub\"")); + assert!(toml.contains("url = \"file:///tmp/vertex.minisig\"")); + assert!(toml.contains("dest = \"usr/share/depot/keys/public/vertex.pub\"")); + assert!(!toml.contains("sha256 = \"skip\"")); + + let val: toml::Value = toml::from_str(&toml).unwrap(); + let arr = val + .get("manual_sources") + .and_then(|v| v.as_array()) + .expect("expected manual_sources array"); + assert_eq!(arr.len(), 2); + } + #[test] fn compute_sha256_for_local_path_and_file_url() { use sha2::Digest as TestDigest;