From 1833a2c42d85d6265ac40b8f6597b50a9fedf23f Mon Sep 17 00:00:00 2001 From: SFG545 Date: Wed, 20 May 2026 19:54:00 -0500 Subject: [PATCH] Implement system state management with TOML serialization - Introduced `SystemState` struct to manage system state, including stage, target, architecture, and layers. - Added functions to load and save system state from/to a TOML file. - Implemented layer management functions: add, set, and remove packages from layers. - Created initialization function for LBI layout, including directory and symlink creation. - Added tests for layer management and LBI layout initialization. --- Cargo.lock | 447 +- Cargo.toml | 9 +- src/bootstrap.rs | 6700 +++++++++++++++++++++++++++++ src/builder/custom.rs | 51 +- src/cli.rs | 105 + src/commands.rs | 52 +- src/commands/build_cmd.rs | 18 +- src/commands/build_cmd/support.rs | 16 + src/commands/misc.rs | 111 + src/commands/misc/internal.rs | 7 + src/commands/tests.rs | 36 + src/cross.rs | 35 +- src/db/mod.rs | 85 +- src/main.rs | 2 + src/source/fetcher.rs | 551 +-- src/staging/mod.rs | 435 +- src/system_state.rs | 504 +++ src/ui.rs | 10 + 18 files changed, 8820 insertions(+), 354 deletions(-) create mode 100644 src/bootstrap.rs create mode 100644 src/system_state.rs diff --git a/Cargo.lock b/Cargo.lock index 40777b8..9143d4c 100755 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adobe-cmap-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3" +dependencies = [ + "pom", +] + [[package]] name = "aes" version = "0.8.4" @@ -15,10 +24,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] +[[package]] +name = "aes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd29a732b644c0431c6140f370d097879203d79b80c94a6747ba0872adaef8" +dependencies = [ + "cipher 0.5.1", + "cpubits", + "cpufeatures 0.3.0", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -175,6 +195,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" dependencies = [ "hybrid-array", + "zeroize", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", ] [[package]] @@ -183,6 +213,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytes" version = "1.11.1" @@ -198,6 +234,15 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "cc" version = "1.2.56" @@ -210,6 +255,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cff-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f5b6e9141c036f3ff4ce7b2f7e432b0f00dee416ddcd4f17741d189ddc2e9d" + [[package]] name = "cfg-if" version = "1.0.4" @@ -240,14 +291,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout", + "inout 0.1.4", +] + +[[package]] +name = "cipher" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34d8227fe1ba289043aeb13792056ff80fd6de1a9f49137a5f499de8e8c78ea" +dependencies = [ + "crypto-common 0.2.1", + "inout 0.2.2", ] [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -267,18 +328,18 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.6.0" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19c9f1dde76b736e3681f28cec9d5a61299cbaae0fce80a68e43724ad56031eb" +checksum = "e0a7a9bfdb35811f9e59832f0f05975114d2251b415fb534108e6f34060fd772" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -302,6 +363,12 @@ dependencies = [ "roff", ] +[[package]] +name = "cmov" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" + [[package]] name = "colorchoice" version = "1.0.4" @@ -364,6 +431,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -443,6 +516,15 @@ version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b10589d1a5e400d61f9f38f12f884cfd080ff345de8f17efda36fe0e4a02aa8" +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "deflate64" version = "0.1.11" @@ -471,13 +553,14 @@ dependencies = [ "md5", "minisign", "nix", + "pdf-extract", "petgraph", "reqwest", "rusqlite", "semver", "serde", "serde_ignored", - "sha1 0.11.0", + "sha1", "sha2 0.11.0", "signal-hook 0.4.4", "suppaftp", @@ -546,6 +629,8 @@ dependencies = [ "block-buffer 0.12.0", "const-oid", "crypto-common 0.2.1", + "ctutils", + "zeroize", ] [[package]] @@ -574,6 +659,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "encode_unicode" version = "1.0.0" @@ -605,6 +699,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -646,13 +749,12 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" dependencies = [ "cfg-if", "libc", - "libredox", ] [[package]] @@ -885,6 +987,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.2", +] + [[package]] name = "http" version = "1.4.0" @@ -1156,9 +1267,19 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ + "block-padding", "generic-array", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", +] + [[package]] name = "inquire" version = "0.9.4" @@ -1258,9 +1379,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.182" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" @@ -1276,17 +1397,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "libredox" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" -dependencies = [ - "bitflags", - "libc", - "redox_syscall 0.7.3", -] - [[package]] name = "libsqlite3-sys" version = "0.37.0" @@ -1357,10 +1467,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] -name = "lz4_flex" -version = "0.13.0" +name = "lopdf" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "c7184fdea2bc3cd272a1acec4030c321a8f9875e877b3f92a53f2f6033fdc289" +dependencies = [ + "aes 0.8.4", + "bitflags", + "cbc", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.3.4", + "indexmap", + "itoa", + "log", + "md-5", + "nom", + "nom_locate", + "rand", + "rangemap", + "sha2 0.10.9", + "stringprep", + "thiserror 2.0.18", + "ttf-parser", + "weezl", +] + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash", ] @@ -1385,6 +1523,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "md5" version = "0.8.0" @@ -1450,9 +1598,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.31.2" +version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ "bitflags", "cfg-if", @@ -1466,6 +1614,26 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom_locate" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +dependencies = [ + "bytecount", + "memchr", + "nom", +] + [[package]] name = "num-conv" version = "0.2.0" @@ -1561,7 +1729,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -1573,7 +1741,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", - "hmac", + "hmac 0.12.1", +] + +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.2", + "hmac 0.13.0", +] + +[[package]] +name = "pdf-extract" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28ba1758a3d3f361459645780e09570b573fc3c82637449e9963174c813a98" +dependencies = [ + "adobe-cmap-parser", + "cff-parser", + "encoding_rs", + "euclid", + "log", + "lopdf", + "postscript", + "type1-encoding-parser", + "unicode-normalization", ] [[package]] @@ -1612,12 +1807,24 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "pom" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" + [[package]] name = "portable-atomic" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +[[package]] +name = "postscript" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" + [[package]] name = "potential_utf" version = "0.1.4" @@ -1639,6 +1846,15 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -1673,6 +1889,41 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1682,15 +1933,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_syscall" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.12.3" @@ -1722,9 +1964,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" dependencies = [ "base64", "bytes", @@ -1850,7 +2092,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -1883,7 +2125,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ - "pbkdf2", + "pbkdf2 0.12.2", "salsa20", "sha2 0.10.9", ] @@ -1979,17 +2221,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha1" version = "0.11.0" @@ -2127,6 +2358,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2141,9 +2383,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "suppaftp" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3da253d7e9993de86df41eb89e8cb1b6f567abe215798645651fca4148d0aa" +checksum = "4275c142b5be3af2eeadd70dd368caf3b65546c8af1035839372dd7a1436127d" dependencies = [ "chrono", "futures-lite", @@ -2311,6 +2553,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.49.0" @@ -2456,12 +2713,27 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + [[package]] name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "type1-encoding-parser" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa10c302f5a53b7ad27fd42a3996e23d096ba39b5b8dd6d9e683a05b01bee749" +dependencies = [ + "pom", +] + [[package]] name = "typed-path" version = "0.12.3" @@ -2474,12 +2746,33 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -2696,6 +2989,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi" version = "0.3.9" @@ -3104,6 +3403,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.6" @@ -3166,24 +3485,24 @@ dependencies = [ [[package]] name = "zip" -version = "8.5.0" +version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2726508a48f38dceb22b35ecbbd2430efe34ff05c62bd3285f965d7911b33464" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ - "aes", + "aes 0.9.0", "bzip2", "constant_time_eq 0.4.2", "crc32fast", "deflate64", "flate2", "getrandom 0.4.1", - "hmac", + "hmac 0.13.0", "indexmap", "lzma-rust2", "memchr", - "pbkdf2", + "pbkdf2 0.13.0", "ppmd-rust", - "sha1 0.10.6", + "sha1", "time", "typed-path", "zeroize", diff --git a/Cargo.toml b/Cargo.toml index 37c9218..067efad 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ clap = { version = "4.6.0", features = ["derive"] } flate2 = "1.1.9" git2 = "0.20.4" indicatif = "0.18.4" -nix = { version = "0.31.2", features = ["user"] } +nix = { version = "0.31.3", features = ["user"] } rusqlite = "0.39.0" semver = "1.0.28" serde = { version = "1.0.228", features = ["derive"] } @@ -36,17 +36,18 @@ minisign = "0.9.1" petgraph = "0.8.3" fd-lock = "4.0.4" reqwest = { version = "0.13.2", default-features = false, features = ["blocking", "native-tls"] } -filetime = "0.2.27" -clap_complete = "4.6.0" +filetime = "0.2.29" +clap_complete = "4.6.5" clap_mangen = "0.3.0" sys-mount = { version = "3.1.0", default-features = false } time = { version = "0.3.47", features = ["formatting", "parsing"] } b2sum-rust = "0.3.0" serde_ignored = "0.1.14" -lz4_flex = "0.13.0" +lz4_flex = "0.13.1" lzma-rust2 = "0.16.2" signal-hook = "0.4.4" sha1 = "0.11.0" +pdf-extract = "0.10.0" [dev-dependencies] tempfile = "=3.27.0" diff --git a/src/bootstrap.rs b/src/bootstrap.rs new file mode 100644 index 0000000..a27c0a2 --- /dev/null +++ b/src/bootstrap.rs @@ -0,0 +1,6700 @@ +use crate::cli::BootstrapArgs; +use crate::{config, source, system_state, ui}; +use anyhow::{Context, Result}; +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use sys_mount::{Mount, MountFlags, Unmount, UnmountFlags}; +use url::Url; + +const TEMP_LAYER: &str = "temp"; +const BASE_LAYER: &str = "base"; +const DEVEL_LAYER: &str = "devel"; +const FILESYSTEM_PACKAGE: &str = "filesystem"; +const BOOTSTRAP_DIR: &str = "bootstrap/lbi"; +const DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS: &str = "DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS"; +const CHAPTER7_RETIRED_PACKAGES: &[&str] = &["llvm-clang-pass1"]; +const BOOTSTRAP_CHROOT_SHIM_DIR: &str = "/tmp/depot-bootstrap-tools"; +const BOOTSTRAP_CHROOT_PATH: &str = "/system/tools/bin:/system/binaries:/system/systembinaries:/tmp/depot-bootstrap-tools:/bin:/usr/bin:/sbin:/usr/sbin"; +const BOOK_FETCH_CACHE_BUST_PARAM: &str = "depot_refresh"; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct BookPackage { + chapter: u8, + section: String, + title: String, + name: String, + version: String, + layer: String, + page_url: String, + recipe_id: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BookOperationKind { + ResetTargetTreeOwnership, + CreateVirtualFilesystemLinkTargets, + CopyBuildProfile, + EnterChroot, + CreateEssentialSystemFiles, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct BookOperation { + section: String, + title: String, + kind: BookOperationKind, + recipe_id: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum BookStep { + Package(BookPackage), + Operation(BookOperation), +} + +#[derive(Debug, Clone)] +struct GeneratedRecipe { + package: BookPackage, + spec_path: PathBuf, + progress_path: PathBuf, +} + +#[derive(Debug, Clone)] +struct PageRecipe { + input_files: Vec, + source_urls: Vec, + extract_dir: Option, + commands: Vec, + dependencies: Vec, + license: String, + description: String, +} + +#[derive(Debug, Clone)] +struct ManifestEntry { + url: String, + output_name: String, +} + +#[derive(Debug, Clone, Default)] +struct SourceManifest { + entries: Vec, + blake2b_512: BTreeMap, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BootstrapBuildMode { + Host, + Cross, + Chroot, +} + +#[derive(Debug, Clone)] +struct BootstrapInstallInvocation { + args: Vec, + env: Vec<(OsString, OsString)>, +} + +pub(crate) fn run(args: BootstrapArgs) -> Result<()> { + let config = config::Config::for_rootfs(&args.sysroot); + let (target, arch) = bootstrap_target_arch(&args, &config)?; + ensure_lbi_layout_for_fresh_bootstrap(&args.sysroot, &config, &target, &arch)?; + let pdf = load_book_pdf(&args)?; + let text = + pdf_extract::extract_text_from_mem(&pdf).context("Failed to extract text from book PDF")?; + let steps = parse_book_steps(&text, &args.book_url)?; + let packages = packages_from_steps(&steps); + if packages.is_empty() { + anyhow::bail!("No package sections were found in the Linux by Intent book"); + } + + let recipes = generate_recipes(&args, &config, &packages, &target, &arch)?; + let recipes_by_id = recipes + .iter() + .map(|recipe| (recipe.package.recipe_id.as_str(), recipe)) + .collect::>(); + let progress_root = bootstrap_progress_root(&config); + + for (idx, step) in steps.iter().enumerate() { + if step_requires_root(step) + && !step_is_done(step, &recipes_by_id, &progress_root, &args.sysroot, &config) + && !crate::fakeroot::is_root() + { + ensure_root_for_bootstrap()?; + return Ok(()); + } + + match step { + BookStep::Package(package) => { + let recipe = recipes_by_id + .get(package.recipe_id.as_str()) + .copied() + .with_context(|| format!("Missing generated recipe for {}", package.title))?; + let mode = build_mode_for_package(package); + ui::merge_package(&package.layer, &package.name); + install_recipe(&args.sysroot, &config, recipe, mode, &target)?; + system_state::set_stage(&config, stage_for_step(step))?; + } + BookStep::Operation(operation) => { + run_operation_step(&args.sysroot, &config, operation, &target, &arch)?; + system_state::set_stage(&config, stage_for_step(step))?; + } + } + + if step_completes_chapter(step, steps.get(idx + 1), 7) { + retire_packages_after_chapter7(&args.sysroot, &config)?; + system_state::set_stage(&config, "bootstrap-chapter7-cleanup".to_string())?; + } + } + + let layers = packages_by_layer(&packages); + for layer in [TEMP_LAYER, BASE_LAYER, DEVEL_LAYER] { + let packages = bootstrap_layer_packages_for_state(&layers, layer); + system_state::set_layer_packages(&config, layer.to_string(), &packages)?; + } + + system_state::set_stage(&config, "bootstrap-complete".to_string())?; + ui::success(format!( + "Bootstrapped {} Linux by Intent package(s) through chapter 9", + packages.len() + )); + Ok(()) +} + +fn bootstrap_target_arch( + args: &BootstrapArgs, + config: &config::Config, +) -> Result<(String, String)> { + let state = system_state::load(config)?; + let target = args + .target + .clone() + .or(state.target) + .unwrap_or_else(|| "x86_64-lbi-linux-musl".to_string()); + let arch = args + .arch + .clone() + .or(state.arch) + .unwrap_or_else(|| crate::cross::target_arch_from_triple(&target).to_string()); + Ok((target, arch)) +} + +fn ensure_lbi_layout_for_fresh_bootstrap( + sysroot: &Path, + config: &config::Config, + target: &str, + arch: &str, +) -> Result<()> { + let state = system_state::load(config)?; + system_state::ensure_lbi_layout_paths(sysroot)?; + if state.stage.is_some() + || !state.layers.is_empty() + || sysroot.join("etc/depot.d/build.toml").exists() + { + ui::info("Resuming existing bootstrap state"); + return Ok(()); + } + + ui::info("Initializing Linux by Intent sysroot layout"); + system_state::init_lbi_layout(sysroot, config, target, Some(arch), false)?; + Ok(()) +} + +fn install_recipe( + sysroot: &Path, + config: &config::Config, + recipe: &GeneratedRecipe, + mode: BootstrapBuildMode, + target: &str, +) -> Result<()> { + if bootstrap_package_is_complete(sysroot, config, recipe)? { + ui::info(format!( + "Skipping already completed package {} ({})", + recipe.package.name, recipe.package.section + )); + return Ok(()); + } + + prepare_bootstrap_package_install(sysroot, config, &recipe.package)?; + let exe = std::env::current_exe().context("Failed to locate depot executable")?; + let invocation = bootstrap_install_invocation(sysroot, recipe, mode, target, &exe)?; + let mut cmd = Command::new(&exe); + cmd.args(&invocation.args); + for (key, value) in &invocation.env { + cmd.env(key, value); + } + let status = cmd + .status() + .with_context(|| format!("Failed to run depot install for {}", recipe.package.name))?; + if !status.success() { + anyhow::bail!( + "depot install failed for {} with status {status}", + recipe.package.name + ); + } + if let Some(parent) = recipe.progress_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + let mut progress = format!( + "section = \"{}\"\npackage = \"{}\"\nlayer = \"{}\"\n", + recipe.package.section, recipe.package.name, recipe.package.layer + ); + if let Some(revision) = bootstrap_recipe_revision(&recipe.package.name) { + progress.push_str(&format!("recipe_revision = {revision}\n")); + } + fs::write(&recipe.progress_path, progress) + .with_context(|| format!("Failed to write {}", recipe.progress_path.display()))?; + Ok(()) +} + +fn bootstrap_package_is_complete( + sysroot: &Path, + config: &config::Config, + recipe: &GeneratedRecipe, +) -> Result { + if package_is_retired(config, &recipe.package.name) { + return Ok(true); + } + + if !recipe.progress_path.exists() { + return Ok(false); + } + + if let Some(revision) = bootstrap_recipe_revision(&recipe.package.name) { + let progress = fs::read_to_string(&recipe.progress_path) + .with_context(|| format!("Failed to read {}", recipe.progress_path.display()))?; + let expected = format!("recipe_revision = {revision}"); + if !progress.lines().any(|line| line.trim() == expected) { + ui::info(format!( + "Reinstalling {} because its bootstrap recipe changed", + recipe.package.name + )); + return Ok(false); + } + } + + let db_path = config.installed_db_path(sysroot); + let replaced = crate::db::get_all_replaces(&db_path).with_context(|| { + format!( + "Failed to inspect replacement metadata for {}", + recipe.package.name + ) + })?; + if replaced.contains(&recipe.package.name) { + return Ok(true); + } + + let files = + crate::db::get_package_files(&db_path, &recipe.package.name).with_context(|| { + format!( + "Failed to inspect installed files for {}", + recipe.package.name + ) + })?; + if files.is_empty() { + ui::info(format!( + "Reinstalling {} because bootstrap progress exists but no installed files were recorded", + recipe.package.name + )); + return Ok(false); + } + + for rel_path in files { + if crate::staging::is_purged_payload_path(&rel_path) { + continue; + } + let disk_path = sysroot.join(&rel_path); + if disk_path.symlink_metadata().is_err() { + ui::info(format!( + "Reinstalling {} because {} is missing from the sysroot", + recipe.package.name, rel_path + )); + return Ok(false); + } + } + + for rel_path in bootstrap_required_payload_paths(&recipe.package.name) { + let disk_path = sysroot.join(rel_path); + if !disk_path.exists() { + ui::info(format!( + "Reinstalling {} because required bootstrap payload {} is missing from the sysroot", + recipe.package.name, rel_path + )); + return Ok(false); + } + } + + Ok(true) +} + +fn bootstrap_required_payload_paths(package: &str) -> &'static [&'static str] { + match package { + "bmake" => &["system/binaries/bmake", "system/share/mk/sys.mk"], + "llvm-clang-pass1" => &[ + "system/tools/bin/llvm-config", + "system/tools/bin/llvm-tblgen", + "system/tools/bin/clang-tblgen", + ], + "ubase" => &["system/binaries/id"], + _ => &[], + } +} + +fn bootstrap_recipe_revision(package: &str) -> Option { + match package { + "bmake" => Some(2), + "llvm" => Some(3), + "ubase" => Some(1), + _ => None, + } +} + +const BSDDIFFUTILS_SBASE_HANDOFF_PATHS: &[&str] = &[ + "system/binaries/cmp", + "system/documentation/man-pages/man1/cmp.1", +]; + +const BSDGREP_SBASE_HANDOFF_PATHS: &[&str] = &[ + "system/binaries/grep", + "system/documentation/man-pages/man1/grep.1", +]; + +fn prepare_bootstrap_package_install( + sysroot: &Path, + config: &config::Config, + package: &BookPackage, +) -> Result<()> { + let handoff_paths = match package.name.as_str() { + "bsddiffutils" => BSDDIFFUTILS_SBASE_HANDOFF_PATHS, + "bsdgrep" => BSDGREP_SBASE_HANDOFF_PATHS, + _ => return Ok(()), + }; + + if handoff_paths.is_empty() { + return Ok(()); + } + + let db_path = config.installed_db_path(sysroot); + if !db_path.exists() { + return Ok(()); + } + + let mut conn = rusqlite::Connection::open(&db_path) + .with_context(|| format!("Failed to open package database {}", db_path.display()))?; + let tx = conn.transaction()?; + + for rel_path in handoff_paths { + let owner = match tx.query_row( + "SELECT p.name FROM files f JOIN packages p ON f.package_id = p.id WHERE f.path = ?1", + rusqlite::params![rel_path], + |row| row.get::<_, String>(0), + ) { + Ok(owner) => owner, + Err(rusqlite::Error::QueryReturnedNoRows) => continue, + Err(err) => return Err(err).context("Failed to query package file ownership"), + }; + + if owner != "sbase" { + continue; + } + + tx.execute( + "DELETE FROM files WHERE path = ?1 AND package_id = (SELECT id FROM packages WHERE name = 'sbase')", + rusqlite::params![rel_path], + ) + .with_context(|| format!("Failed to clear sbase ownership for {rel_path}"))?; + + let disk_path = sysroot.join(rel_path); + if disk_path.exists() { + fs::remove_file(&disk_path) + .with_context(|| format!("Failed to remove {}", disk_path.display()))?; + } + ui::info(format!( + "Handed off {} from sbase to {}", + rel_path, package.name + )); + } + + tx.commit() + .with_context(|| format!("Failed to update package database {}", db_path.display()))?; + Ok(()) +} + +fn bootstrap_install_invocation( + sysroot: &Path, + recipe: &GeneratedRecipe, + mode: BootstrapBuildMode, + target: &str, + depot_exe: &Path, +) -> Result { + let mut args = vec![ + OsString::from("install"), + OsString::from("-r"), + sysroot.as_os_str().to_os_string(), + OsString::from("--yes"), + OsString::from("--no-deps"), + ]; + if mode == BootstrapBuildMode::Cross { + args.push(OsString::from("--cross-prefix")); + args.push(OsString::from(target)); + } + args.push(recipe.spec_path.as_os_str().to_os_string()); + + let mut env = Vec::new(); + if matches!(mode, BootstrapBuildMode::Cross | BootstrapBuildMode::Chroot) { + env.push(( + OsString::from("PATH"), + bootstrap_tool_path( + sysroot, + std::env::var_os("PATH").as_deref(), + mode == BootstrapBuildMode::Chroot, + )?, + )); + } + env.push(( + OsString::from("DEPOT_LBI_SYSROOT"), + bootstrap_sysroot_env_path(sysroot)?.into_os_string(), + )); + env.push(( + OsString::from("LWI_MAKE_FLAGS"), + OsString::from(bootstrap_parallel_makeflags()), + )); + env.push(( + OsString::from("LWI_MAKE_JOBS"), + OsString::from(bootstrap_parallel_make_jobs()), + )); + env.push(( + OsString::from(DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS), + OsString::from("1"), + )); + if mode == BootstrapBuildMode::Chroot { + env.extend([ + (OsString::from("DEPOT_LBI_CHROOT"), OsString::from("1")), + ( + OsString::from("DEPOT_LBI_CHROOT_ROOT"), + sysroot.as_os_str().to_os_string(), + ), + ( + OsString::from("DEPOT_LBI_DEPOT_EXE"), + depot_exe.as_os_str().to_os_string(), + ), + ]); + } + + Ok(BootstrapInstallInvocation { args, env }) +} + +fn bootstrap_sysroot_env_path(sysroot: &Path) -> Result { + if sysroot.is_absolute() { + Ok(sysroot.to_path_buf()) + } else { + Ok(std::env::current_dir() + .context("Failed to resolve current directory for bootstrap sysroot")? + .join(sysroot)) + } +} + +fn bootstrap_tool_path( + sysroot: &Path, + inherited_path: Option<&OsStr>, + include_target_paths: bool, +) -> Result { + let mut paths = vec![ + sysroot.join("system/tools/bin"), + sysroot.join("system/tools/sbin"), + ]; + if include_target_paths { + paths.extend([ + sysroot.join("system/binaries"), + sysroot.join("system/systembinaries"), + ]); + } + if let Some(inherited) = inherited_path { + paths.extend(std::env::split_paths(inherited)); + } + std::env::join_paths(paths).context("Failed to construct bootstrap PATH") +} + +fn bootstrap_parallel_makeflags() -> String { + if let Ok(value) = std::env::var("LWI_MAKE_FLAGS") { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + + format!("-j{}", bootstrap_parallel_make_jobs()) +} + +fn bootstrap_parallel_make_jobs() -> String { + if let Ok(value) = std::env::var("LWI_MAKE_JOBS") { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + + std::thread::available_parallelism() + .map(|parallelism| parallelism.get()) + .unwrap_or(1) + .to_string() +} + +fn bootstrap_progress_root(config: &config::Config) -> PathBuf { + config.db_dir.join(BOOTSTRAP_DIR).join("progress") +} + +fn retired_package_progress_path(config: &config::Config, package: &str) -> PathBuf { + bootstrap_progress_root(config).join(format!("retired-{package}.done")) +} + +fn package_is_retired(config: &config::Config, package: &str) -> bool { + retired_package_progress_path(config, package).exists() +} + +fn step_is_done( + step: &BookStep, + recipes: &BTreeMap<&str, &GeneratedRecipe>, + progress_root: &Path, + sysroot: &Path, + config: &config::Config, +) -> bool { + match step { + BookStep::Package(package) => { + recipes + .get(package.recipe_id.as_str()) + .is_some_and(|recipe| { + bootstrap_package_is_complete(sysroot, config, recipe).unwrap_or(false) + }) + } + BookStep::Operation(operation) => { + operation_is_complete(progress_root, operation, sysroot, config).unwrap_or(false) + } + } +} + +fn step_chapter(step: &BookStep) -> Option { + section_chapter(step.section()) +} + +fn step_completes_chapter(step: &BookStep, next: Option<&BookStep>, chapter: u8) -> bool { + step_chapter(step) == Some(chapter) && next.and_then(step_chapter) != Some(chapter) +} + +fn step_requires_root(step: &BookStep) -> bool { + match step { + BookStep::Package(package) => build_mode_for_package(package) == BootstrapBuildMode::Chroot, + BookStep::Operation(_) => true, + } +} + +fn build_mode_for_package(package: &BookPackage) -> BootstrapBuildMode { + if use_cross_toolchain_by_default(package) { + return BootstrapBuildMode::Cross; + } + + match package.chapter { + 5 => BootstrapBuildMode::Host, + 6 => BootstrapBuildMode::Cross, + 7..=9 => BootstrapBuildMode::Chroot, + _ => BootstrapBuildMode::Host, + } +} + +fn stage_for_step(step: &BookStep) -> String { + match step { + BookStep::Package(package) => format!("bootstrap-{}", package.recipe_id), + BookStep::Operation(operation) => format!("bootstrap-{}", operation.recipe_id), + } +} + +fn operation_progress_path(progress_root: &Path, operation: &BookOperation) -> PathBuf { + progress_root.join(format!("{}.done", operation.recipe_id)) +} + +fn operation_is_complete( + progress_root: &Path, + operation: &BookOperation, + sysroot: &Path, + config: &config::Config, +) -> Result { + if !operation_progress_path(progress_root, operation).exists() { + return Ok(false); + } + + match operation.kind { + BookOperationKind::CreateEssentialSystemFiles => { + filesystem_package_is_registered(sysroot, config) + } + _ => Ok(true), + } +} + +fn retire_packages_after_chapter7(sysroot: &Path, config: &config::Config) -> Result<()> { + let db_path = config.installed_db_path(sysroot); + let installed = crate::db::get_installed_packages(&db_path).with_context(|| { + format!( + "Failed to inspect installed packages in {}", + db_path.display() + ) + })?; + let needs_removal = CHAPTER7_RETIRED_PACKAGES + .iter() + .any(|package| installed.contains(*package)); + + if needs_removal && !crate::fakeroot::is_root() { + ensure_root_for_bootstrap()?; + } + + for package in CHAPTER7_RETIRED_PACKAGES { + if installed.contains(*package) { + ui::info(format!( + "Retiring bootstrap-only package {package} after chapter 7" + )); + crate::commands::remove_installed_package_with_hooks(package, sysroot, config) + .with_context(|| format!("Failed to retire bootstrap-only package {package}"))?; + } + + let progress_path = retired_package_progress_path(config, package); + if let Some(parent) = progress_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + fs::write( + &progress_path, + "reason = \"chapter7-complete\"\nretired = true\n", + ) + .with_context(|| format!("Failed to write {}", progress_path.display()))?; + } + + Ok(()) +} + +fn run_operation_step( + sysroot: &Path, + config: &config::Config, + operation: &BookOperation, + target: &str, + arch: &str, +) -> Result<()> { + let progress_path = operation_progress_path(&bootstrap_progress_root(config), operation); + if operation_is_complete(&bootstrap_progress_root(config), operation, sysroot, config)? { + ui::info(format!( + "Skipping already completed bootstrap step {} ({})", + operation.title, operation.section + )); + return Ok(()); + } + + ui::info(format!("Running bootstrap step: {}", operation.title)); + match operation.kind { + BookOperationKind::ResetTargetTreeOwnership => reset_target_tree_ownership(sysroot)?, + BookOperationKind::CreateVirtualFilesystemLinkTargets => { + create_virtual_filesystem_link_targets(sysroot)? + } + BookOperationKind::CopyBuildProfile => copy_build_profile(sysroot, target, arch)?, + BookOperationKind::EnterChroot => mark_chroot_entry(sysroot)?, + BookOperationKind::CreateEssentialSystemFiles => { + create_essential_system_files(sysroot)?; + register_filesystem_package(sysroot, config)?; + } + } + + if let Some(parent) = progress_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + fs::write( + &progress_path, + format!( + "section = \"{}\"\noperation = \"{}\"\n", + operation.section, + operation_slug(operation.kind) + ), + ) + .with_context(|| format!("Failed to write {}", progress_path.display()))?; + Ok(()) +} + +fn reset_target_tree_ownership(sysroot: &Path) -> Result<()> { + let status = Command::new("chown") + .arg("-R") + .arg("0:0") + .arg(sysroot) + .status() + .with_context(|| format!("Failed to run chown for {}", sysroot.display()))?; + if !status.success() { + anyhow::bail!( + "Failed to reset target tree ownership for {}: chown exited with {}", + sysroot.display(), + status + ); + } + Ok(()) +} + +fn create_virtual_filesystem_link_targets(sysroot: &Path) -> Result<()> { + system_state::ensure_lbi_layout_paths(sysroot)?; + for rel in [ + "system/devices/pts", + "system/devices/shm", + "system/temporary", + ] { + let path = sysroot.join(rel); + fs::create_dir_all(&path) + .with_context(|| format!("Failed to create {}", path.display()))?; + } + Ok(()) +} + +fn copy_build_profile(sysroot: &Path, target: &str, arch: &str) -> Result<()> { + let profile_dir = sysroot.join("etc/profile.d"); + fs::create_dir_all(&profile_dir) + .with_context(|| format!("Failed to create {}", profile_dir.display()))?; + let profile_path = profile_dir.join("lbi-build.sh"); + let content = format!( + r#"# Generated by `depot bootstrap`. +export LBI_TARGET="{target}" +export LBI_ARCH="{arch}" +export CHOST="$LBI_TARGET" +export CARCH="$LBI_ARCH" +export LBI_ROOT="/" +export LBI_SOURCES="/sources" +export PATH="/system/tools/bin:/system/binaries:/system/systembinaries:/bin:/usr/bin:/sbin:/usr/sbin" + +lbi_configure() {{ + ./configure \ + --target="$LBI_TARGET" \ + --host="$LBI_TARGET" \ + --prefix=/system \ + --bindir=/system/binaries \ + --sbindir=/system/systembinaries \ + --libdir=/system/libraries \ + --includedir=/system/headers \ + --sysconfdir=/system/configuration \ + --localstatedir=/system/variable \ + --datarootdir=/system/share \ + --mandir=/system/documentation/man-pages \ + --infodir=/system/documentation/info \ + "$@" +}} +"# + ); + fs::write(&profile_path, content) + .with_context(|| format!("Failed to write {}", profile_path.display()))?; + Ok(()) +} + +fn mark_chroot_entry(sysroot: &Path) -> Result<()> { + let marker = sysroot.join("var/lib/depot/bootstrap-chroot-ready"); + if let Some(parent) = marker.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + fs::write(&marker, "ready\n").with_context(|| format!("Failed to write {}", marker.display())) +} + +fn create_essential_system_files(sysroot: &Path) -> Result<()> { + write_if_missing( + &sysroot.join("etc/passwd"), + "root:x:0:0:root:/system/charlie:/bin/oksh\n\ +bin:x:1:1:bin:/dev/null:/system/binaries/false\n\ +daemon:x:6:6:Daemon User:/dev/null:/system/binaries/false\n\ +messagebus:x:18:18:D-Bus Message Daemon User:/run/dbus:/system/binaries/false\n\ +uuidd:x:80:80:UUID Generation Daemon User:/dev/null:/system/binaries/false\n\ +nobody:x:65534:65534:Unprivileged User:/dev/null:/system/binaries/false\n", + )?; + write_if_missing( + &sysroot.join("etc/group"), + "root:x:0:\n\ +bin:x:1:daemon\n\ +sys:x:2:\n\ +kmem:x:3:\n\ +tape:x:4:\n\ +tty:x:5:\n\ +daemon:x:6:\n\ +floppy:x:7:\n\ +disk:x:8:\n\ +lp:x:9:\n\ +dialout:x:10:\n\ +audio:x:11:\n\ +video:x:12:\n\ +utmp:x:13:\n\ +clock:x:14:\n\ +cdrom:x:15:\n\ +adm:x:16:\n\ +messagebus:x:18:\n\ +input:x:24:\n\ +mail:x:34:\n\ +kvm:x:61:\n\ +uuidd:x:80:\n\ +wheel:x:97:\n\ +users:x:999:\n\ +nogroup:x:65534:\n", + )?; + write_if_missing( + &sysroot.join("etc/hosts"), + "127.0.0.1 localhost\n::1 localhost\n", + )?; + write_if_missing( + &sysroot.join("etc/fstab"), + "# file system mount point type options dump pass\n", + )?; + write_if_missing( + &sysroot.join("etc/shells"), + "/bin/sh\n/system/binaries/sh\n", + )?; + Ok(()) +} + +fn filesystem_package_is_registered(sysroot: &Path, config: &config::Config) -> Result { + let installed = crate::db::get_installed_packages(&config.installed_db_path(sysroot))?; + Ok(installed.contains(FILESYSTEM_PACKAGE)) +} + +fn register_filesystem_package(sysroot: &Path, config: &config::Config) -> Result<()> { + ui::info("Registering system layout as package filesystem"); + system_state::ensure_lbi_layout_paths(sysroot)?; + create_essential_system_files(sysroot)?; + let staged = tempfile::tempdir().context("Failed to create filesystem package staging dir")?; + system_state::ensure_lbi_layout_paths(staged.path())?; + create_essential_system_files(staged.path())?; + crate::db::register_package( + &config.installed_db_path(sysroot), + &filesystem_package_spec(), + staged.path(), + ) + .context("Failed to register filesystem package") +} + +fn filesystem_package_spec() -> crate::package::PackageSpec { + crate::package::PackageSpec { + package: crate::package::PackageInfo { + name: FILESYSTEM_PACKAGE.to_string(), + real_name: None, + version: "1.0".to_string(), + revision: 1, + description: "Linux by Intent system layout".to_string(), + homepage: "https://www.vertexlinux.net/lbi/".to_string(), + abi_breaking: false, + license: vec!["MIT".to_string()], + }, + packages: Vec::new(), + alternatives: crate::package::Alternatives::default(), + manual_sources: Vec::new(), + source: Vec::new(), + build: crate::package::Build { + build_type: crate::package::BuildType::Meta, + flags: crate::package::BuildFlags::default(), + }, + dependencies: crate::package::Dependencies { + groups: vec![BASE_LAYER.to_string()], + ..crate::package::Dependencies::default() + }, + package_alternatives: BTreeMap::new(), + package_dependencies: BTreeMap::new(), + spec_dir: PathBuf::new(), + } +} + +fn write_if_missing(path: &Path, content: &str) -> Result<()> { + if path.exists() { + return Ok(()); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + fs::write(path, content).with_context(|| format!("Failed to write {}", path.display())) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum RootTransition { + AlreadyRoot, + Reexec(PathBuf), +} + +fn ensure_root_for_bootstrap() -> Result<()> { + match root_transition( + crate::fakeroot::is_root(), + std::env::var_os("PATH").as_deref(), + )? { + RootTransition::AlreadyRoot => Ok(()), + RootTransition::Reexec(helper) => { + let exe = std::env::current_exe().context("Failed to locate depot executable")?; + ui::info(format!( + "Re-executing bootstrap through {} for root-owned and chroot steps", + helper.display() + )); + let status = Command::new(&helper) + .arg(exe) + .args(std::env::args_os().skip(1)) + .status() + .with_context(|| { + format!( + "Failed to re-execute depot bootstrap via {}", + helper.display() + ) + })?; + if status.success() { + std::process::exit(0); + } + anyhow::bail!( + "depot bootstrap via {} failed with status {}", + helper.display(), + status + ); + } + } +} + +fn root_transition(is_root: bool, path: Option<&OsStr>) -> Result { + if is_root { + return Ok(RootTransition::AlreadyRoot); + } + let Some(helper) = privilege_helper(path) else { + anyhow::bail!( + "Bootstrap needs root for ownership and chroot steps, but neither sudo nor doas was found in PATH" + ); + }; + Ok(RootTransition::Reexec(helper)) +} + +fn privilege_helper(path: Option<&OsStr>) -> Option { + for name in ["sudo", "doas"] { + if let Some(found) = find_executable_in_path(name, path) { + return Some(found); + } + } + None +} + +fn find_executable_in_path(name: &str, path: Option<&OsStr>) -> Option { + let path = path?; + for dir in std::env::split_paths(path) { + let candidate = dir.join(name); + if !candidate.is_file() { + continue; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let Ok(metadata) = fs::metadata(&candidate) else { + continue; + }; + if metadata.permissions().mode() & 0o111 == 0 { + continue; + } + } + return Some(candidate); + } + None +} + +#[derive(Default)] +struct BootstrapChrootMountGuard { + mounted: Vec, + created_files: Vec, +} + +impl BootstrapChrootMountGuard { + fn mount_path( + &mut self, + source: &Path, + target: &Path, + fstype: Option<&str>, + flags: MountFlags, + data: Option<&str>, + ) -> Result<()> { + let mut builder = Mount::builder().flags(flags); + if let Some(fstype) = fstype { + builder = builder.fstype(fstype); + } + if let Some(data) = data { + builder = builder.data(data); + } + let mount = builder + .mount(source, target) + .with_context(|| format!("Failed to mount {}", target.display()))?; + self.mounted.push(mount); + Ok(()) + } + + fn prepare_file_mount_target(&mut self, target: &Path) -> Result<()> { + if let Some(parent) = target.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + + match target.symlink_metadata() { + Ok(metadata) if metadata.file_type().is_dir() => { + anyhow::bail!("Mount target is a directory: {}", target.display()); + } + Ok(_) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + fs::File::create(target) + .with_context(|| format!("Failed to create {}", target.display()))?; + self.created_files.push(target.to_path_buf()); + Ok(()) + } + Err(err) => Err(err).with_context(|| format!("Failed to inspect {}", target.display())), + } + } + + fn mount_host_resolver_config(&mut self, rootfs: &Path) -> Result { + let source = Path::new("/etc/resolv.conf"); + if !source.is_file() { + return Ok(false); + } + + let target = rootfs.join("etc/resolv.conf"); + self.prepare_file_mount_target(&target)?; + self.mount_path(source, &target, None, MountFlags::BIND, None)?; + Ok(true) + } +} + +impl Drop for BootstrapChrootMountGuard { + fn drop(&mut self) { + for mount in self.mounted.iter().rev() { + if mount.unmount(UnmountFlags::empty()).is_ok() { + continue; + } + let _ = mount.unmount(UnmountFlags::DETACH); + } + for path in self.created_files.iter().rev() { + let _ = fs::remove_file(path); + } + } +} + +pub(crate) fn run_bootstrap_chroot( + rootfs: &Path, + sources: &Path, + destdir: &Path, + workdir: &str, + script: &str, +) -> Result<()> { + if !crate::fakeroot::is_root() { + anyhow::bail!("internal bootstrap-chroot requires root"); + } + if !sources.is_dir() { + anyhow::bail!( + "Bootstrap source workspace is not a directory: {}", + sources.display() + ); + } + fs::create_dir_all(destdir) + .with_context(|| format!("Failed to create {}", destdir.display()))?; + + let _mounts = mount_bootstrap_chroot(rootfs, sources, destdir)?; + install_bootstrap_chroot_tool_shims(rootfs)?; + let command = format!( + "cd {} && exec /bin/sh {}", + shell_quote(workdir), + shell_quote(script) + ); + let mut cmd = Command::new("chroot"); + cmd.arg(rootfs) + .arg("/bin/sh") + .arg("-lc") + .arg(command) + .env("DEPOT_LBI_INSIDE_CHROOT", "1") + .env("DESTDIR", "/destdir") + .env("DEPOT_PRIMARY_DESTDIR", "/destdir") + .env("DEPOT_LBI_SYSROOT", "/") + .env("DEPOT_STARBUILD_WORKDIR", "/sources") + .env("LBI_ROOT", "/destdir") + .env("LBI_SYSROOT", "/") + .env("LBI_SOURCES", "/sources") + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + apply_bootstrap_chroot_tool_env(&mut cmd); + let status = crate::interrupts::command_status(&mut cmd) + .with_context(|| format!("Failed to execute bootstrap build in {}", rootfs.display()))?; + if !status.success() { + anyhow::bail!("Bootstrap chroot build failed with status {}", status); + } + Ok(()) +} + +fn bootstrap_chroot_tool_env() -> &'static [(&'static str, &'static str)] { + &[ + ("CC", "cc"), + ("CXX", "c++"), + ("AR", "ar"), + ("RANLIB", "ranlib"), + ("NM", "nm"), + ("STRIP", "strip"), + ("PATH", BOOTSTRAP_CHROOT_PATH), + ] +} + +fn apply_bootstrap_chroot_tool_env(cmd: &mut Command) { + for (key, value) in bootstrap_chroot_tool_env() { + cmd.env(key, value); + } + cmd.env_remove("CROSS_COMPILE"); +} + +fn install_bootstrap_chroot_tool_shims(rootfs: &Path) -> Result<()> { + let shim_dir = rootfs.join(BOOTSTRAP_CHROOT_SHIM_DIR.trim_start_matches('/')); + fs::create_dir_all(&shim_dir) + .with_context(|| format!("Failed to create {}", shim_dir.display()))?; + + let id_path = shim_dir.join("id"); + fs::write( + &id_path, + r#"#!/bin/sh +case "${1:-}" in + "" ) + printf '%s\n' 'uid=0(root) gid=0(root) groups=0(root)' + ;; + -u ) + printf '%s\n' 0 + ;; + -g ) + printf '%s\n' 0 + ;; + -G ) + printf '%s\n' 0 + ;; + -un ) + printf '%s\n' root + ;; + -gn ) + printf '%s\n' root + ;; + * ) + printf '%s\n' "id: unsupported bootstrap option: $1" >&2 + exit 1 + ;; +esac +"#, + ) + .with_context(|| format!("Failed to write {}", id_path.display()))?; + make_executable(&id_path)?; + Ok(()) +} + +#[cfg(unix)] +fn make_executable(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let mut perms = fs::metadata(path) + .with_context(|| format!("Failed to stat {}", path.display()))? + .permissions(); + perms.set_mode(0o755); + fs::set_permissions(path, perms).with_context(|| format!("Failed to chmod {}", path.display())) +} + +#[cfg(not(unix))] +fn make_executable(_path: &Path) -> Result<()> { + Ok(()) +} + +fn mount_bootstrap_chroot( + rootfs: &Path, + sources: &Path, + destdir: &Path, +) -> Result { + for rel in [ + "proc", "dev", "dev/pts", "sys", "run", "tmp", "sources", "destdir", + ] { + let path = rootfs.join(rel); + fs::create_dir_all(&path) + .with_context(|| format!("Failed to create {}", path.display()))?; + } + set_world_writable_sticky(&rootfs.join("tmp"))?; + + let mut guard = BootstrapChrootMountGuard::default(); + guard.mount_path( + Path::new("proc"), + &rootfs.join("proc"), + Some("proc"), + MountFlags::NODEV | MountFlags::NOEXEC | MountFlags::NOSUID, + None, + )?; + guard.mount_path( + Path::new("/dev"), + &rootfs.join("dev"), + None, + MountFlags::BIND, + None, + )?; + guard.mount_path( + Path::new("sysfs"), + &rootfs.join("sys"), + Some("sysfs"), + MountFlags::NODEV | MountFlags::NOEXEC | MountFlags::NOSUID, + None, + )?; + if let Err(_err) = guard.mount_path( + Path::new("devpts"), + &rootfs.join("dev/pts"), + Some("devpts"), + MountFlags::NOSUID | MountFlags::NOEXEC, + Some("gid=5,mode=620"), + ) { + guard.mount_path( + Path::new("devpts"), + &rootfs.join("dev/pts"), + Some("devpts"), + MountFlags::NOSUID | MountFlags::NOEXEC, + None, + )?; + } + guard.mount_path( + Path::new("/run"), + &rootfs.join("run"), + None, + MountFlags::BIND, + None, + )?; + guard.mount_path( + sources, + &rootfs.join("sources"), + None, + MountFlags::BIND, + None, + )?; + guard.mount_path( + destdir, + &rootfs.join("destdir"), + None, + MountFlags::BIND, + None, + )?; + if guard.mount_host_resolver_config(rootfs)? { + ui::info("Mounted host resolver configuration for bootstrap chroot"); + } + Ok(guard) +} + +#[cfg(unix)] +fn set_world_writable_sticky(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let mut perms = fs::metadata(path) + .with_context(|| format!("Failed to stat {}", path.display()))? + .permissions(); + perms.set_mode(0o1777); + fs::set_permissions(path, perms) + .with_context(|| format!("Failed to set permissions on {}", path.display())) +} + +#[cfg(not(unix))] +fn set_world_writable_sticky(_path: &Path) -> Result<()> { + Ok(()) +} + +fn shell_quote(input: &str) -> String { + if input.is_empty() { + return "''".to_string(); + } + format!("'{}'", input.replace('\'', "'\\''")) +} + +fn load_book_pdf(args: &BootstrapArgs) -> Result> { + if let Some(path) = &args.book_pdf { + return fs::read(path).with_context(|| format!("Failed to read {}", path.display())); + } + + ui::info(format!("Fetching Linux by Intent book: {}", args.book_url)); + let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); + let client = source::build_blocking_client(&ua, Some(Duration::from_secs(60)))?; + let fetch_url = fresh_book_fetch_url(&args.book_url)?; + let mut response = client + .get(fetch_url.as_str()) + .send() + .with_context(|| format!("Failed to fetch {}", args.book_url))?; + let status = response.status(); + if !status.is_success() { + anyhow::bail!("HTTP error fetching {}: {}", args.book_url, status); + } + let mut body = Vec::new(); + response + .copy_to(&mut body) + .with_context(|| format!("Failed to read {}", args.book_url))?; + Ok(body) +} + +fn generate_recipes( + args: &BootstrapArgs, + config: &config::Config, + packages: &[BookPackage], + target: &str, + arch: &str, +) -> Result> { + let root = config.db_dir.join(BOOTSTRAP_DIR); + let recipe_root = root.join("recipes"); + let page_root = root.join("pages"); + let progress_root = root.join("progress"); + fs::create_dir_all(&recipe_root) + .with_context(|| format!("Failed to create {}", recipe_root.display()))?; + fs::create_dir_all(&page_root) + .with_context(|| format!("Failed to create {}", page_root.display()))?; + fs::create_dir_all(&progress_root) + .with_context(|| format!("Failed to create {}", progress_root.display()))?; + + let manifest = load_source_manifest(args, &root)?; + let mut generated = Vec::new(); + for package in packages { + let (actual_page_url, html) = fetch_package_page(package)?; + let mut package = package.clone(); + package.page_url = actual_page_url; + + let page_path = page_root.join(format!("{}.html", package.recipe_id)); + fs::write(&page_path, &html) + .with_context(|| format!("Failed to write {}", page_path.display()))?; + + let page_recipe = parse_page_recipe(&html, &package)?; + let recipe_dir = recipe_root.join(&package.layer).join(&package.recipe_id); + fs::create_dir_all(&recipe_dir) + .with_context(|| format!("Failed to create {}", recipe_dir.display()))?; + let spec_path = recipe_dir.join(format!("{}.toml", package.name)); + let build_path = recipe_dir.join("build.sh"); + write_generated_recipe( + &spec_path, + &build_path, + &package, + &page_recipe, + &manifest, + target, + arch, + )?; + generated.push(GeneratedRecipe { + package: package.clone(), + spec_path, + progress_path: progress_root.join(format!("{}.done", package.recipe_id)), + }); + } + Ok(generated) +} + +fn load_source_manifest(args: &BootstrapArgs, root: &Path) -> Result { + let manifest_url = book_base_url(&args.book_url)?.join("scripts/sources.manifest")?; + let b2sums_url = book_base_url(&args.book_url)?.join("scripts/sources.b2sums")?; + let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); + let client = source::build_blocking_client(&ua, Some(Duration::from_secs(60)))?; + let manifest_fetch_url = fresh_book_fetch_url(manifest_url.as_str())?; + let b2sums_fetch_url = fresh_book_fetch_url(b2sums_url.as_str())?; + ui::info(format!( + "Fetching Linux by Intent source manifest: {manifest_url}" + )); + let body = client + .get(manifest_fetch_url.as_str()) + .send() + .with_context(|| format!("Failed to fetch {manifest_url}"))? + .error_for_status() + .with_context(|| format!("HTTP error fetching {manifest_url}"))? + .text() + .with_context(|| format!("Failed to read {manifest_url}"))?; + let path = root.join("sources.manifest"); + fs::write(&path, &body).with_context(|| format!("Failed to write {}", path.display()))?; + ui::info(format!( + "Fetching Linux by Intent BLAKE2 source manifest: {b2sums_url}" + )); + let b2sums = client + .get(b2sums_fetch_url.as_str()) + .send() + .with_context(|| format!("Failed to fetch {b2sums_url}"))? + .error_for_status() + .with_context(|| format!("HTTP error fetching {b2sums_url}"))? + .text() + .with_context(|| format!("Failed to read {b2sums_url}"))?; + let b2sums_path = root.join("sources.b2sums"); + fs::write(&b2sums_path, &b2sums) + .with_context(|| format!("Failed to write {}", b2sums_path.display()))?; + Ok(SourceManifest { + entries: parse_source_manifest(&body), + blake2b_512: parse_source_b2sums(&b2sums)?, + }) +} + +fn fetch_page(url: &str) -> Result { + let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); + let client = source::build_blocking_client(&ua, Some(Duration::from_secs(60)))?; + let fetch_url = fresh_book_fetch_url(url)?; + client + .get(fetch_url.as_str()) + .send() + .with_context(|| format!("Failed to fetch {url}"))? + .error_for_status() + .with_context(|| format!("HTTP error fetching {url}"))? + .text() + .with_context(|| format!("Failed to read {url}")) +} + +fn fresh_book_fetch_url(url: &str) -> Result { + let mut parsed = Url::parse(url).with_context(|| format!("Invalid book fetch URL: {url}"))?; + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("System clock is before UNIX epoch")? + .as_secs() + .to_string(); + parsed + .query_pairs_mut() + .append_pair(BOOK_FETCH_CACHE_BUST_PARAM, &stamp); + Ok(parsed.to_string()) +} + +fn fetch_package_page(package: &BookPackage) -> Result<(String, String)> { + match fetch_page(&package.page_url) { + Ok(html) => Ok((package.page_url.clone(), html)), + Err(primary_err) => { + let Some(fallback_url) = fallback_package_page_url(package)? else { + return Err(primary_err); + }; + + ui::info(format!( + "Primary page fetch failed for {}; trying fallback {}", + package.page_url, fallback_url + )); + + fetch_page(&fallback_url) + .map(|html| (fallback_url, html)) + .with_context(|| { + format!( + "Failed to fetch package page for {}. Primary URL was {}; fallback URL was tried after primary error: {primary_err}", + package.name, + package.page_url + ) + }) + } + } +} + +fn fallback_package_page_url(package: &BookPackage) -> Result> { + if !package.page_url.ends_with("-stage2.html") { + return Ok(None); + } + + let mut url = Url::parse(&package.page_url) + .with_context(|| format!("Invalid package page URL: {}", package.page_url))?; + + let path = url.path().to_string(); + let fallback_path = path.replace( + &format!("{}-stage2.html", package.name), + &format!("{}.html", package.name), + ); + + if fallback_path == path { + return Ok(None); + } + + url.set_path(&fallback_path); + Ok(Some(url.to_string())) +} + +fn rewrite_make_flags(input: &str) -> String { + input + .lines() + .map(|line| { + let trimmed = line.trim_start(); + let indent_len = line.len() - trimmed.len(); + let indent = &line[..indent_len]; + + if trimmed == "make" { + format!("{indent}make ${{LWI_MAKE_FLAGS:-}}") + } else if let Some(rest) = trimmed.strip_prefix("make ") { + if rest.contains("LWI_MAKE_FLAGS") || rest.contains("MAKEFLAGS") { + line.to_string() + } else { + format!("{indent}make ${{LWI_MAKE_FLAGS:-}} {rest}") + } + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") +} + +fn rewrite_parallel_job_counts(input: &str) -> String { + input + .replace("\"$(nproc)\"", "\"${LWI_MAKE_JOBS}\"") + .replace("'$(nproc)'", "\"${LWI_MAKE_JOBS}\"") + .replace("$(nproc)", "${LWI_MAKE_JOBS}") +} + +fn write_generated_recipe( + spec_path: &Path, + build_path: &Path, + package: &BookPackage, + recipe: &PageRecipe, + manifest: &SourceManifest, + target: &str, + arch: &str, +) -> Result<()> { + let archive_inputs = recipe + .input_files + .iter() + .filter(|file| is_archive_filename(file)) + .cloned() + .collect::>(); + + let primary_source = archive_inputs.first(); + let mut spec = String::new(); + spec.push_str("[package]\n"); + spec.push_str(&format!("name = \"{}\"\n", toml_escape(&package.name))); + spec.push_str(&format!( + "version = \"{}\"\n", + toml_escape(&package.version) + )); + spec.push_str("revision = 1\n"); + spec.push_str(&format!( + "description = \"{}\"\n", + toml_escape(&recipe.description) + )); + spec.push_str(&format!( + "homepage = \"{}\"\n", + toml_escape(&package.page_url) + )); + spec.push_str(&format!( + "license = \"{}\"\n\n", + toml_escape(&recipe.license) + )); + let provides = bootstrap_package_provides(package); + let replacements = bootstrap_package_replacements(package); + if !provides.is_empty() || !replacements.is_empty() { + spec.push_str("[alternatives]\n"); + write_toml_string_array(&mut spec, "provides", &provides); + write_toml_static_array(&mut spec, "replaces", replacements); + spec.push('\n'); + } + if !recipe.dependencies.is_empty() { + spec.push_str("[dependencies]\n"); + write_toml_string_array(&mut spec, "runtime", &recipe.dependencies); + spec.push('\n'); + } + spec.push_str("[build]\n"); + spec.push_str("type = \"custom\"\n\n"); + spec.push_str("[build.flags]\n"); + spec.push_str("skip_tests = true\n"); + spec.push_str("no_flags = true\n"); + if bootstrap_preserves_static_archives(package) { + spec.push_str("no_delete_static = true\n"); + } + spec.push_str("no_strip = true\n"); + spec.push_str( + "passthrough_env = [\"DEPOT_LBI_CHROOT\", \"DEPOT_LBI_CHROOT_ROOT\", \"DEPOT_LBI_DEPOT_EXE\", \"DEPOT_LBI_SYSROOT\", \"LWI_MAKE_FLAGS\", \"LWI_MAKE_JOBS\"]\n\n", + ); + + if let Some(primary) = primary_source { + let source_url = source_url_for_input(primary, package, recipe, &manifest.entries)?; + let checksum = source_checksum_for_input(primary, &source_url, manifest); + spec.push_str("[[source]]\n"); + spec.push_str(&format!("url = \"{}\"\n", toml_escape(&source_url))); + spec.push_str(&format!("sha256 = \"{}\"\n", toml_escape(&checksum))); + spec.push_str(&format!( + "extract_dir = \"{}\"\n\n", + toml_escape( + recipe + .extract_dir + .as_deref() + .unwrap_or_else(|| strip_archive_suffix(primary)) + ) + )); + } + + for input in &recipe.input_files { + if Some(input) == primary_source || input.trim().is_empty() { + continue; + } + let source_url = source_url_for_input(input, package, recipe, &manifest.entries)?; + let checksum = source_checksum_for_input(input, &source_url, manifest); + spec.push_str("[[manual_sources]]\n"); + spec.push_str(&format!("url = \"{}\"\n", toml_escape(&source_url))); + spec.push_str(&format!("sha256 = \"{}\"\n", toml_escape(&checksum))); + spec.push_str(&format!("dest = \"{}\"\n\n", toml_escape(input))); + } + + let build_script = generated_build_script(package, recipe, target, arch); + fs::write(spec_path, spec) + .with_context(|| format!("Failed to write {}", spec_path.display()))?; + fs::write(build_path, build_script) + .with_context(|| format!("Failed to write {}", build_path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(build_path)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(build_path, perms)?; + } + Ok(()) +} + +fn bootstrap_preserves_static_archives(package: &BookPackage) -> bool { + matches!(package.name.as_str(), "musl" | "rustc") + || package.name.starts_with("llvm") + || package.name.starts_with("musl-") +} + +fn write_toml_string_array(spec: &mut String, key: &str, values: &[String]) { + if values.is_empty() { + return; + } + spec.push_str(key); + spec.push_str(" = ["); + for (idx, value) in values.iter().enumerate() { + if idx > 0 { + spec.push_str(", "); + } + spec.push_str(&format!("\"{}\"", toml_escape(value))); + } + spec.push_str("]\n"); +} + +fn write_toml_static_array(spec: &mut String, key: &str, values: &[&str]) { + if values.is_empty() { + return; + } + spec.push_str(key); + spec.push_str(" = ["); + for (idx, value) in values.iter().enumerate() { + if idx > 0 { + spec.push_str(", "); + } + spec.push_str(&format!("\"{}\"", toml_escape(value))); + } + spec.push_str("]\n"); +} + +fn bootstrap_package_provides(package: &BookPackage) -> Vec { + let provides: &[&str] = match package.name.as_str() { + "musl-libc-pass2" | "musl" => &["musl", "libc"], + "llvm-clang-pass1" => &["llvm", "clang", "lld", "cc", "c++"], + "llvm-runtimes" => &["libunwind", "libcxxabi", "libcxx"], + "llvm-clang-pass2" | "llvm" => &[ + "llvm", + "clang", + "lld", + "cc", + "c++", + "compiler-rt", + "libunwind", + "libcxxabi", + "libcxx", + ], + "byacc" => &["yacc"], + "dash" => &["sh"], + "libressl" => &["openssl", "libssl", "libcrypto"], + "make" => &["gmake"], + "oksh" => &["ksh"], + "om4" => &["m4"], + "pigz" => &["gzip", "gunzip", "zcat"], + "python" => &["python3", "pip"], + "rustc" => &["rust", "cargo"], + "samurai" => &["ninja", "samu"], + "zlib-ng" => &["zlib", "libz"], + _ => &[], + }; + provides + .iter() + .filter(|provided| **provided != package.name) + .map(|provided| (*provided).to_string()) + .collect() +} + +fn bootstrap_package_replacements(package: &BookPackage) -> &'static [&'static str] { + match package.name.as_str() { + "musl-libc-pass2" => &["musl-libc-headers"], + "musl" => &["musl-libc-pass2", "musl-libc-headers", "musl-libc"], + "llvm-clang-pass2" => &["llvm-runtimes"], + "llvm" => &["llvm-clang-pass2", "llvm-clang-pass1", "llvm-runtimes"], + _ => &[], + } +} + +fn generated_build_script( + package: &BookPackage, + recipe: &PageRecipe, + target: &str, + arch: &str, +) -> String { + let mut script = String::new(); + script.push_str("#!/bin/sh\nset -eu\n\n"); + script.push_str(&format!("# Generated from {}\n", package.page_url)); + script.push_str("export LBI_ROOT=\"${DESTDIR:?DESTDIR is required}\"\n"); + script.push_str("export LBI_SYSROOT=\"${DEPOT_LBI_SYSROOT:-$LBI_ROOT}\"\n"); + script.push_str(&format!( + "export LBI_TARGET=\"${{LBI_TARGET:-{target}}}\"\n" + )); + script.push_str(&format!("export LBI_ARCH=\"${{LBI_ARCH:-{arch}}}\"\n")); + script.push_str("export LBI_SOURCES=\"${DEPOT_STARBUILD_WORKDIR:-$PWD}\"\n"); + script.push_str( + r#"if [ -z "${LWI_MAKE_JOBS:-}" ]; then + jobs="$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 1)" + case "$jobs" in + ""|*[!0-9]*) jobs=1 ;; + esac + export LWI_MAKE_JOBS="$jobs" +else + export LWI_MAKE_JOBS +fi +if [ -z "${LWI_MAKE_FLAGS:-}" ]; then + if [ -n "${MAKEFLAGS:-}" ]; then + export LWI_MAKE_FLAGS="$MAKEFLAGS" + else + export LWI_MAKE_FLAGS="-j${LWI_MAKE_JOBS}" + fi +else + export LWI_MAKE_FLAGS +fi +"#, + ); + script.push_str("export LWI_CFLAGS=\"${LWI_CFLAGS:-${CFLAGS:-}}\"\n"); + script.push_str("export LWI_CXXFLAGS=\"${LWI_CXXFLAGS:-$LWI_CFLAGS}\"\n"); + script.push_str("export LBI_CUSTOM_LDFLAGS=\"${LBI_CUSTOM_LDFLAGS:-${LDFLAGS:-}}\"\n\n"); + let uses_cross_toolchain = use_cross_toolchain_by_default(package); + if uses_cross_toolchain { + script.push_str( + r#"lbi_find_cross_tool() { + for name in "$@"; do + for dir in "$LBI_SYSROOT/system/tools/bin"; do + if [ -x "$dir/$LBI_TARGET-$name" ]; then + printf '%s\n' "$dir/$LBI_TARGET-$name" + return 0 + fi + done + if command -v "$LBI_TARGET-$name" >/dev/null 2>&1; then + command -v "$LBI_TARGET-$name" + return 0 + fi + done + for name in "$@"; do + for dir in "$LBI_SYSROOT/system/tools/bin"; do + if [ -x "$dir/$name" ]; then + printf '%s\n' "$dir/$name" + return 0 + fi + done + if command -v "$name" >/dev/null 2>&1; then + command -v "$name" + return 0 + fi + done + echo "depot: could not find cross tool for $LBI_TARGET: $*" >&2 + return 1 +} + +export PATH="$LBI_SYSROOT/system/tools/bin:$PATH" +export CC="${CC:-$(lbi_find_cross_tool clang)}" +export CXX="${CXX:-$(lbi_find_cross_tool clang++)}" +export AR="${AR:-$(lbi_find_cross_tool ar llvm-ar)}" +export AS="${AS:-$CC}" +export NM="${NM:-$(lbi_find_cross_tool nm llvm-nm)}" +export RANLIB="${RANLIB:-$(lbi_find_cross_tool ranlib llvm-ranlib llvm-ar)}" +export OBJCOPY="${OBJCOPY:-$(lbi_find_cross_tool objcopy llvm-objcopy)}" +export OBJDUMP="${OBJDUMP:-$(lbi_find_cross_tool objdump llvm-objdump)}" +export STRIP="${STRIP:-$(lbi_find_cross_tool strip llvm-strip llvm-objcopy)}" +export LD="${LD:-$(lbi_find_cross_tool ld ld.lld lld)}" + +"#, + ); + } + if package.name == "musl-libc-pass2" { + script.push_str( + r#"if [ -z "${LIBCC:-}" ]; then + case "$LBI_ARCH" in + x86_64|amd64) compiler_rt_arch=x86_64 ;; + i?86) compiler_rt_arch=i386 ;; + aarch64|arm64) compiler_rt_arch=aarch64 ;; + *) compiler_rt_arch=$LBI_ARCH ;; + esac + + LIBCC=$( + for dir in \ + "$LBI_SYSROOT/system/tools/lib/clang" \ + "$LBI_SYSROOT/system/tools/lib" \ + "$LBI_SYSROOT/system/libraries/clang" \ + "$LBI_SYSROOT/system/lib/clang" + do + if [ -d "$dir" ]; then + find "$dir" -type f -name "libclang_rt.builtins-${compiler_rt_arch}.a" 2>/dev/null + fi + done | head -n1 + ) + + if [ -z "$LIBCC" ]; then + candidate=$("$CC" -print-file-name="libclang_rt.builtins-${compiler_rt_arch}.a" 2>/dev/null || true) + if [ -n "$candidate" ] && [ "$candidate" != "libclang_rt.builtins-${compiler_rt_arch}.a" ]; then + LIBCC=$candidate + fi + fi + + if [ -z "$LIBCC" ]; then + candidate=$("$CC" -print-libgcc-file-name 2>/dev/null || true) + if [ -n "$candidate" ] && [ "$candidate" != "libgcc.a" ]; then + LIBCC=$candidate + fi + fi + + if [ -z "$LIBCC" ]; then + echo "depot: musl pass2 requires compiler-rt builtins for $compiler_rt_arch before building libc.so" >&2 + exit 1 + fi + + export LIBCC +fi + +"#, + ); + } + let chroot_workdir = recipe + .extract_dir + .as_deref() + .map(|dir| format!("/sources/{dir}")) + .unwrap_or_else(|| "/sources".to_string()); + script.push_str(&format!( + "export DEPOT_LBI_CHROOT_WORKDIR=\"${{DEPOT_LBI_CHROOT_WORKDIR:-{}}}\"\n", + shell_double_quote_literal(&chroot_workdir) + )); + script.push_str( + "export DEPOT_LBI_CHROOT_SCRIPT=\"${DEPOT_LBI_CHROOT_SCRIPT:-$DEPOT_LBI_CHROOT_WORKDIR/build.sh}\"\n", + ); + script.push_str( + r#"if [ "${DEPOT_LBI_CHROOT:-0}" = "1" ] && [ "${DEPOT_LBI_INSIDE_CHROOT:-0}" != "1" ]; then + : "${DEPOT_LBI_CHROOT_ROOT:?DEPOT_LBI_CHROOT_ROOT is required}" + : "${DEPOT_LBI_DEPOT_EXE:?DEPOT_LBI_DEPOT_EXE is required}" + exec "$DEPOT_LBI_DEPOT_EXE" internal bootstrap-chroot \ + --rootfs "$DEPOT_LBI_CHROOT_ROOT" \ + --sources "$LBI_SOURCES" \ + --destdir "$DESTDIR" \ + --workdir "$DEPOT_LBI_CHROOT_WORKDIR" \ + --script "$DEPOT_LBI_CHROOT_SCRIPT" +fi + +"#, + ); + script.push_str( + r#"lbi_configure() { + ./configure \ + --target="$LBI_TARGET" \ + --host="$LBI_TARGET" \ + --prefix=/system \ + --bindir=/system/binaries \ + --sbindir=/system/systembinaries \ + --libdir=/system/libraries \ + --includedir=/system/headers \ + --sysconfdir=/system/configuration \ + --localstatedir=/system/variable \ + --datarootdir=/system/share \ + --mandir=/system/documentation/man-pages \ + --infodir=/system/documentation/info \ + "$@" +} + +lbi_cmake() { + build_dir="$1" + shift + cmake -S . -B "$build_dir" -G Ninja \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR="$LBI_ARCH" \ + -DCMAKE_SYSROOT="$LBI_SYSROOT" \ + -DCMAKE_C_COMPILER_TARGET="$LBI_TARGET" \ + -DCMAKE_CXX_COMPILER_TARGET="$LBI_TARGET" \ + -DCMAKE_ASM_COMPILER_TARGET="$LBI_TARGET" \ + -DCMAKE_INSTALL_PREFIX=/system \ + -DCMAKE_INSTALL_BINDIR=/system/binaries \ + -DCMAKE_INSTALL_SBINDIR=/system/systembinaries \ + -DCMAKE_INSTALL_LIBDIR=/system/libraries \ + -DCMAKE_INSTALL_INCLUDEDIR=/system/headers \ + -DCMAKE_INSTALL_SYSCONFDIR=/system/configuration \ + -DCMAKE_INSTALL_LOCALSTATEDIR=/system/variable \ + -DCMAKE_INSTALL_DATAROOTDIR=/system/share \ + -DCMAKE_INSTALL_MANDIR=/system/documentation/man-pages \ + -DCMAKE_INSTALL_INFODIR=/system/documentation/info \ + "$@" +} + +lbi_meson() { + build_dir=build + use_lbi_cross_file=1 + + case "${1-}" in + "") ;; + -*) ;; + *) + build_dir=$1 + shift + ;; + esac + + for arg in "$@"; do + case "$arg" in + --cross-file|--cross-file=*) use_lbi_cross_file=0 ;; + esac + done + + case "$LBI_ARCH" in + amd64) lbi_meson_cpu_family=x86_64 ;; + i?86) lbi_meson_cpu_family=x86 ;; + arm64) lbi_meson_cpu_family=aarch64 ;; + *) lbi_meson_cpu_family=$LBI_ARCH ;; + esac + + if [ "$use_lbi_cross_file" = "1" ]; then + lbi_meson_cross_file="$PWD/.lbi-meson-cross.ini" + cat > "$lbi_meson_cross_file" <= 7 { + rewritten = restore_chroot_compiler_search_paths(&rewritten); + } + if package.name == "rustc" { + rewritten = restore_rust_bootstrap_runtime_tool_paths(&rewritten); + } + if is_clang_driver_config_command(&rewritten) { + rewritten = restore_clang_driver_config_runtime_paths(&rewritten); + } + if !rewritten.trim().is_empty() { + script.push_str(&rewritten); + script.push_str("\n\n"); + } + } + script +} + +fn use_cross_toolchain_by_default(package: &BookPackage) -> bool { + if !(5..=6).contains(&package.chapter) || package.name == "llvm-clang-pass1" { + return false; + } + if package.chapter == 6 { + return true; + } + section_at_or_after(&package.section, "5.4") +} + +fn section_at_or_after(section: &str, threshold: &str) -> bool { + let left = parse_section_numbers(section); + let right = parse_section_numbers(threshold); + if left.is_empty() || right.is_empty() { + return false; + } + let max_len = left.len().max(right.len()); + for idx in 0..max_len { + let l = *left.get(idx).unwrap_or(&0); + let r = *right.get(idx).unwrap_or(&0); + match l.cmp(&r) { + std::cmp::Ordering::Greater => return true, + std::cmp::Ordering::Less => return false, + std::cmp::Ordering::Equal => {} + } + } + true +} + +fn parse_section_numbers(section: &str) -> Vec { + section + .split('.') + .map(str::trim) + .filter(|part| !part.is_empty()) + .filter_map(|part| part.parse::().ok()) + .collect() +} + +fn apply_bootstrap_package_overrides(package: &BookPackage, command: &str) -> String { + if package.name == "llvm-clang-pass1" { + let command = ensure_llvm_pass1_unsets_destdir(command); + let command = ensure_llvm_cmake_uses_ccache(&command); + let command = ensure_llvm_pass1_builtins_only(&command); + let command = ensure_llvm_pass1_default_target_only(&command); + let command = ensure_llvm_pass1_builtins_sysroot(&command); + let command = ensure_llvm_pass1_default_sysroot(&command); + ensure_llvm_pass1_fresh_build_dir(&command) + } else if package.name == "llvm-runtimes" { + let command = ensure_llvm_runtime_links_skip_missing_crt(command); + ensure_llvm_runtimes_builds_compiler_rt_crt(&command) + } else if package.name == "llvm-clang-pass2" { + let command = ensure_llvm_cmake_uses_ccache(command); + let command = ensure_llvm_runtime_links_skip_missing_crt(&command); + let command = ensure_llvm_pass2_uses_pass1_native_tools(&command); + ensure_llvm_pass2_resource_layout(&command) + } else if matches!(package.name.as_str(), "musl-libc-pass2" | "musl") { + ensure_musl_loader_paths(command) + } else if package.name == "ncurses" { + let command = if package.chapter == 6 { + ensure_ncurses_host_tic_uses_build_toolchain(command) + } else { + command.to_string() + }; + ensure_ncurses_progs_avoid_private_safe_fopen(&command) + } else if package.chapter == 6 && package.name == "file" { + ensure_file_uses_host_magic_compiler(command) + } else if package.name == "gettext-tiny" { + ensure_gettext_tiny_uses_musl_libintl_mode(command) + } else if package.name == "shadow" { + ensure_shadow_uses_staged_account_tools(command) + } else if package.name == "bsddiffutils" { + ensure_bsddiffutils_uses_lbi_header_layout(command) + } else if package.name == "mandoc" { + ensure_mandoc_postinstall_uses_staged_man(command) + } else if package.name == "bmake" { + ensure_bmake_install_avoids_double_destdir(command) + } else if package.name == "ubase" { + ensure_ubase_moves_bin_payload(command) + } else if package.name == "sqlite" { + ensure_sqlite_uses_supported_autosetup_dirs(command) + } else { + command.to_string() + } +} + +fn ensure_ubase_moves_bin_payload(command: &str) -> String { + if !command_runs_install_step(command) { + return command.to_string(); + } + + format!( + "{command}\n{}", + r#"if [ -d "$DESTDIR/system/bin" ]; then + mkdir -p "$DESTDIR/system/binaries" + set -- "$DESTDIR/system/bin"/* + if [ -e "$1" ] || [ -L "$1" ]; then + for path do + mv "$path" "$DESTDIR/system/binaries/" + done + fi + rmdir "$DESTDIR/system/bin" 2>/dev/null || true +fi"# + ) +} + +fn command_runs_install_step(command: &str) -> bool { + command.lines().any(|line| { + let trimmed = line.trim_start(); + !trimmed.starts_with('#') + && (trimmed == "install" + || trimmed.starts_with("install ") + || trimmed.contains(" install") + || trimmed.contains(" install ")) + }) +} + +fn ensure_gettext_tiny_uses_musl_libintl_mode(command: &str) -> String { + command.replace("LIBINTL=musl", "LIBINTL=MUSL") +} + +fn ensure_shadow_uses_staged_account_tools(command: &str) -> String { + command.replace( + "pwconv\ngrpconv\nmkdir -p /etc/default\nuseradd -D --gid 999\npasswd root", + "\"$DESTDIR/system/systembinaries/pwconv\" -R /\nif ! grep -q '^[^:]*:[^:]*:999:' /etc/group; then\n printf '%s\\n' 'users:x:999:' >> /etc/group\nfi\n\"$DESTDIR/system/systembinaries/grpconv\" -R /\nmkdir -p /etc/default\n\"$DESTDIR/system/systembinaries/useradd\" -D -R / --gid 999\n\"$DESTDIR/system/binaries/passwd\" -R / -d root", + ) +} + +fn ensure_bsddiffutils_uses_lbi_header_layout(command: &str) -> String { + command + .replace( + r#"sed -i \ + 's|char\\s*\\*splice(char \\*, char \\*);|char\t*diff_splice(char *, char *);|' \ + src/diff/diff.h"#, + r#"sed -i \ + 's|char[[:space:]]*\*splice(char \*, char \*);|char *diff_splice(char *, char *);|' \ + src/diff/diff.h"#, + ) + .replace( + r#"sed -i \ + 's/\bsplice(/diff_splice(/g' \ + src/diff/diff.c src/diff/diffreg.c"#, + r#"sed -i \ + -e 's|= splice(|= diff_splice(|g' \ + -e 's|^splice(char \*dir, char \*file)|diff_splice(char *dir, char *file)|' \ + src/diff/diff.c src/diff/diffreg.c"#, + ) + .replace( + r#"sed -i \ + 's/u_char ch, \\*p1, \\*p2;/unsigned char ch, *p1, *p2;/' \ + src/cmp/regular.c"#, + r#"sed -i \ + -e 's|u_char ch, \*p1, \*p2;|unsigned char ch, *p1, *p2;|' \ + src/cmp/regular.c"#, + ) + .replace( + "CPPFLAGS=\"--target=$LBI_TARGET --sysroot=$LBI_ROOT -include ../../include/sys/cdefs.h\"", + "CPPFLAGS=\"--target=$LBI_TARGET --sysroot=$LBI_SYSROOT -I../../include -isystem $LBI_SYSROOT/system/headers -Wno-#warnings -include sys/cdefs.h\"", + ) + .replace( + "LDFLAGS=\"--target=$LBI_TARGET --sysroot=$LBI_ROOT $LBI_CUSTOM_LDFLAGS\"", + "LDFLAGS=\"--target=$LBI_TARGET --sysroot=$LBI_SYSROOT -L$LBI_SYSROOT/system/libraries $LBI_CUSTOM_LDFLAGS\"", + ) + .replace( + r#"sed -i '' \ + '/#include /a char *fgetln(FILE *, size_t *);' \ + diff/diff.c"#, + r##"awk ' + { print } + $0 == "#include " { + print "char *fgetln(FILE *, size_t *);" + } +' diff/diff.c > diff/diff.c.new +mv diff/diff.c.new diff/diff.c"##, + ) + .replace( + r#"sed -i \ + '/#include /a char *fgetln(FILE *, size_t *);' \ + diff/diff.c"#, + r##"awk ' + { print } + $0 == "#include " { + print "char *fgetln(FILE *, size_t *);" + } +' diff/diff.c > diff/diff.c.new +mv diff/diff.c.new diff/diff.c"##, + ) + .replace( + r#"sed -i '' \ + '/#include /a char *fgetln(FILE *, size_t *);' \ + diff3/diff3prog.c"#, + r##"awk ' + { print } + $0 == "#include " { + print "char *fgetln(FILE *, size_t *);" + } +' diff3/diff3prog.c > diff3/diff3prog.c.new +mv diff3/diff3prog.c.new diff3/diff3prog.c"##, + ) + .replace( + r#"sed -i \ + '/#include /a char *fgetln(FILE *, size_t *);' \ + diff3/diff3prog.c"#, + r##"awk ' + { print } + $0 == "#include " { + print "char *fgetln(FILE *, size_t *);" + } +' diff3/diff3prog.c > diff3/diff3prog.c.new +mv diff3/diff3prog.c.new diff3/diff3prog.c"##, + ) + .replace( + r#"sed -i '' \ + '/include_directories: \[sysdefs\],/a \ link_with: [libcompat],' \ + diff3/meson.build"#, + r#"awk ' + { print } + index($0, "include_directories: [sysdefs],") { + print " link_with: [libcompat]," + } +' diff3/meson.build > diff3/meson.build.new +mv diff3/meson.build.new diff3/meson.build"#, + ) + .replace( + r#"sed -i \ + '/include_directories: \[sysdefs\],/a \ link_with: [libcompat],' \ + diff3/meson.build"#, + r#"awk ' + { print } + index($0, "include_directories: [sysdefs],") { + print " link_with: [libcompat]," + } +' diff3/meson.build > diff3/meson.build.new +mv diff3/meson.build.new diff3/meson.build"#, + ) +} + +fn ensure_ncurses_host_tic_uses_build_toolchain(command: &str) -> String { + if !command.contains("make -C progs tic") { + return command.to_string(); + } + + r#"rm -rf build obj obj_s obj_g obj_x lib +mkdir -pv build +( + cd build + env \ + CC="${BUILD_CC:-cc}" \ + CXX="${BUILD_CXX:-c++}" \ + CPP="${BUILD_CPP:-cpp}" \ + AR="${BUILD_AR:-ar}" \ + AS="${BUILD_AS:-as}" \ + LD="${BUILD_LD:-ld}" \ + NM="${BUILD_NM:-nm}" \ + RANLIB="${BUILD_RANLIB:-ranlib}" \ + STRIP="${BUILD_STRIP:-strip}" \ + ../configure --prefix="$LBI_SYSROOT/system/tools" AWK=gawk + make -C include + make -C progs tic + install -vm755 progs/tic "$LBI_SYSROOT/system/tools/bin/tic" +) +rm -rf obj obj_s obj_g obj_x lib"# + .to_string() +} + +fn ensure_ncurses_progs_avoid_private_safe_fopen(command: &str) -> String { + if !command.contains("lbi_configure") { + return command.to_string(); + } + + let mut rewritten = command.to_string(); + if !rewritten.contains("--enable-root-access") { + rewritten = rewritten.replace( + " --with-shared \\", + " --with-shared \\\n --enable-root-access \\", + ); + } + if !rewritten.contains("USE_ROOT_ACCESS") { + rewritten = + format!("export CPPFLAGS=\"${{CPPFLAGS:+$CPPFLAGS }}-DUSE_ROOT_ACCESS\"\n{rewritten}"); + } + rewritten +} + +fn ensure_sqlite_uses_supported_autosetup_dirs(command: &str) -> String { + command.replace( + "lbi_configure \\", + "./configure \\\n --prefix=/system \\\n --bindir=/system/binaries \\\n --libdir=/system/libraries \\\n --includedir=/system/headers \\\n --mandir=/system/documentation/man-pages \\", + ) +} + +fn ensure_bmake_install_avoids_double_destdir(command: &str) -> String { + command.replace( + "MAKESYSPATH=mk \\\n./bmake -f Makefile install \\", + "DESTDIR= \\\nMAKESYSPATH=mk \\\n./bmake -f Makefile install \\", + ) +} + +fn ensure_mandoc_postinstall_uses_staged_man(command: &str) -> String { + command.replace( + "MANPAGER=cat man mandoc >/dev/null", + "MANPATH=\"$DESTDIR/system/documentation/man-pages\" MANPAGER=cat \"$DESTDIR/system/binaries/man\" mandoc >/dev/null", + ) +} + +fn ensure_file_uses_host_magic_compiler(command: &str) -> String { + if command.contains("lbi_configure") && command.contains("--host=\"$LBI_TARGET\"") { + return format!( + r#"rm -rf build-host-file +mkdir -p build-host-file +( + cd build-host-file + env \ + CC="${{BUILD_CC:-cc}}" \ + CXX="${{BUILD_CXX:-c++}}" \ + CPP="${{BUILD_CPP:-cpp}}" \ + AR="${{BUILD_AR:-ar}}" \ + RANLIB="${{BUILD_RANLIB:-ranlib}}" \ + ../configure --prefix="$PWD/host-tools" --disable-shared --enable-static --disable-libseccomp + make ${{LWI_MAKE_FLAGS:-}} -C src file +) + +{command}"# + ); + } + + if command.trim() == "make $LWI_MAKE_FLAGS" { + "make $LWI_MAKE_FLAGS FILE_COMPILE=\"$PWD/build-host-file/src/file\"".to_string() + } else { + command.to_string() + } +} + +fn ensure_llvm_pass1_fresh_build_dir(command: &str) -> String { + if command.contains("rm -rf build-llvm") { + return command.to_string(); + } + command.replace( + "mkdir -p build-llvm\ncd build-llvm", + "rm -rf build-llvm\nmkdir -p build-llvm\ncd build-llvm", + ) +} + +fn ensure_llvm_pass1_unsets_destdir(command: &str) -> String { + if command.contains("cmake -G Ninja \"../llvm\"") + && command.contains("-DCMAKE_INSTALL_PREFIX=$LBI_ROOT/system/tools") + && !command.contains("unset DESTDIR") + { + format!("unset DESTDIR\n{command}") + } else { + command.to_string() + } +} + +fn ensure_llvm_pass1_builtins_only(command: &str) -> String { + if !command.contains("-DLLVM_ENABLE_RUNTIMES=\"compiler-rt\"") { + return command.to_string(); + } + + let missing_flags = [ + "-DCOMPILER_RT_BUILD_CRT=OFF", + "-DCOMPILER_RT_BUILD_MEMPROF=OFF", + "-DCOMPILER_RT_BUILD_ORC=OFF", + "-DCOMPILER_RT_BUILD_CTX_PROFILE=OFF", + "-DCOMPILER_RT_INCLUDE_TESTS=OFF", + ] + .into_iter() + .filter(|flag| !command.contains(flag)) + .map(|flag| format!(" {flag} \\")) + .collect::>(); + + if missing_flags.is_empty() { + return command.to_string(); + } + + let block = missing_flags.join("\n"); + let profile_anchor = " -DCOMPILER_RT_BUILD_PROFILE=OFF \\"; + if command.contains(profile_anchor) { + return command.replace(profile_anchor, &format!("{profile_anchor}\n{block}")); + } + + let builtins_anchor = " -DCOMPILER_RT_BUILD_BUILTINS=ON \\"; + command.replace(builtins_anchor, &format!("{builtins_anchor}\n{block}")) +} + +fn ensure_llvm_pass1_default_target_only(command: &str) -> String { + if !command.contains("-DLLVM_ENABLE_RUNTIMES=\"compiler-rt\"") + || command.contains("-DCOMPILER_RT_DEFAULT_TARGET_ONLY=") + { + return command.to_string(); + } + + command.replace( + " -DCOMPILER_RT_BUILD_BUILTINS=ON \\", + " -DCOMPILER_RT_BUILD_BUILTINS=ON \\\n -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \\", + ) +} + +fn ensure_llvm_pass1_builtins_sysroot(command: &str) -> String { + if !command.contains("-DLLVM_ENABLE_RUNTIMES=\"compiler-rt\"") + || command.contains("-DBUILTINS_CMAKE_ARGS=") + { + return command.to_string(); + } + + command.replace( + " -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \\", + " -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \\\n -DBUILTINS_CMAKE_ARGS=\"-DCMAKE_C_FLAGS=--sysroot=$LBI_SYSROOT;-DCMAKE_ASM_FLAGS=--sysroot=$LBI_SYSROOT\" \\", + ) +} + +fn ensure_llvm_pass1_default_sysroot(command: &str) -> String { + if !command.contains("-DLLVM_ENABLE_RUNTIMES=\"compiler-rt\"") { + return command.to_string(); + } + + command.replace( + "-DDEFAULT_SYSROOT=$LBI_ROOT", + "-DDEFAULT_SYSROOT=$LBI_SYSROOT", + ) +} + +fn ensure_musl_loader_paths(command: &str) -> String { + let command = command.replace( + "$LBI_ROOT/usr/lib/ld-musl-${LBI_ARCH}.so.1", + "$LBI_ROOT/system/libraries/ld-musl-${LBI_ARCH}.so.1", + ); + + command.replace( + "ln -snf ./libc.so \\\n \"$LBI_ROOT/system/libraries/ld-musl-${LBI_ARCH}.so.1\"", + "rm -f \"$LBI_ROOT/system/libraries/ld-musl-${LBI_ARCH}.so.1\"\nln -sf ./libc.so \\\n \"$LBI_ROOT/system/libraries/ld-musl-${LBI_ARCH}.so.1\"\nrm -f \"$LBI_ROOT/lib/ld-musl-${LBI_ARCH}.so.1\"\nrmdir \"$LBI_ROOT/lib\" 2>/dev/null || true", + ) +} + +fn ensure_llvm_runtime_links_skip_missing_crt(command: &str) -> String { + if !command.contains("-DLLVM_ENABLE_RUNTIMES=\"libunwind;libcxxabi;libcxx\"") { + return command.to_string(); + } + + let command = ensure_cmake_linker_flag(command, "CMAKE_SHARED_LINKER_FLAGS", "-nostartfiles"); + ensure_cmake_linker_flag(&command, "CMAKE_MODULE_LINKER_FLAGS", "-nostartfiles") +} + +fn ensure_llvm_cmake_uses_ccache(command: &str) -> String { + if command.contains("-DLLVM_CCACHE_BUILD=ON") { + return command.to_string(); + } + + if command.contains("lbi_cmake build-llvm") { + return command.replace( + "lbi_cmake build-llvm \\", + "lbi_cmake build-llvm \\\n -DLLVM_CCACHE_BUILD=ON \\", + ); + } + + if command.contains("cmake -G Ninja \"../llvm\"") { + return command.replace( + "cmake -G Ninja \"../llvm\" \\", + "cmake -G Ninja \"../llvm\" \\\n -DLLVM_CCACHE_BUILD=ON \\", + ); + } + + command.to_string() +} + +fn ensure_llvm_pass2_uses_pass1_native_tools(command: &str) -> String { + let command = command.replace( + "-DLLVM_NATIVE_TOOL_DIR=\"$LBI_ROOT/system/tools/bin\"", + "-DLLVM_NATIVE_TOOL_DIR=\"$LBI_SYSROOT/system/tools/bin\"", + ); + if command.contains("-DLLVM_CONFIG_PATH=") { + return command; + } + + command.replace( + "-DLLVM_NATIVE_TOOL_DIR=\"$LBI_SYSROOT/system/tools/bin\" \\", + "-DLLVM_NATIVE_TOOL_DIR=\"$LBI_SYSROOT/system/tools/bin\" \\\n -DLLVM_CONFIG_PATH=\"$LBI_SYSROOT/system/tools/bin/llvm-config\" \\", + ) +} + +fn ensure_llvm_pass2_resource_layout(command: &str) -> String { + let command = command.replace( + r#"mkdir -p "$LBI_ROOT/system/lib/clang/22/lib/$LBI_TARGET" + +if [ -f "$LBI_ROOT/system/tools/lib/clang/22/lib/$LBI_TARGET/libclang_rt.builtins.a" ]; then + ln -sf "/system/tools/lib/clang/22/lib/$LBI_TARGET/libclang_rt.builtins.a" \ + "$LBI_ROOT/system/lib/clang/22/lib/$LBI_TARGET/libclang_rt.builtins.a" +fi"#, + r#"case "$LBI_ARCH" in + x86_64|amd64) compiler_rt_arch=x86_64 ;; + i?86) compiler_rt_arch=i386 ;; + aarch64|arm64) compiler_rt_arch=aarch64 ;; + *) compiler_rt_arch=$LBI_ARCH ;; +esac + +mkdir -p \ + "$LBI_ROOT/system/libraries/clang/22/lib/linux" \ + "$LBI_ROOT/system/libraries/clang/22/lib/$LBI_TARGET" + +if [ -d "$LBI_ROOT/system/lib/clang/22" ]; then + cp -R "$LBI_ROOT/system/lib/clang/22/." "$LBI_ROOT/system/libraries/clang/22/" + rm -rf "$LBI_ROOT/system/lib/clang/22" + rmdir "$LBI_ROOT/system/lib/clang" "$LBI_ROOT/system/lib" 2>/dev/null || true +fi + +builtins_name="libclang_rt.builtins-${compiler_rt_arch}.a" +builtins_src=$({ find \ + "$LBI_ROOT/system/libraries/clang/22/lib" \ + "$LBI_ROOT/system/lib/clang/22/lib" \ + -type f -name "$builtins_name" 2>/dev/null || true; } | head -n1) + +if [ -z "$builtins_src" ]; then + echo "depot: compiler-rt builtins archive $builtins_name was not installed" >&2 + exit 1 +fi + +clang_resource_root="$LBI_ROOT/system/libraries/clang/22" +mkdir -p "$clang_resource_root/lib/linux" "$clang_resource_root/lib/$LBI_TARGET" +if [ ! -f "$clang_resource_root/lib/linux/$builtins_name" ]; then + install -m644 "$builtins_src" "$clang_resource_root/lib/linux/$builtins_name" +fi +ln -sf "../linux/$builtins_name" \ + "$clang_resource_root/lib/$LBI_TARGET/libclang_rt.builtins.a""#, + ); + + command.replace( + r#"CRTBEGIN_OBJ=$(find "$LBI_ROOT/system/libraries/clang" \ + -type f \( -name 'crtbeginS.o' -o -name 'clang_rt.crtbegin*.o' \) | head -n1) +CRTEND_OBJ=$(find "$LBI_ROOT/system/libraries/clang" \ + -type f \( -name 'crtendS.o' -o -name 'clang_rt.crtend*.o' \) | head -n1) + +CRT_DIR=$(dirname "$CRTBEGIN_OBJ") + +if [ -n "$CRTBEGIN_OBJ" ] && [ -n "$CRTEND_OBJ" ]; then + ln -sf "$(basename "$CRTBEGIN_OBJ")" "$CRT_DIR/crtbeginS.o" + ln -sf "$(basename "$CRTEND_OBJ")" "$CRT_DIR/crtendS.o" + + ln -sf "${CRTBEGIN_OBJ#$LBI_ROOT/system}" "$LBI_ROOT/system/libraries/crtbeginS.o" + ln -sf "${CRTEND_OBJ#$LBI_ROOT/system}" "$LBI_ROOT/system/libraries/crtendS.o" +fi"#, + r#"CRTBEGIN_OBJ=$({ find \ + "$LBI_ROOT/system/libraries/clang" \ + "$LBI_ROOT/system/lib/clang" \ + -type f \( -name 'crtbeginS.o' -o -name 'clang_rt.crtbegin*.o' \) 2>/dev/null || true; } | head -n1) +CRTEND_OBJ=$({ find \ + "$LBI_ROOT/system/libraries/clang" \ + "$LBI_ROOT/system/lib/clang" \ + -type f \( -name 'crtendS.o' -o -name 'clang_rt.crtend*.o' \) 2>/dev/null || true; } | head -n1) + +if [ -n "$CRTBEGIN_OBJ" ] && [ -n "$CRTEND_OBJ" ]; then + CRT_DIR=$(dirname "$CRTBEGIN_OBJ") + ln -sf "$(basename "$CRTBEGIN_OBJ")" "$CRT_DIR/crtbeginS.o" + ln -sf "$(basename "$CRTEND_OBJ")" "$CRT_DIR/crtendS.o" + + install -m644 "$CRTBEGIN_OBJ" "$LBI_ROOT/system/libraries/crtbeginS.o" + install -m644 "$CRTEND_OBJ" "$LBI_ROOT/system/libraries/crtendS.o" +fi"#, + ) +} + +fn ensure_llvm_runtimes_builds_compiler_rt_crt(command: &str) -> String { + let installs_runtimes = command.contains("install") && command.contains("build-runtimes"); + if !installs_runtimes || command.contains("build-compiler-rt-crt") { + return command.to_string(); + } + + format!( + r#"{command} + +cd ../compiler-rt +rm -rf build-compiler-rt-crt +lbi_cmake build-compiler-rt-crt \ + -G Ninja \ + -DCMAKE_C_COMPILER="$CC" \ + -DCMAKE_CXX_COMPILER="$CXX" \ + -DCMAKE_ASM_COMPILER="$CC" \ + -DCMAKE_AR="$AR" \ + -DCMAKE_NM="$NM" \ + -DCMAKE_RANLIB="$RANLIB" \ + -DLLVM_CMAKE_DIR="$LBI_SYSROOT/system/tools/lib/cmake/llvm" \ + -DCMAKE_SYSROOT="$LBI_SYSROOT" \ + -DCMAKE_C_COMPILER_TARGET="$LBI_TARGET" \ + -DCMAKE_CXX_COMPILER_TARGET="$LBI_TARGET" \ + -DCMAKE_ASM_COMPILER_TARGET="$LBI_TARGET" \ + -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \ + -DCMAKE_C_FLAGS="--target=$LBI_TARGET --sysroot=$LBI_SYSROOT $LWI_CFLAGS" \ + -DCMAKE_CXX_FLAGS="--target=$LBI_TARGET --sysroot=$LBI_SYSROOT ${{LWI_CXXFLAGS:-$LWI_CFLAGS}}" \ + -DCMAKE_ASM_FLAGS="--target=$LBI_TARGET --sysroot=$LBI_SYSROOT $LWI_CFLAGS" \ + -DCOMPILER_RT_INSTALL_PATH=/system/tools/lib/clang/22 \ + -DCOMPILER_RT_BUILD_BUILTINS=ON \ + -DCOMPILER_RT_BUILD_CRT=ON \ + -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ + -DCOMPILER_RT_BUILD_MEMPROF=OFF \ + -DCOMPILER_RT_BUILD_ORC=OFF \ + -DCOMPILER_RT_BUILD_PROFILE=OFF \ + -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ + -DCOMPILER_RT_BUILD_XRAY=OFF \ + -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \ + -DCOMPILER_RT_INCLUDE_TESTS=OFF \ + -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \ + -DCMAKE_BUILD_TYPE=Release + +cmake --build build-compiler-rt-crt --target crt $LWI_MAKE_FLAGS + +crt_resource_dir="$LBI_ROOT/system/tools/lib/clang/22/lib/$LBI_TARGET" +mkdir -p "$crt_resource_dir" "$LBI_ROOT/system/libraries" +crtbegin_obj=$(find build-compiler-rt-crt \ + -type f \( -name 'crtbeginS.o' -o -name 'clang_rt.crtbegin*.o' \) 2>/dev/null | head -n1) +crtend_obj=$(find build-compiler-rt-crt \ + -type f \( -name 'crtendS.o' -o -name 'clang_rt.crtend*.o' \) 2>/dev/null | head -n1) + +if [ -z "$crtbegin_obj" ] || [ -z "$crtend_obj" ]; then + echo "depot: compiler-rt CRT build did not produce crtbegin/crtend objects" >&2 + exit 1 +fi + +install -m644 "$crtbegin_obj" "$crt_resource_dir/crtbeginS.o" +install -m644 "$crtend_obj" "$crt_resource_dir/crtendS.o" +install -m644 "$crtbegin_obj" "$LBI_ROOT/system/libraries/crtbeginS.o" +install -m644 "$crtend_obj" "$LBI_ROOT/system/libraries/crtendS.o""# + ) +} + +fn ensure_cmake_linker_flag(command: &str, variable: &str, flag: &str) -> String { + let define = format!("-D{variable}="); + if let Some(start) = command.find(&define) { + let value_start = start + define.len(); + let Some(first) = command[value_start..].chars().next() else { + return command.to_string(); + }; + + let (content_start, content_end) = if first == '"' { + let content_start = value_start + 1; + let Some(rel_end) = command[content_start..].find('"') else { + return command.to_string(); + }; + (content_start, content_start + rel_end) + } else { + let rel_end = command[value_start..] + .find(|ch: char| ch.is_whitespace()) + .unwrap_or(command.len() - value_start); + (value_start, value_start + rel_end) + }; + + let value = &command[content_start..content_end]; + if value.split_whitespace().any(|part| part == flag) { + return command.to_string(); + } + + let mut out = String::with_capacity(command.len() + flag.len() + 1); + out.push_str(&command[..content_start]); + out.push_str(flag); + if !value.is_empty() { + out.push(' '); + out.push_str(value); + } + out.push_str(&command[content_end..]); + return out; + } + + let inserted = format!(" -D{variable}=\"{flag}\" \\"); + let anchor = " -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \\"; + if command.contains(anchor) { + command.replace(anchor, &format!("{anchor}\n{inserted}")) + } else { + command.to_string() + } +} + +fn parse_source_manifest(input: &str) -> Vec { + let mut entries = Vec::new(); + for line in input.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let mut fields = trimmed.split_whitespace(); + let Some(url) = fields.next() else { + continue; + }; + let output_name = fields + .next() + .map(str::to_string) + .unwrap_or_else(|| filename_from_url(url)); + if !output_name.is_empty() { + entries.push(ManifestEntry { + url: url.trim_start_matches("git+").to_string(), + output_name, + }); + } + } + entries +} + +fn parse_source_b2sums(input: &str) -> Result> { + let mut sums = BTreeMap::new(); + for (idx, line) in input.lines().enumerate() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + let mut fields = trimmed.split_whitespace(); + let Some(hash) = fields.next() else { + continue; + }; + let Some(filename) = fields.next() else { + anyhow::bail!("Malformed BLAKE2 source manifest line {}", idx + 1); + }; + if hash.len() != 128 || !hash.chars().all(|ch| ch.is_ascii_hexdigit()) { + anyhow::bail!("Invalid BLAKE2b-512 digest on line {}", idx + 1); + } + let filename = filename.trim_start_matches('*'); + if filename.is_empty() { + anyhow::bail!("Empty filename in BLAKE2 source manifest line {}", idx + 1); + } + let hash = hash.to_ascii_lowercase(); + if let Some(existing) = sums.insert(filename.to_string(), hash.clone()) + && existing != hash + { + anyhow::bail!( + "Conflicting BLAKE2 source checksums for {} on line {}", + filename, + idx + 1 + ); + } + } + Ok(sums) +} + +fn source_checksum_for_input(input: &str, source_url: &str, manifest: &SourceManifest) -> String { + if let Some(hash) = manifest.blake2b_512.get(input) { + return format!("b2sum:{hash}"); + } + + let source_filename = filename_from_url(source_url); + if source_filename != input + && let Some(hash) = manifest.blake2b_512.get(&source_filename) + { + return format!("b2sum:{hash}"); + } + + "skip".to_string() +} + +fn parse_page_recipe(html: &str, package: &BookPackage) -> Result { + let prelude = html.split("

").next().unwrap_or(html); + let input_files = input_files_from_prelude(prelude); + let source_urls = source_urls_from_prelude(prelude); + let extract_dir = extract_dir_from_html(html); + let mut commands = Vec::new(); + for (heading, code) in shell_code_blocks_by_heading(html) { + let heading_lower = heading.to_ascii_lowercase(); + if heading_lower.contains("verify") || heading_lower.contains("quick verification") { + continue; + } + let command = strip_book_extract_scaffolding(&code, extract_dir.as_deref()); + if !command.trim().is_empty() { + commands.push(command); + } + } + if commands.is_empty() { + anyhow::bail!( + "No shell command blocks were found for {} at {}", + package.name, + package.page_url + ); + } + + Ok(PageRecipe { + input_files, + source_urls, + extract_dir, + commands, + dependencies: dependency_names_from_items(list_items_after_heading(html, "Dependencies")), + license: first_list_item_after_heading(html, "Licenses") + .unwrap_or_else(|| "unknown".into()), + description: lead_text(html).unwrap_or_else(|| package.title.clone()), + }) +} + +fn shell_code_blocks_by_heading(html: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + let mut rest = html; + let mut heading = String::new(); + + loop { + let h2_pos = rest.find(" break, + + (Some(h), Some(p)) if h < p => { + let after_h2_tag = &rest[h..]; + let Some(h2_close) = after_h2_tag.find('>') else { + break; + }; + let after = &after_h2_tag[h2_close + 1..]; + let Some(end) = after.find("

") else { + break; + }; + + heading = clean_code_block(&after[..end]); + rest = &after[end + "".len()..]; + } + + (Some(_), Some(p)) | (None, Some(p)) => { + let after_pre = &rest[p..]; + let Some(pre_tag_end) = after_pre.find('>') else { + break; + }; + + let pre_body = &after_pre[pre_tag_end + 1..]; + let Some(pre_end) = pre_body.find("") else { + break; + }; + + let raw = &pre_body[..pre_end]; + + let code = if let Some(code_start_tag) = raw.find("') { + let code_body = &after_code_tag[code_tag_end + 1..]; + let code_body = code_body + .split_once("") + .map(|(before, _)| before) + .unwrap_or(code_body); + + clean_code_block(code_body) + } else { + clean_code_block(raw) + } + } else { + clean_code_block(raw) + }; + + out.push((heading.clone(), code)); + rest = &pre_body[pre_end + "".len()..]; + } + + (Some(h), None) => { + let after_h2_tag = &rest[h..]; + let Some(h2_close) = after_h2_tag.find('>') else { + break; + }; + let after = &after_h2_tag[h2_close + 1..]; + let Some(end) = after.find("") else { + break; + }; + + heading = clean_code_block(&after[..end]); + rest = &after[end + "".len()..]; + } + } + } + + out +} + +fn strip_book_extract_scaffolding(command: &str, extract_dir: Option<&str>) -> String { + let mut out = Vec::new(); + let mut skip_continuation = false; + for line in command.lines() { + let trimmed = line.trim(); + if skip_continuation { + skip_continuation = trimmed.ends_with('\\'); + continue; + } + if trimmed == "cd \"$LBI_SOURCES\"" + || trimmed == "cd /sources" + || trimmed == "cd \"/sources\"" + { + continue; + } + if extract_dir + .is_some_and(|dir| trimmed == format!("cd {dir}") || trimmed == format!("rm -rf {dir}")) + { + continue; + } + if let Some(dir) = extract_dir + && let Some(subdir) = cd_subdir_after_extract_dir(trimmed, dir) + { + let indent_len = line.len() - line.trim_start().len(); + out.push(format!("{}cd {subdir}", &line[..indent_len])); + continue; + } + if is_book_archive_extract_command(trimmed) { + skip_continuation = trimmed.ends_with('\\'); + continue; + } + out.push(line.to_string()); + } + out.join("\n") +} + +fn is_book_archive_extract_command(trimmed: &str) -> bool { + if trimmed.contains("../") { + return false; + } + + let Some(command) = trimmed.split_whitespace().next() else { + return false; + }; + command == "unzip" || (command == "tar" && trimmed.split_whitespace().any(|arg| arg == "-xf")) +} + +fn cd_subdir_after_extract_dir<'a>(trimmed: &'a str, extract_dir: &str) -> Option<&'a str> { + let dir = trimmed.strip_prefix("cd ")?; + let dir = dir.trim_matches('"').trim_matches('\''); + let subdir = dir.strip_prefix(&format!("{extract_dir}/"))?; + (!subdir.is_empty() && !subdir.starts_with('/')).then_some(subdir) +} + +fn rewrite_lbi_command(command: &str, extract_dir: Option<&str>) -> String { + let mut rewritten = command + .replace(" /sources", " \"$LBI_SOURCES\"") + .replace("cd \"/sources\"", "cd \"$LBI_SOURCES\"") + .replace("/sources/", "$LBI_SOURCES/") + .replace( + "$LBI_ROOT/system/tools/bin/", + "$LBI_SYSROOT/system/tools/bin/", + ) + .replace( + "$LBI_ROOT/system/tools/lib/", + "$LBI_SYSROOT/system/tools/lib/", + ) + .replace( + "-DCMAKE_SYSROOT=\"$LBI_ROOT\"", + "-DCMAKE_SYSROOT=\"$LBI_SYSROOT\"", + ) + .replace( + "-DCMAKE_FIND_ROOT_PATH=\"$LBI_ROOT;$LBI_ROOT/system\"", + "-DCMAKE_FIND_ROOT_PATH=\"$LBI_SYSROOT;$LBI_SYSROOT/system\"", + ) + .replace("--sysroot=$LBI_ROOT", "--sysroot=$LBI_SYSROOT") + .replace(">", ">") + .replace("<", "<"); + if let Some(dir) = extract_dir { + rewritten = rewritten + .replace(&format!("cd {dir}\n"), "") + .replace(&format!("rm -rf {dir}\n"), ""); + } + let rewritten = rewrite_absolute_system_paths(&rewritten); + let rewritten = rewrite_make_flags(&rewritten); + rewrite_parallel_job_counts(&rewritten) +} + +fn rewrite_cross_tool_paths(input: &str) -> String { + input + .replace("$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-clang++", "$CXX") + .replace("$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-clang", "$CC") + .replace("$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-ar", "$AR") + .replace("$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-nm", "$NM") + .replace( + "$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-ranlib", + "$RANLIB", + ) + .replace( + "$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-objcopy", + "$OBJCOPY", + ) + .replace( + "$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-objdump", + "$OBJDUMP", + ) + .replace("$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-strip", "$STRIP") + .replace("$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-ld", "$LD") +} + +fn restore_chroot_compiler_search_paths(input: &str) -> String { + input + .replace("-B$DESTDIR/system/", "-B/system/") + .replace("-B${DESTDIR}/system/", "-B/system/") + .replace("-I$DESTDIR/system/", "-I/system/") + .replace("-I${DESTDIR}/system/", "-I/system/") + .replace("-L$DESTDIR/system/", "-L/system/") + .replace("-L${DESTDIR}/system/", "-L/system/") + .replace("-isystem $DESTDIR/system/", "-isystem /system/") + .replace("-isystem ${DESTDIR}/system/", "-isystem /system/") +} + +fn is_clang_driver_config_command(input: &str) -> bool { + input.contains("/configuration/clang/clang.cfg") + || input.contains("/configuration/clang/clang++.cfg") +} + +fn restore_clang_driver_config_runtime_paths(input: &str) -> String { + input + .replace("-B$DESTDIR/system/", "-B/system/") + .replace("-B${DESTDIR}/system/", "-B/system/") + .replace("-I$DESTDIR/system/", "-I/system/") + .replace("-I${DESTDIR}/system/", "-I/system/") + .replace("-L$DESTDIR/system/", "-L/system/") + .replace("-L${DESTDIR}/system/", "-L/system/") + .replace("-isystem $DESTDIR/system/", "-isystem /system/") + .replace("-isystem ${DESTDIR}/system/", "-isystem /system/") + .replace("rpath,$DESTDIR/system/", "rpath,/system/") + .replace("rpath,${DESTDIR}/system/", "rpath,/system/") +} + +fn restore_rust_bootstrap_runtime_tool_paths(input: &str) -> String { + if !input.contains("bootstrap.toml") { + return input.to_string(); + } + + input + .replace("$DESTDIR/system", "/system") + .replace("${DESTDIR}/system", "/system") + .replace("$DESTDIR/system/binaries/cc", "/system/binaries/cc") + .replace("${DESTDIR}/system/binaries/cc", "/system/binaries/cc") + .replace("$DESTDIR/system/binaries/c++", "/system/binaries/c++") + .replace("${DESTDIR}/system/binaries/c++", "/system/binaries/c++") + .replace( + "$DESTDIR/system/binaries/llvm-ar", + "/system/binaries/llvm-ar", + ) + .replace( + "${DESTDIR}/system/binaries/llvm-ar", + "/system/binaries/llvm-ar", + ) + .replace( + "$DESTDIR/system/binaries/llvm-ranlib", + "/system/binaries/llvm-ranlib", + ) + .replace( + "${DESTDIR}/system/binaries/llvm-ranlib", + "/system/binaries/llvm-ranlib", + ) + .replace( + "$DESTDIR/system/binaries/llvm-config", + "/system/binaries/llvm-config", + ) + .replace( + "${DESTDIR}/system/binaries/llvm-config", + "/system/binaries/llvm-config", + ) +} + +fn rewrite_absolute_system_paths(input: &str) -> String { + let mut out = String::with_capacity(input.len()); + let bytes = input.as_bytes(); + let mut i = 0usize; + + while i < bytes.len() { + if bytes[i..].starts_with(b"/system") && is_system_path_boundary(input, i) { + let prefix = &input[..i]; + + let already_destdir = prefix.ends_with("$DESTDIR") + || prefix.ends_with("${DESTDIR}") + || prefix.ends_with("\"$DESTDIR") + || prefix.ends_with("\"${DESTDIR}") + || prefix.ends_with("$LBI_ROOT") + || prefix.ends_with("${LBI_ROOT}") + || prefix.ends_with("\"$LBI_ROOT\"") + || prefix.ends_with("\"${LBI_ROOT}\"") + || prefix.ends_with("$LBI_SYSROOT") + || prefix.ends_with("${LBI_SYSROOT}") + || prefix.ends_with("\"$LBI_SYSROOT\"") + || prefix.ends_with("\"${LBI_SYSROOT}\"") + || prefix_ends_with_shell_assignment(prefix) + || prefix_ends_with_configure_path_option(prefix); + + if !already_destdir { + out.push_str("$DESTDIR"); + } + + out.push_str("/system"); + i += "/system".len(); + } else { + let ch = input[i..] + .chars() + .next() + .expect("byte index must be inside string"); + out.push(ch); + i += ch.len_utf8(); + } + } + + out +} + +fn prefix_ends_with_configure_path_option(prefix: &str) -> bool { + let token_start = prefix + .char_indices() + .rev() + .find_map(|(idx, ch)| { + (ch.is_whitespace() || matches!(ch, ';' | '|' | '&' | '(')) + .then_some(idx + ch.len_utf8()) + }) + .unwrap_or(0); + let token = &prefix[token_start..]; + let Some((option, _)) = token.split_once('=') else { + return false; + }; + matches!( + option, + "--prefix" + | "--exec-prefix" + | "--bindir" + | "--sbindir" + | "--libdir" + | "--includedir" + | "--sysconfdir" + | "--localstatedir" + | "--datarootdir" + | "--datadir" + | "--mandir" + | "--infodir" + | "--with-default-sys-path" + ) +} + +fn is_system_path_boundary(input: &str, start: usize) -> bool { + let after = start + "/system".len(); + matches!( + input.as_bytes().get(after).copied(), + None | Some(b'/') + | Some(b':') + | Some(b'"') + | Some(b'\'') + | Some(b' ') + | Some(b'\t') + | Some(b'\n') + ) +} + +fn prefix_ends_with_shell_assignment(prefix: &str) -> bool { + let token_start = prefix + .char_indices() + .rev() + .find_map(|(idx, ch)| { + (ch.is_whitespace() || matches!(ch, ';' | '|' | '&' | '(')) + .then_some(idx + ch.len_utf8()) + }) + .unwrap_or(0); + let token = &prefix[token_start..]; + let Some((name, _)) = token.split_once('=') else { + return false; + }; + let name = name.strip_prefix("-D").unwrap_or(name); + let mut chars = name.chars(); + let Some(first) = chars.next() else { + return false; + }; + (first == '_' || first.is_ascii_alphabetic()) + && chars.all(|ch| ch == '_' || ch.is_ascii_alphanumeric()) +} + +fn source_url_for_input( + input: &str, + package: &BookPackage, + recipe: &PageRecipe, + manifest: &[ManifestEntry], +) -> Result { + for url in &recipe.source_urls { + if filename_from_url(url) == input || url.ends_with(input) { + return Ok(url.trim_start_matches("git+").to_string()); + } + } + if let Some(entry) = manifest.iter().find(|entry| entry.output_name == input) { + return Ok(entry.url.clone()); + } + let mut filename_matches = manifest + .iter() + .filter(|entry| filename_from_url(&entry.url) == input) + .collect::>(); + if filename_matches.len() > 1 { + filename_matches.retain(|entry| manifest_entry_matches_package(entry, package)); + } + if let Some(entry) = filename_matches.first() { + return Ok(entry.url.clone()); + } + if input.ends_with(".patch") || input.ends_with(".diff") { + return Ok(Url::parse(&package.page_url)? + .join(&format!("../../patches/{input}"))? + .to_string()); + } + anyhow::bail!( + "Could not find source URL for input file {} required by {}", + input, + package.name + ) +} + +fn manifest_entry_matches_package(entry: &ManifestEntry, package: &BookPackage) -> bool { + let package = normalize_source_match_token(&package.name); + !package.is_empty() + && (normalize_source_match_token(&entry.output_name).contains(&package) + || normalize_source_match_token(&entry.url).contains(&package)) +} + +fn source_urls_from_prelude(html: &str) -> Vec { + code_values(html) + .into_iter() + .filter(|code| source_value_looks_like_url(code)) + .collect() +} + +fn input_files_from_prelude(html: &str) -> Vec { + let explicit_files = input_files_from_explicit_metadata(html); + if !explicit_files.is_empty() { + return explicit_files; + } + + collect_input_files_from_code_values(code_values(html)) +} + +fn input_files_from_explicit_metadata(html: &str) -> Vec { + let mut input_files = Vec::new(); + let mut seen_input_files = BTreeSet::new(); + + for paragraph in paragraph_bodies(html) { + if !paragraph_has_input_file_metadata_label(paragraph) { + continue; + } + + for file in collect_input_files_from_code_values(code_values(paragraph)) { + if seen_input_files.insert(file.clone()) { + input_files.push(file); + } + } + } + + input_files +} + +fn paragraph_bodies(html: &str) -> Vec<&str> { + let mut paragraphs = Vec::new(); + let mut rest = html; + while let Some(start) = rest.find("') else { + break; + }; + let body_start = start + tag_end + 1; + let after_body = &rest[body_start..]; + let Some(body_end) = after_body.find("

") else { + break; + }; + + paragraphs.push(&rest[body_start..body_start + body_end]); + rest = &after_body[body_end + "

".len()..]; + } + paragraphs +} + +fn paragraph_has_input_file_metadata_label(paragraph: &str) -> bool { + let text = strip_html_tags(paragraph).to_ascii_lowercase(); + let Some((label, _)) = text.split_once(':') else { + return false; + }; + matches!( + label.trim(), + "input assumption" + | "input assumptions" + | "input file" + | "input files" + | "source package" + | "source packages" + | "source archive" + | "source archives" + | "patch" + | "patches" + ) +} + +fn collect_input_files_from_code_values(code_values: Vec) -> Vec { + let mut input_files = Vec::new(); + let mut seen_input_files = BTreeSet::new(); + for code in code_values { + for file in words_that_look_like_input_files(&code) { + if seen_input_files.insert(file.clone()) { + input_files.push(file); + } + } + } + input_files +} + +fn extract_dir_from_html(html: &str) -> Option { + for (_, code) in shell_code_blocks_by_heading(html) { + for line in code.lines() { + let trimmed = line.trim(); + if trimmed == "cd \"$LBI_SOURCES\"" + || trimmed == "cd /sources" + || trimmed == "cd \"/sources\"" + { + continue; + } + if let Some(dir) = trimmed.strip_prefix("cd ") { + let dir = dir.trim_matches('"').trim_matches('\''); + if !dir.starts_with('$') && !dir.starts_with('/') && !dir.contains(' ') { + return dir.split('/').next().map(str::to_string); + } + } + } + } + None +} + +fn code_values(html: &str) -> Vec { + let mut values = Vec::new(); + let mut rest = html; + while let Some(start) = rest.find("') else { + break; + }; + let value_start = start + close + 1; + let after_value = &rest[value_start..]; + let Some(end) = after_value.find("") else { + break; + }; + values.push(html_decode(&after_value[..end])); + rest = &after_value[end + "".len()..]; + } + values +} + +fn words_that_look_like_input_files(input: &str) -> Vec { + input + .split(|c: char| { + c.is_whitespace() || c == ',' || c == '(' || c == ')' || c == '"' || c == '\'' + }) + .map(|word| word.trim_matches(|c: char| c == '`' || c == '.' || c == ';')) + .filter(|word| { + !source_value_looks_like_url(word) + && (is_archive_filename(word) + || word.ends_with(".patch") + || word.ends_with(".diff") + || word.ends_with(".pem")) + }) + .map(str::to_string) + .collect() +} + +fn source_value_looks_like_url(input: &str) -> bool { + let lower = input.to_ascii_lowercase(); + lower.starts_with("http://") + || lower.starts_with("https://") + || lower.starts_with("git+http://") + || lower.starts_with("git+https://") +} + +fn first_list_item_after_heading(html: &str, heading: &str) -> Option { + list_items_after_heading(html, heading).into_iter().next() +} + +fn list_items_after_heading(html: &str, heading: &str) -> Vec { + let marker = format!("

{heading}:

"); + let Some(after) = html.split_once(&marker).map(|(_, after)| after) else { + return Vec::new(); + }; + let list = after + .split_once("") + .map(|(list, _)| list) + .unwrap_or(after); + let mut items = Vec::new(); + let mut rest = list; + while let Some((_, after_li)) = rest.split_once("
  • ") { + let Some((item, after_item)) = after_li.split_once("
    • ") else { + break; + }; + let item = strip_html_tags(item); + if !item.is_empty() { + items.push(item); + } + rest = after_item; + } + items +} + +fn dependency_names_from_items(items: Vec) -> Vec { + let mut deps = Vec::new(); + let mut seen = BTreeSet::new(); + for item in items { + for dep in normalize_dependency_item(&item) { + if seen.insert(dep.clone()) { + deps.push(dep); + } + } + } + deps +} + +fn normalize_dependency_item(item: &str) -> Vec { + let item = item.replace('`', ""); + let lower = item.trim().to_ascii_lowercase(); + if lower.is_empty() + || lower.contains("(host)") + || lower.starts_with("host ") + || lower.contains("host c compiler") + { + return Vec::new(); + } + + let dep = if lower.contains("musl") || lower == "pthreads" { + "musl" + } else if lower.contains("clang/llvm") || lower.contains("llvm toolchain") { + "llvm" + } else if lower.starts_with("compiler-rt") { + "compiler-rt" + } else if lower.starts_with("libunwind") { + "libunwind" + } else if lower.starts_with("libcxxabi") { + "libcxxabi" + } else if lower.starts_with("libcxx") { + "libcxx" + } else if lower.starts_with("lld") { + "lld" + } else if lower.starts_with("llvm") { + "llvm" + } else if lower.starts_with("clang") || lower.starts_with("c++ compiler") { + "clang" + } else if lower.starts_with("cargo") { + "cargo" + } else if lower.starts_with("rustc") { + "rustc" + } else if lower.starts_with("libressl") { + "libressl" + } else if lower.starts_with("sqlite") { + "sqlite" + } else if lower.starts_with("zlib-ng") { + "zlib-ng" + } else if lower.starts_with("xz") { + "xz" + } else if lower.starts_with("zstd") { + "zstd" + } else if lower.starts_with("curl") { + "curl" + } else if lower.starts_with("pkgconf") { + "pkgconf" + } else if lower.starts_with("ca-certificates") { + "ca-certificates" + } else if lower.starts_with("python-flit-core") || lower.starts_with("flit-core") { + "python-flit-core" + } else if lower.starts_with("python-packaging") || lower.starts_with("packaging") { + "python-packaging" + } else if lower.starts_with("python-wheel") || lower.starts_with("wheel") { + "python-wheel" + } else if lower.starts_with("python-setuptools") || lower.starts_with("setuptools") { + "python-setuptools" + } else if lower.starts_with("python") { + "python" + } else if lower.starts_with("pip") { + "pip" + } else if lower.starts_with("cmake") { + "cmake" + } else if lower.starts_with("meson") { + "meson" + } else if lower.starts_with("samurai") { + "samurai" + } else if lower.starts_with("ninja") { + "ninja" + } else if lower.starts_with("bmake") { + "bmake" + } else if lower.starts_with("make") { + "make" + } else if lower.starts_with("bison") + || lower.starts_with("yacc") + || lower.contains("yacc-compatible") + { + "byacc" + } else if lower.starts_with("lex") || lower.starts_with("flex") { + "flex" + } else if lower == "m4" || lower.starts_with("m4 ") { + "m4" + } else if lower.starts_with("om4") { + "om4" + } else if lower.starts_with("ncurses") { + "ncurses" + } else if lower.starts_with("libedit") { + "libedit" + } else if lower.starts_with("libarchive") { + "libarchive" + } else if lower.starts_with("bheaded") { + "bheaded" + } else if lower.starts_with("awk") { + "awk" + } else { + return Vec::new(); + }; + + vec![dep.to_string()] +} + +fn lead_text(html: &str) -> Option { + let after = html.split_once("

    ")?.1; + let text = after.split_once("

    ")?.0; + Some(strip_html_tags(text)) +} + +fn strip_html_tags(input: &str) -> String { + let mut out = String::new(); + let mut in_tag = false; + for c in input.chars() { + match c { + '<' => in_tag = true, + '>' => in_tag = false, + _ if !in_tag => out.push(c), + _ => {} + } + } + html_decode(out.trim()) +} + +fn html_decode(input: &str) -> String { + input + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") + .replace(">", ">") + .replace("<", "<") + .replace("&", "&") +} + +fn clean_code_block(input: &str) -> String { + let stripped = strip_html_tags(input); + stripped + .lines() + .map(|line| line.trim_end()) + .collect::>() + .join("\n") +} + +fn is_archive_filename(input: &str) -> bool { + let lower = input.to_ascii_lowercase(); + lower.ends_with(".tar.gz") + || lower.ends_with(".tgz") + || lower.ends_with(".tar.xz") + || lower.ends_with(".txz") + || lower.ends_with(".tar.bz2") + || lower.ends_with(".tbz2") + || lower.ends_with(".zip") + || lower.ends_with(".tar") +} + +fn strip_archive_suffix(input: &str) -> &str { + for suffix in [ + ".tar.gz", ".tar.xz", ".tar.bz2", ".tgz", ".txz", ".tbz2", ".zip", ".tar", + ] { + if let Some(stripped) = input.strip_suffix(suffix) { + return stripped; + } + } + input +} + +fn filename_from_url(url: &str) -> String { + let clean = url + .trim_start_matches("git+") + .split('#') + .next() + .unwrap_or(url); + Url::parse(clean) + .ok() + .and_then(|url| { + url.path_segments() + .and_then(|mut segments| segments.next_back().map(str::to_string)) + }) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| clean.rsplit('/').next().unwrap_or(clean).to_string()) +} + +fn normalize_source_match_token(input: &str) -> String { + input + .chars() + .filter(|c| c.is_ascii_alphanumeric()) + .flat_map(char::to_lowercase) + .collect() +} + +fn toml_escape(input: &str) -> String { + input.replace('\\', "\\\\").replace('"', "\\\"") +} + +fn shell_double_quote_literal(input: &str) -> String { + input + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('$', "\\$") + .replace('`', "\\`") +} + +fn version_from_title(title: &str) -> String { + title + .split_whitespace() + .rev() + .find(|word| word.chars().any(|c| c.is_ascii_digit())) + .map(|word| { + word.trim_matches(|c: char| c == ',' || c == ';' || c == ':' || c == '.') + .to_string() + }) + .unwrap_or_else(|| "0".to_string()) +} + +fn page_slug_from_title(title: &str, package: &str) -> String { + let lower = normalize_title(title); + + match lower.as_str() { + s if s.starts_with("musl libc final pass") => "musl-libc-final-pass".to_string(), + s if s.starts_with("llvm final") => "llvm-final".to_string(), + + // These chapter 8 pages are stage-2 packages in the title, + // but the actual LBI page filenames do not include "-stage2". + s if s.starts_with("zstd") => "zstd".to_string(), + s if s.starts_with("samurai") => "samurai".to_string(), + + s if s.contains("stage 2") => format!("{package}-stage2"), + + _ => package.to_string(), + } +} + +fn book_base_url(book_url: &str) -> Result { + let mut url = Url::parse(book_url).with_context(|| format!("Invalid book URL: {book_url}"))?; + { + let mut segments = url + .path_segments_mut() + .map_err(|_| anyhow::anyhow!("Book URL cannot be a base URL: {book_url}"))?; + segments.pop(); + segments.push(""); + } + Ok(url) +} + +fn parse_book_steps(text: &str, book_url: &str) -> Result> { + let book_base = book_base_url(book_url)?; + let mut sections = Vec::<(String, String)>::new(); + let mut seen_sections = BTreeSet::::new(); + for line in text.lines() { + if let Some((section, title)) = parse_section_line(line) + && seen_sections.insert(section.clone()) + { + sections.push((section, title)); + } + } + + let mut steps = Vec::new(); + for (section, title) in sections { + let Some(chapter) = section_chapter(§ion) else { + continue; + }; + if let Some(kind) = operation_kind_from_title(&title) { + steps.push(BookStep::Operation(BookOperation { + section: section.clone(), + title: title.clone(), + kind, + recipe_id: format!("{}-{}", section.replace('.', "-"), operation_slug(kind)), + })); + continue; + } + + let Some(name) = package_name_from_title(&title) else { + continue; + }; + let slug = page_slug_from_title(&title, &name); + let page_url = book_base + .join(&format!("chapters/chapter{chapter}/{slug}.html"))? + .to_string(); + let recipe_id = format!("{}-{}", section.replace('.', "-"), name); + steps.push(BookStep::Package(BookPackage { + chapter, + section: section.clone(), + title: title.clone(), + version: version_from_title(&title), + layer: layer_for_package(chapter, &name), + name, + page_url, + recipe_id, + })); + } + Ok(steps) +} + +fn packages_from_steps(steps: &[BookStep]) -> Vec { + steps + .iter() + .filter_map(|step| match step { + BookStep::Package(package) => Some(package.clone()), + BookStep::Operation(_) => None, + }) + .collect() +} + +impl BookStep { + fn section(&self) -> &str { + match self { + BookStep::Package(package) => &package.section, + BookStep::Operation(operation) => &operation.section, + } + } +} + +fn packages_by_layer(packages: &[BookPackage]) -> BTreeMap> { + let mut layers = BTreeMap::>::new(); + let mut seen = BTreeMap::>::new(); + for package in packages { + if seen + .entry(package.layer.clone()) + .or_default() + .insert(package.name.clone()) + { + layers + .entry(package.layer.clone()) + .or_default() + .push(package.name.clone()); + } + } + layers +} + +fn bootstrap_layer_packages_for_state( + layers: &BTreeMap>, + layer: &str, +) -> Vec { + let mut packages = layers.get(layer).cloned().unwrap_or_default(); + if layer == BASE_LAYER && !packages.iter().any(|package| package == FILESYSTEM_PACKAGE) { + packages.insert(0, FILESYSTEM_PACKAGE.to_string()); + } + filter_retired_layer_packages(layer, packages) +} + +fn filter_retired_layer_packages(layer: &str, packages: Vec) -> Vec { + if layer != TEMP_LAYER { + return packages; + } + + packages + .into_iter() + .filter(|package| !CHAPTER7_RETIRED_PACKAGES.contains(&package.as_str())) + .collect() +} + +fn parse_section_line(line: &str) -> Option<(String, String)> { + let trimmed = line + .trim_start_matches(|c: char| c == '\u{c}' || c.is_whitespace()) + .trim_start_matches(['▪', '•', '◦', '*', '-']) + .trim(); + let mut fields = trimmed.split_whitespace(); + let section_raw = fields.next()?.trim_end_matches('.'); + if !section_raw.contains('.') { + return None; + } + if !section_raw + .split('.') + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) + { + return None; + } + let chapter = section_chapter(section_raw)?; + if !(5..=9).contains(&chapter) { + return None; + } + let title = fields.collect::>().join(" "); + if title.is_empty() { + return None; + } + Some((section_raw.to_string(), title)) +} + +fn section_chapter(section: &str) -> Option { + section.split('.').next()?.parse::().ok() +} + +fn package_name_from_title(title: &str) -> Option { + let lower = normalize_title(title); + if should_skip_title(&lower) { + return None; + } + + let mapped = match lower.as_str() { + s if s.starts_with("linux api headers") => "linux-api-headers", + s if s.starts_with("musl libc headers") => "musl-libc-headers", + s if s.starts_with("llvm/clang pass 1") => "llvm-clang-pass1", + s if s.starts_with("musl libc pass 2") => "musl-libc-pass2", + s if s.starts_with("llvm runtimes") => "llvm-runtimes", + s if s.starts_with("llvm/clang pass 2") => "llvm-clang-pass2", + s if s.starts_with("musl libc final pass") => "musl", + s if s.starts_with("gnu make") => "make", + s if s.starts_with("bsd-diffutils") => "bsddiffutils", + s if s.starts_with("patch stage 2") => "bsdpatch", + s if s.starts_with("llvm final") => "llvm", + s if s.starts_with("cmake") => "cmake", + s if s.starts_with("shadow") => "shadow", + s if s.starts_with("libressl") => "libressl", + s if s.starts_with("python-flit-core") => "python-flit-core", + s if s.starts_with("ca-certificates") => "ca-certificates", + _ => return generic_package_name(&lower), + }; + Some(mapped.to_string()) +} + +fn normalize_title(title: &str) -> String { + let no_parens = title + .split_once('(') + .map(|(before, _)| before) + .unwrap_or(title); + no_parens + .trim() + .trim_end_matches('.') + .split_whitespace() + .collect::>() + .join(" ") + .to_ascii_lowercase() +} + +fn should_skip_title(title: &str) -> bool { + if title.starts_with("copy selected build variables and helper functions into target") { + return false; + } + + matches!( + title, + "introduction" + | "sources" + | "final checks" + | "reset target tree ownership to root" + | "create virtual filesystem link targets" + | "copy selected build variables and helper functions into target profile" + | "enter chroot environment" + | "create essential system files" + | "cleanup" + | "dinit service setup" + ) +} + +fn operation_kind_from_title(title: &str) -> Option { + let title = normalize_title(title); + if title.starts_with("copy selected build variables and helper functions into target") { + return Some(BookOperationKind::CopyBuildProfile); + } + + match title.as_str() { + "reset target tree ownership to root" => Some(BookOperationKind::ResetTargetTreeOwnership), + "create virtual filesystem link targets" => { + Some(BookOperationKind::CreateVirtualFilesystemLinkTargets) + } + "enter chroot environment" => Some(BookOperationKind::EnterChroot), + "create essential system files" => Some(BookOperationKind::CreateEssentialSystemFiles), + _ => None, + } +} + +fn operation_slug(kind: BookOperationKind) -> &'static str { + match kind { + BookOperationKind::ResetTargetTreeOwnership => "reset-ownership", + BookOperationKind::CreateVirtualFilesystemLinkTargets => "virtual-filesystems", + BookOperationKind::CopyBuildProfile => "copy-build-profile", + BookOperationKind::EnterChroot => "enter-chroot", + BookOperationKind::CreateEssentialSystemFiles => "essential-files", + } +} + +fn generic_package_name(title: &str) -> Option { + let mut words = Vec::new(); + for word in title.split_whitespace() { + let cleaned = word.trim_matches(|c: char| { + c == ',' || c == ';' || c == ':' || c == '.' || c == '(' || c == ')' + }); + if cleaned.is_empty() || !cleaned.chars().any(|c| c.is_ascii_alphanumeric()) { + continue; + } + if cleaned.eq_ignore_ascii_case("stage") + || cleaned.eq_ignore_ascii_case("pass") + || cleaned.eq_ignore_ascii_case("final") + || cleaned.eq_ignore_ascii_case("git") + || cleaned.eq_ignore_ascii_case("main") + || cleaned.eq_ignore_ascii_case("master") + || cleaned.eq_ignore_ascii_case("snapshot") + || cleaned.chars().next().is_some_and(|c| c.is_ascii_digit()) + { + break; + } + words.push(cleaned); + } + if words.is_empty() { + return None; + } + let package = words.join("-").replace('/', "-"); + package + .chars() + .any(|c| c.is_ascii_alphanumeric()) + .then_some(package) +} + +fn layer_for_package(chapter: u8, package: &str) -> String { + if package == "linux-api-headers" { + return DEVEL_LAYER.to_string(); + } + + if chapter == 5 || chapter == 6 { + return TEMP_LAYER.to_string(); + } + + if is_devel_package(package) { + DEVEL_LAYER.to_string() + } else { + BASE_LAYER.to_string() + } +} + +fn is_devel_package(package: &str) -> bool { + matches!( + package, + "byacc" + | "bmake" + | "cmake" + | "flex" + | "llvm" + | "make" + | "pkgconf" + | "python" + | "python-flit-core" + | "python-packaging" + | "python-setuptools" + | "python-wheel" + | "rustc" + | "meson" + | "samurai" + | "sqlite" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::TestEnv; + use std::ffi::OsStr; + + const SAMPLE_TOC: &str = r#" + 5.1 Linux API Headers 7.0 + 5.3 llvm/clang pass 1 22.1.3 + 6.17 GNU Make 4.4.1 + 7.1 Introduction + ▪ 7.2 Reset Target Tree Ownership to root + ▪ 7.3 Create virtual filesystem link targets + ▪ 7.4 Copy selected build variables and helper functions into target + profile + ▪ 7.5 Enter chroot environment + ▪ 7.6 Create essential system files + 7.7 gettext-tiny 0.3.3 + 7.8 byacc 20260126 + 8.2 iana-etc 20260409 + 8.4 pigz stage 2 2.8 + 8.14.1 ca-certificates 2026-03-19 + ▪ 8.17 libffi 3.5.2 + ▪ 8.18 python 3.14.4 + ▪ 8.20 Python-Packaging 26.2 + ▪ 8.21 Python-Wheel 0.47.0 + ▪ 8.22 Python-Setuptools 82.0.1 + ▪ 8.23 Meson 1.11.1 + 8.29 CMake 4.3.2 + 8.34 LLVM final 22.1.3 + 9.2 Limine 11.4.1 + 9.3 dinit service setup + "#; + + #[test] + fn parses_book_packages_into_layers() { + let steps = + parse_book_steps(SAMPLE_TOC, "https://www.vertexlinux.net/lbi/book.pdf").unwrap(); + let packages = packages_from_steps(&steps); + let layers = packages_by_layer(&packages); + assert_eq!(layers[TEMP_LAYER], vec!["llvm-clang-pass1", "make"]); + assert_eq!( + layers[BASE_LAYER], + vec![ + "gettext-tiny", + "iana-etc", + "pigz", + "ca-certificates", + "libffi", + "limine" + ] + ); + assert_eq!( + layers[DEVEL_LAYER], + vec![ + "linux-api-headers", + "byacc", + "python", + "python-packaging", + "python-wheel", + "python-setuptools", + "meson", + "cmake", + "llvm" + ] + ); + assert_eq!(packages[0].recipe_id, "5-1-linux-api-headers"); + assert_eq!( + packages[0].page_url, + "https://www.vertexlinux.net/lbi/chapters/chapter5/linux-api-headers.html" + ); + assert_eq!(packages.last().unwrap().name, "limine"); + assert!(packages.iter().all(|package| package.chapter <= 9)); + } + + #[test] + fn bootstrap_state_base_layer_includes_filesystem_package() { + let steps = + parse_book_steps(SAMPLE_TOC, "https://www.vertexlinux.net/lbi/book.pdf").unwrap(); + let packages = packages_from_steps(&steps); + let layers = packages_by_layer(&packages); + + assert_eq!( + bootstrap_layer_packages_for_state(&layers, BASE_LAYER), + vec![ + "filesystem", + "gettext-tiny", + "iana-etc", + "pigz", + "ca-certificates", + "libffi", + "limine" + ] + ); + } + + #[test] + fn parses_book_steps_including_operational_sections() { + let steps = + parse_book_steps(SAMPLE_TOC, "https://www.vertexlinux.net/lbi/book.pdf").unwrap(); + let operations = steps + .iter() + .filter_map(|step| match step { + BookStep::Operation(operation) => { + Some((operation.section.as_str(), operation.kind)) + } + BookStep::Package(_) => None, + }) + .collect::>(); + + assert_eq!( + operations, + vec![ + ("7.2", BookOperationKind::ResetTargetTreeOwnership), + ("7.3", BookOperationKind::CreateVirtualFilesystemLinkTargets), + ("7.4", BookOperationKind::CopyBuildProfile), + ("7.5", BookOperationKind::EnterChroot), + ("7.6", BookOperationKind::CreateEssentialSystemFiles), + ] + ); + assert!(matches!(steps[0], BookStep::Package(_))); + assert!(matches!(steps[3], BookStep::Operation(_))); + } + + #[test] + fn parse_book_steps_preserves_book_order_for_reordered_sections() { + let steps = parse_book_steps( + r#" + 8.25.1 samurai stage 2 1.3 + 8.26 BSD-Diffutils stage 2 0.99.0 + 8.25.1 samurai stage 2 1.3 + "#, + "https://www.vertexlinux.net/lbi/book.pdf", + ) + .unwrap(); + let packages = packages_from_steps(&steps); + + assert_eq!( + packages + .iter() + .map(|package| package.name.as_str()) + .collect::>(), + vec!["samurai", "bsddiffutils"] + ); + } + + #[test] + fn fresh_book_fetch_url_adds_cache_buster_without_changing_path() { + let fetch_url = + fresh_book_fetch_url("https://www.vertexlinux.net/lbi/book.pdf?existing=1").unwrap(); + let parsed = Url::parse(&fetch_url).unwrap(); + + assert_eq!(parsed.scheme(), "https"); + assert_eq!(parsed.host_str(), Some("www.vertexlinux.net")); + assert_eq!(parsed.path(), "/lbi/book.pdf"); + let pairs = parsed.query_pairs().collect::>(); + assert!( + pairs + .iter() + .any(|(key, value)| key == "existing" && value == "1") + ); + assert!( + pairs + .iter() + .any(|(key, value)| key == BOOK_FETCH_CACHE_BUST_PARAM && !value.is_empty()) + ); + } + + #[test] + fn parses_lbi_b2sum_manifest_by_filename() { + let sums = parse_source_b2sums( + r#" +# Linux by Intent starter source checksums (BLAKE2b-512) +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa musl-1.2.6.tar.gz +BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB *linux-by-intent-patches.zip +"#, + ) + .unwrap(); + + assert_eq!( + sums["musl-1.2.6.tar.gz"], + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ); + assert_eq!( + sums["linux-by-intent-patches.zip"], + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ); + } + + #[test] + fn registers_filesystem_package_for_lbi_layout() { + let tmp = tempfile::tempdir().unwrap(); + let config = config::Config::for_rootfs(tmp.path()); + + register_filesystem_package(tmp.path(), &config).unwrap(); + + let db_path = config.installed_db_path(tmp.path()); + let installed = crate::db::get_installed_packages(&db_path).unwrap(); + assert!(installed.contains(FILESYSTEM_PACKAGE)); + + let files = crate::db::get_package_files(&db_path, FILESYSTEM_PACKAGE).unwrap(); + assert!(files.contains(&"etc".to_string())); + assert!(files.contains(&"system/configuration/passwd".to_string())); + + let groups = crate::db::get_package_groups(&db_path, FILESYSTEM_PACKAGE).unwrap(); + assert_eq!(groups, vec![BASE_LAYER.to_string()]); + } + + #[test] + fn page_recipe_preserves_input_order_and_strips_extract_scaffolding() { + let package = BookPackage { + chapter: 6, + section: "6.7".to_string(), + title: "musl libc pass 2 1.2.5".to_string(), + name: "musl-libc-pass2".to_string(), + version: "1.2.5".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://www.vertexlinux.net/lbi/chapters/chapter6/musl-libc-pass2.html" + .to_string(), + recipe_id: "6-7-musl-libc-pass2".to_string(), + }; + let html = r#" +

    Build musl for the temporary toolchain.

    +

    Source package: musl-1.2.5.tar.gz

    +

    Source package: mimalloc-v3.3.0.tar.gz

    +

    Source URLs: https://musl.libc.org/releases/musl-1.2.5.tar.gz and https://github.com/microsoft/mimalloc/archive/refs/tags/v3.3.0.tar.gz

    +

    Source package: musl-fix.patch

    +

    Extract:

    + 
    cd "$LBI_SOURCES"
+tar -xf musl-1.2.5.tar.gz
+cd musl-1.2.5
    +

    Build:

    + 
    patch -Np1 -i ../musl-fix.patch
+tar -xf ../mimalloc-v3.3.0.tar.gz
+./configure --prefix=/system
+make
    +

    Install:

    + 
    make DESTDIR="$LBI_ROOT" install
+rm -rf musl-1.2.5
    +

    Licenses:

    • MIT
    + "#; + + let recipe = parse_page_recipe(html, &package).unwrap(); + + assert_eq!( + recipe.input_files, + vec![ + "musl-1.2.5.tar.gz", + "mimalloc-v3.3.0.tar.gz", + "musl-fix.patch" + ] + ); + assert_eq!( + recipe.source_urls, + vec![ + "https://musl.libc.org/releases/musl-1.2.5.tar.gz", + "https://github.com/microsoft/mimalloc/archive/refs/tags/v3.3.0.tar.gz" + ] + ); + assert_eq!(recipe.extract_dir.as_deref(), Some("musl-1.2.5")); + assert!( + !recipe + .commands + .iter() + .any(|cmd| cmd.contains("tar -xf musl")) + ); + assert!( + recipe + .commands + .iter() + .any(|cmd| cmd.contains("../musl-fix.patch")) + ); + assert!( + recipe + .commands + .iter() + .any(|cmd| cmd.contains("../mimalloc-v3.3.0.tar.gz")) + ); + assert_eq!(recipe.license, "MIT"); + } + + #[test] + fn page_recipe_ignores_source_note_archives_when_input_assumption_exists() { + let package = BookPackage { + chapter: 9, + section: "9.2".to_string(), + title: "Limine 11.4.1 binary release".to_string(), + name: "limine".to_string(), + version: "11.4.1".to_string(), + layer: BASE_LAYER.to_string(), + page_url: "https://www.vertexlinux.net/lbi/chapters/chapter9/limine.html".to_string(), + recipe_id: "9-2-limine".to_string(), + }; + let html = r#" +

    Install Limine's bootloader payloads.

    +
    +

    Input assumption: limine-5be26a73d7b7.tar.gz is already present in /sources.

    +

    Source URL: https://github.com/Limine-Bootloader/Limine/commit/5be26a73d7b7b4d4477d18be94e1d16e615adf56

    +

    Source note: this is the upstream v11.4.1-binary snapshot. Unlike the full limine-11.4.1.tar.xz source release, it does not need nasm.

    +
    +

    Extract and Enter the Source Tree

    + 
    cd /sources
+tar -xf limine-5be26a73d7b7.tar.gz
+cd limine-5be26a73d7b7
    +

    Build the Limine Utility

    + 
    make $LWI_MAKE_FLAGS CC=clang
    +

    Install Limine

    + 
    install -Dm755 limine /system/binaries/limine
    +

    Licenses:

    • BSD 2-Clause
    + "#; + let manifest = parse_source_manifest( + "git+https://github.com/Limine-Bootloader/Limine.git#5be26a73d7b7b4d4477d18be94e1d16e615adf56 limine-5be26a73d7b7.tar.gz", + ); + + let recipe = parse_page_recipe(html, &package).unwrap(); + let source_url = + source_url_for_input(&recipe.input_files[0], &package, &recipe, &manifest).unwrap(); + + assert_eq!( + recipe.input_files, + vec!["limine-5be26a73d7b7.tar.gz".to_string()] + ); + assert_eq!( + source_url, + "https://github.com/Limine-Bootloader/Limine.git#5be26a73d7b7b4d4477d18be94e1d16e615adf56" + ); + } + + #[test] + fn page_recipe_preserves_book_subdir_after_archive_root() { + let package = BookPackage { + chapter: 5, + section: "5.5".to_string(), + title: "LLVM runtimes (libunwind, libcxxabi, libcxx) 22.1.3".to_string(), + name: "llvm-runtimes".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/llvm-runtimes.html".to_string(), + recipe_id: "5-5-llvm-runtimes".to_string(), + }; + let html = r#" +

    Build runtimes.

    +

    Source package: llvm-project-22.1.3.src.tar.xz

    +

    Source URLs: https://example.invalid/llvm-project-22.1.3.src.tar.xz

    +

    Extract:

    + 
    cd "$LBI_SOURCES"
+tar -xf llvm-project-22.1.3.src.tar.xz
+cd llvm-project-22.1.3.src/runtimes
    +

    Configure:

    + 
    lbi_cmake build-runtimes \
+    -DLIBUNWIND_INSTALL_LIBRARY_DIR=/system/libraries
    + "#; + + let recipe = parse_page_recipe(html, &package).unwrap(); + + assert_eq!( + recipe.extract_dir.as_deref(), + Some("llvm-project-22.1.3.src") + ); + assert!( + recipe + .commands + .iter() + .any(|cmd| cmd.trim() == "cd runtimes") + ); + } + + #[test] + fn rewrites_absolute_system_paths_for_destdir_installs() { + let input = + "PREFIX=/system\ninstall -Dm755 foo /system/binaries/foo\nln -s /system/bin foo"; + let rewritten = rewrite_absolute_system_paths(input); + + assert!(rewritten.contains("PREFIX=/system")); + assert!(rewritten.contains("install -Dm755 foo $DESTDIR/system/binaries/foo")); + assert!(rewritten.contains("ln -s $DESTDIR/system/bin foo")); + } + + #[test] + fn rewrite_absolute_system_paths_keeps_configure_path_options() { + let input = "./configure --prefix=/system \\\n --bindir=/system/binaries \\\n --mandir=/system/documentation/man-pages"; + let rewritten = rewrite_absolute_system_paths(input); + + assert!(rewritten.contains("--prefix=/system")); + assert!(rewritten.contains("--bindir=/system/binaries")); + assert!(rewritten.contains("--mandir=/system/documentation/man-pages")); + assert!(!rewritten.contains("--prefix=$DESTDIR/system")); + assert!(!rewritten.contains("--bindir=$DESTDIR/system/binaries")); + } + + #[test] + fn rewrite_absolute_system_paths_does_not_double_prefix_quoted_destdir() { + let input = "\"$DESTDIR/system/systembinaries/pwconv\" -R /"; + let rewritten = rewrite_absolute_system_paths(input); + + assert_eq!(rewritten, input); + } + + #[test] + fn rewrite_absolute_system_paths_does_not_rewrite_systembinaries_segment() { + let input = "/system/systembinaries/pwconv"; + let rewritten = rewrite_absolute_system_paths(input); + + assert_eq!(rewritten, "$DESTDIR/system/systembinaries/pwconv"); + } + + #[test] + fn generated_oksh_keeps_prefixes_and_uses_resolved_cross_tools() { + let package = BookPackage { + chapter: 6, + section: "6.14".to_string(), + title: "oksh 7.8".to_string(), + name: "oksh".to_string(), + version: "7.8".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/oksh.html".to_string(), + recipe_id: "6-14-oksh".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["oksh-7.8.tar.gz".to_string()], + source_urls: vec!["https://example.invalid/oksh-7.8.tar.gz".to_string()], + extract_dir: Some("oksh-7.8".to_string()), + commands: vec![ + r#"./configure \ + --no-thanks \ + --disable-curses \ + --prefix=/system \ + --bindir=/system/binaries \ + --mandir=/system/documentation/man-pages \ + --cc="$LBI_ROOT/system/tools/bin/$LBI_TARGET-clang" \ + --cflags="--target=$LBI_TARGET --sysroot=$LBI_ROOT $LWI_CFLAGS""# + .to_string(), + r#"make $LWI_MAKE_FLAGS \ + LDFLAGS="--target=$LBI_TARGET --sysroot=$LBI_ROOT $LBI_CUSTOM_LDFLAGS""# + .to_string(), + r#"make install DESTDIR="$LBI_ROOT" +ln -sf oksh "$LBI_ROOT/system/binaries/ksh""# + .to_string(), + ], + dependencies: Vec::new(), + license: "Public domain".to_string(), + description: "oksh shell".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-lbi-linux-musl", "x86_64"); + + assert!(build_script.contains("--prefix=/system")); + assert!(build_script.contains("--bindir=/system/binaries")); + assert!(build_script.contains("--mandir=/system/documentation/man-pages")); + assert!(!build_script.contains("--prefix=$DESTDIR/system")); + assert!(!build_script.contains("--bindir=$DESTDIR/system/binaries")); + assert!(build_script.contains(r#"--cc="$CC""#)); + assert!(build_script.contains("lbi_find_cross_tool ar llvm-ar")); + assert!(build_script.contains("export PATH=\"$LBI_SYSROOT/system/tools/bin:$PATH\"")); + assert!( + !build_script + .contains("$LBI_SYSROOT/system/tools/bin\" \"$LBI_SYSROOT/system/binaries") + ); + assert!(build_script.contains("ln -sf oksh \"$LBI_ROOT/system/binaries/ksh\"")); + } + + #[test] + fn generated_om4_parser_fix_preserves_c_include_headers() { + let package = BookPackage { + chapter: 6, + section: "6.2".to_string(), + title: "om4 6.7".to_string(), + name: "om4".to_string(), + version: "6.7".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/om4.html".to_string(), + recipe_id: "6-2-om4".to_string(), + }; + let html = r#" +

    Build om4.

    +

    Source package: om4-6.7.tar.gz

    +

    Source URLs: https://example.invalid/om4-6.7.tar.gz

    +

    Extract and Enter the Source Tree

    + 
    cd "$LBI_SOURCES"
+tar -xf om4-6.7.tar.gz
+cd om4-6.7
    +

    Apply the Parser Compatibility Fix

    + 
    grep -q '^#include <stdlib.h>$' parser.y || \
+    sed -i '/^#include <stdint.h>$/a #include <stdlib.h>' parser.y
    +

    Build and Install om4

    + 
    make -j1 CC="$LBI_ROOT/system/tools/bin/$LBI_TARGET-clang"
+make CC="$LBI_ROOT/system/tools/bin/$LBI_TARGET-clang" install DESTDIR="$LBI_ROOT"
    +

    Licenses:

    • BSD-3-Clause
    + "#; + + let recipe = parse_page_recipe(html, &package).unwrap(); + let build_script = + generated_build_script(&package, &recipe, "x86_64-unknown-linux-musl", "x86_64"); + + assert!(build_script.contains("grep -q '^#include $' parser.y")); + assert!( + build_script.contains("sed -i '/^#include $/a #include ' parser.y") + ); + assert!(!build_script.contains("grep -q '^#include $' parser.y")); + } + + #[test] + fn rewrite_absolute_system_paths_keeps_cmake_define_values() { + let input = + "cmake -DLIBCXX_INSTALL_LIBRARY_DIR=/system/libraries -DCMAKE_INSTALL_PREFIX=/system"; + let rewritten = rewrite_absolute_system_paths(input); + + assert!(rewritten.contains("-DLIBCXX_INSTALL_LIBRARY_DIR=/system/libraries")); + assert!(rewritten.contains("-DCMAKE_INSTALL_PREFIX=/system")); + assert!(!rewritten.contains("$DESTDIR/system/libraries")); + } + + #[test] + fn chapter7_chroot_compiler_search_paths_stay_runtime_absolute() { + let package = BookPackage { + chapter: 7, + section: "7.7".to_string(), + title: "gettext-tiny 0.3.3".to_string(), + name: "gettext-tiny".to_string(), + version: "0.3.3".to_string(), + layer: BASE_LAYER.to_string(), + page_url: "https://example.invalid/chapter7/gettext-tiny.html".to_string(), + recipe_id: "7-7-gettext-tiny".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["gettext-tiny-0.3.3.tar.xz".to_string()], + source_urls: vec!["https://example.invalid/gettext-tiny-0.3.3.tar.xz".to_string()], + extract_dir: Some("gettext-tiny-0.3.3".to_string()), + commands: vec![ + r#"make $LWI_MAKE_FLAGS \ + LIBINTL=musl \ + CPPFLAGS="-I/system/headers" \ + CFLAGS="-I/system/headers" \ + LDFLAGS="-L/system/libraries" \ + CC="cc -B/system/libraries -B/system/libraries/clang/22/lib/linux""# + .to_string(), + ], + dependencies: Vec::new(), + license: "MIT".to_string(), + description: "gettext-tiny".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-lbi-linux-musl", "x86_64"); + + assert!( + build_script + .contains("CC=\"cc -B/system/libraries -B/system/libraries/clang/22/lib/linux\"") + ); + assert!(build_script.contains("LIBINTL=MUSL")); + assert!(!build_script.contains("LIBINTL=musl")); + assert!(!build_script.contains("-B$DESTDIR/system/libraries")); + } + + #[test] + fn rustc_bootstrap_toml_uses_live_chroot_tool_paths() { + let package = BookPackage { + chapter: 8, + section: "8.40".to_string(), + title: "rustc 1.95.0".to_string(), + name: "rustc".to_string(), + version: "1.95.0".to_string(), + layer: DEVEL_LAYER.to_string(), + page_url: "https://example.invalid/chapter8/rustc.html".to_string(), + recipe_id: "8-40-rustc".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["rustc-1.95.0-src.tar.xz".to_string()], + source_urls: vec![ + "https://static.rust-lang.org/dist/rustc-1.95.0-src.tar.xz".to_string(), + ], + extract_dir: Some("rustc-1.95.0-src".to_string()), + commands: vec![ + r#"cat > bootstrap.toml <> /etc/group")); + assert!(build_script.contains("\"$DESTDIR/system/systembinaries/grpconv\" -R /")); + assert!( + build_script.contains("\"$DESTDIR/system/systembinaries/useradd\" -D -R / --gid 999") + ); + assert!(build_script.contains("\"$DESTDIR/system/binaries/passwd\" -R / -d root")); + assert!(!build_script.contains("passwd\" -R / root")); + } + + #[test] + fn generated_cross_runtime_commands_use_sysroot_toolchain() { + let package = BookPackage { + chapter: 5, + section: "5.5".to_string(), + title: "LLVM runtimes (libunwind, libcxxabi, libcxx) 22.1.3".to_string(), + name: "llvm-runtimes".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/llvm-runtimes.html".to_string(), + recipe_id: "5-5-llvm-runtimes".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["llvm-project-22.1.3.src.tar.xz".to_string()], + source_urls: vec!["https://example.invalid/llvm-project-22.1.3.src.tar.xz".to_string()], + extract_dir: Some("llvm-project-22.1.3.src".to_string()), + commands: vec![ + r#"lbi_cmake build-runtimes \ + -DCMAKE_C_COMPILER="$LBI_ROOT/system/tools/bin/$LBI_TARGET-clang" \ + -DCMAKE_CXX_COMPILER="$LBI_ROOT/system/tools/bin/$LBI_TARGET-clang++" \ + -DCMAKE_SYSROOT="$LBI_ROOT" \ + -DCMAKE_FIND_ROOT_PATH="$LBI_ROOT;$LBI_ROOT/system" \ + -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \ + -DLLVM_ENABLE_RUNTIMES="libunwind;libcxxabi;libcxx""# + .to_string(), + ], + dependencies: Vec::new(), + license: "Apache-2.0".to_string(), + description: "LLVM runtimes".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-unknown-linux-musl", "x86_64"); + + assert!(build_script.contains("-DCMAKE_C_COMPILER=\"$CC\"")); + assert!(build_script.contains("-DCMAKE_CXX_COMPILER=\"$CXX\"")); + assert!(build_script.contains("-DCMAKE_SYSROOT=\"$LBI_SYSROOT\"")); + assert!( + build_script.contains("-DCMAKE_FIND_ROOT_PATH=\"$LBI_SYSROOT;$LBI_SYSROOT/system\"") + ); + assert!(build_script.contains("-DCMAKE_SHARED_LINKER_FLAGS=\"-nostartfiles\"")); + assert!(build_script.contains("-DCMAKE_MODULE_LINKER_FLAGS=\"-nostartfiles\"")); + assert!(!build_script.contains("destdir/llvm-runtimes/system/tools")); + } + + #[test] + fn llvm_clang_pass2_generated_script_uses_ccache() { + let package = BookPackage { + chapter: 5, + section: "5.6".to_string(), + title: "llvm/clang pass 2 22.1.3".to_string(), + name: "llvm-clang-pass2".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/llvm-clang-pass2.html".to_string(), + recipe_id: "5-6-llvm-clang-pass2".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["llvm-project-22.1.3.src.tar.xz".to_string()], + source_urls: vec!["https://example.invalid/llvm-project-22.1.3.src.tar.xz".to_string()], + extract_dir: Some("llvm-project-22.1.3.src".to_string()), + commands: vec![ + r#"cmake -G Ninja "../llvm" \ + -DCMAKE_INSTALL_PREFIX=$LBI_ROOT/system/tools \ + -DLLVM_NATIVE_TOOL_DIR="$LBI_ROOT/system/tools/bin" \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt""# + .to_string(), + "ninja".to_string(), + ], + dependencies: Vec::new(), + license: "Apache-2.0".to_string(), + description: "LLVM pass2".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-unknown-linux-musl", "x86_64"); + + assert!(build_script.contains("-DLLVM_CCACHE_BUILD=ON")); + assert!( + build_script.contains("cmake -G Ninja \"../llvm\" \\\n -DLLVM_CCACHE_BUILD=ON \\") + ); + assert!(build_script.contains("-DLLVM_NATIVE_TOOL_DIR=\"$LBI_SYSROOT/system/tools/bin\"")); + assert!( + build_script + .contains("-DLLVM_CONFIG_PATH=\"$LBI_SYSROOT/system/tools/bin/llvm-config\"") + ); + assert!(build_script.contains("-DLLVM_ENABLE_RUNTIMES=\"compiler-rt\"")); + } + + #[test] + fn llvm_clang_pass2_generated_script_fixes_clang_resource_layout() { + let package = BookPackage { + chapter: 6, + section: "6.22".to_string(), + title: "llvm/clang pass 2 22.1.3".to_string(), + name: "llvm-clang-pass2".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/llvm-clang-pass2.html".to_string(), + recipe_id: "6-22-llvm-clang-pass2".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["llvm-project-22.1.3.src.tar.xz".to_string()], + source_urls: vec!["https://example.invalid/llvm-project-22.1.3.src.tar.xz".to_string()], + extract_dir: Some("llvm-project-22.1.3.src".to_string()), + commands: vec![ + r#"DESTDIR="$LBI_ROOT" cmake --install build-compiler-rt-pass2 + +mkdir -p "$LBI_ROOT/system/lib/clang/22/lib/$LBI_TARGET" + +if [ -f "$LBI_ROOT/system/tools/lib/clang/22/lib/$LBI_TARGET/libclang_rt.builtins.a" ]; then + ln -sf "/system/tools/lib/clang/22/lib/$LBI_TARGET/libclang_rt.builtins.a" \ + "$LBI_ROOT/system/lib/clang/22/lib/$LBI_TARGET/libclang_rt.builtins.a" +fi + +CRTBEGIN_OBJ=$(find "$LBI_ROOT/system/libraries/clang" \ + -type f \( -name 'crtbeginS.o' -o -name 'clang_rt.crtbegin*.o' \) | head -n1) +CRTEND_OBJ=$(find "$LBI_ROOT/system/libraries/clang" \ + -type f \( -name 'crtendS.o' -o -name 'clang_rt.crtend*.o' \) | head -n1) + +CRT_DIR=$(dirname "$CRTBEGIN_OBJ") + +if [ -n "$CRTBEGIN_OBJ" ] && [ -n "$CRTEND_OBJ" ]; then + ln -sf "$(basename "$CRTBEGIN_OBJ")" "$CRT_DIR/crtbeginS.o" + ln -sf "$(basename "$CRTEND_OBJ")" "$CRT_DIR/crtendS.o" + + ln -sf "${CRTBEGIN_OBJ#$LBI_ROOT/system}" "$LBI_ROOT/system/libraries/crtbeginS.o" + ln -sf "${CRTEND_OBJ#$LBI_ROOT/system}" "$LBI_ROOT/system/libraries/crtendS.o" +fi"# + .to_string(), + ], + dependencies: Vec::new(), + license: "Apache-2.0".to_string(), + description: "LLVM pass2".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-lbi-linux-musl", "x86_64"); + + assert!( + build_script.contains("builtins_name=\"libclang_rt.builtins-${compiler_rt_arch}.a\"") + ); + assert!(build_script.contains("\"$LBI_ROOT/system/libraries/clang/22/lib/$LBI_TARGET\"")); + assert!(build_script.contains("ln -sf \"../linux/$builtins_name\"")); + assert!(build_script.contains("\"$LBI_ROOT/system/lib/clang\"")); + assert!(build_script.contains("rm -rf \"$LBI_ROOT/system/lib/clang/22\"")); + assert!(!build_script.contains("\"$LBI_ROOT/system/lib/clang/22/lib/linux\"")); + assert!(!build_script.contains( + "cp -R \"$LBI_ROOT/system/libraries/clang/22/.\" \"$LBI_ROOT/system/lib/clang/22/\"" + )); + assert!(build_script.contains("2>/dev/null || true; } | head -n1")); + assert!(build_script.contains( + "install -m644 \"$CRTBEGIN_OBJ\" \"$LBI_ROOT/system/libraries/crtbeginS.o\"" + )); + assert!( + !build_script.contains( + "$DESTDIR/system/tools/lib/clang/22/lib/$LBI_TARGET/libclang_rt.builtins.a" + ) + ); + assert!( + !build_script + .contains("CRT_DIR=$(dirname \"$CRTBEGIN_OBJ\")\n\nif [ -n \"$CRTBEGIN_OBJ\"") + ); + } + + #[test] + fn llvm_clang_pass2_driver_configs_keep_runtime_paths() { + let package = BookPackage { + chapter: 6, + section: "6.22".to_string(), + title: "llvm/clang pass 2 22.1.3".to_string(), + name: "llvm-clang-pass2".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/llvm-clang-pass2.html".to_string(), + recipe_id: "6-22-llvm-clang-pass2".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["llvm-project-22.1.3.src.tar.xz".to_string()], + source_urls: vec!["https://example.invalid/llvm-project-22.1.3.src.tar.xz".to_string()], + extract_dir: Some("llvm-project-22.1.3.src".to_string()), + commands: vec![ + r#"cat > "$LBI_ROOT/system/configuration/clang/clang++.cfg" </dev/null".to_string(), + ], + dependencies: Vec::new(), + license: "ISC".to_string(), + description: "mandoc".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-unknown-linux-musl", "x86_64"); + + assert!(!build_script.contains("MANPAGER=cat man mandoc >/dev/null")); + assert!(build_script.contains( + "MANPATH=\"$DESTDIR/system/documentation/man-pages\" MANPAGER=cat \"$DESTDIR/system/binaries/man\" mandoc >/dev/null" + )); + assert!(build_script.contains( + "$DESTDIR/system/systembinaries/makewhatis $DESTDIR/system/documentation/man-pages" + )); + } + + #[test] + fn chapter6_file_uses_host_magic_compiler() { + let package = BookPackage { + chapter: 6, + section: "6.9".to_string(), + title: "File 5.47".to_string(), + name: "file".to_string(), + version: "5.47".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/file.html".to_string(), + recipe_id: "6-9-file".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["file-5.47.tar.gz".to_string()], + source_urls: vec!["https://example.invalid/file-5.47.tar.gz".to_string()], + extract_dir: Some("file-5.47".to_string()), + commands: vec![ + r#"CC="$LBI_ROOT/system/tools/bin/$LBI_TARGET-clang" \ +lbi_configure \ + --host="$LBI_TARGET""# + .to_string(), + "make $LWI_MAKE_FLAGS".to_string(), + r#"make install DESTDIR="$LBI_ROOT""#.to_string(), + ], + dependencies: Vec::new(), + license: "BSD-2-Clause".to_string(), + description: "file".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-unknown-linux-musl", "x86_64"); + + assert!(build_script.contains("rm -rf build-host-file")); + assert!(build_script.contains("CC=\"${BUILD_CC:-cc}\" \\")); + assert!(build_script.contains( + "../configure --prefix=\"$PWD/host-tools\" --disable-shared --enable-static --disable-libseccomp" + )); + assert!(build_script.contains("make ${LWI_MAKE_FLAGS:-} -C src file")); + assert!( + build_script + .contains("make $LWI_MAKE_FLAGS FILE_COMPILE=\"$PWD/build-host-file/src/file\"") + ); + assert!(build_script.contains("CC=\"$CC\"")); + assert!(build_script.contains("--host=\"$LBI_TARGET\"")); + } + + #[test] + fn source_manifest_uses_output_names() { + let manifest = parse_source_manifest( + r#" + https://example.invalid/upstream.tar.gz renamed.tar.gz + git+https://example.invalid/project.git#deadbeef project-git.tar.gz + "#, + ); + + assert_eq!(manifest[0].output_name, "renamed.tar.gz"); + assert_eq!(manifest[0].url, "https://example.invalid/upstream.tar.gz"); + assert_eq!(manifest[1].output_name, "project-git.tar.gz"); + assert_eq!( + manifest[1].url, + "https://example.invalid/project.git#deadbeef" + ); + } + + #[test] + fn source_url_matches_manifest_url_filename_for_renamed_generic_archive() { + let package = BookPackage { + chapter: 8, + section: "8.26".to_string(), + title: "BSD-Diffutils stage 2 0.99.0".to_string(), + name: "bsddiffutils".to_string(), + version: "0.99.0".to_string(), + layer: BASE_LAYER.to_string(), + page_url: "https://www.vertexlinux.net/lbi/chapters/chapter8/bsddiffutils-stage2.html" + .to_string(), + recipe_id: "8-26-bsddiffutils".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["bsddiff-34e64c08674b.tar.gz".to_string()], + source_urls: Vec::new(), + extract_dir: None, + commands: vec!["make".to_string()], + dependencies: Vec::new(), + license: "BSD-2-Clause".to_string(), + description: "BSD diff utilities".to_string(), + }; + let manifest = parse_source_manifest( + r#" + https://github.com/other/project/archive/refs/heads/main.zip other-main.zip + git+https://github.com/chimera-linux/bsddiff.git#34e64c08674ba6f96da41075a17be60944e61e33 bsddiff-34e64c08674b.tar.gz + "#, + ); + + let url = source_url_for_input("bsddiff-34e64c08674b.tar.gz", &package, &recipe, &manifest) + .unwrap(); + + assert_eq!( + url, + "https://github.com/chimera-linux/bsddiff.git#34e64c08674ba6f96da41075a17be60944e61e33" + ); + } + + #[test] + fn page_recipe_strips_book_unzip_scaffolding() { + let package = BookPackage { + chapter: 6, + section: "6.8".to_string(), + title: "BSD-Diffutils 0.99.0".to_string(), + name: "bsddiffutils".to_string(), + version: "0.99.0".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/bsddiffutils.html".to_string(), + recipe_id: "6-8-bsddiffutils".to_string(), + }; + let html = r#" +

    BSD diff utilities.

    +

    Input assumption: bsddiff-34e64c08674b.tar.gz

    +

    Extract and Enter the Source Tree

    + 
    cd "$LBI_SOURCES"
+tar -xf bsddiff-34e64c08674b.tar.gz
+cd bsddiff-34e64c08674b
    +

    Build BSD-Diffutils

    + 
    meson compile -C build -j "$(nproc)"
    +

    Licenses:

    • BSD-2-Clause
    + "#; + + let recipe = parse_page_recipe(html, &package).unwrap(); + + assert_eq!(recipe.extract_dir.as_deref(), Some("bsddiff-34e64c08674b")); + assert!( + !recipe + .commands + .iter() + .any(|cmd| cmd.contains("bsddiff-34e64c08674b.tar.gz")) + ); + assert!( + !recipe + .commands + .iter() + .any(|cmd| cmd.trim() == "cd bsddiff-34e64c08674b") + ); + assert!( + recipe + .commands + .iter() + .any(|cmd| cmd.trim() == "meson compile -C build -j \"$(nproc)\"") + ); + } + + #[test] + fn generated_bsddiffutils_supports_meson_helpers_and_compatibility_edits() { + let package = BookPackage { + chapter: 8, + section: "8.26".to_string(), + title: "BSD-Diffutils stage 2 0.99.0".to_string(), + name: "bsddiffutils".to_string(), + version: "0.99.0".to_string(), + layer: BASE_LAYER.to_string(), + page_url: "https://example.invalid/chapter8/bsddiffutils-stage2.html".to_string(), + recipe_id: "8-26-bsddiffutils".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["bsddiff-34e64c08674b.tar.gz".to_string()], + source_urls: vec![ + "git+https://github.com/chimera-linux/bsddiff.git#34e64c08674ba6f96da41075a17be60944e61e33" + .to_string(), + ], + extract_dir: Some("bsddiff-34e64c08674b".to_string()), + commands: vec![ + r#"sed -i \ + "s|'strtonum.c', 'warnc.c', 'xmalloc.c'|'strtonum.c', 'warnc.c', 'xmalloc.c', 'fgetln.c'|" \ + compat/meson.build + +sed -i \ + '/#include /a char *fgetln(FILE *, size_t *);' \ + diff/diff.c + +sed -i \ + '/#include /a char *fgetln(FILE *, size_t *);' \ + diff3/diff3prog.c + +sed -i \ + '/include_directories: \[sysdefs\],/a \ link_with: [libcompat],' \ + diff3/meson.build"# + .to_string(), + "lbi_meson build -Dbuildtype=release".to_string(), + "meson compile -C build -j \"$(nproc)\"".to_string(), + ], + dependencies: vec!["meson".to_string()], + license: "BSD-2-Clause".to_string(), + description: "BSD diff utilities".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-unknown-linux-musl", "x86_64"); + + assert!(build_script.contains("lbi_meson()")); + assert!(build_script.contains("--libexecdir=/system/systembinaries")); + assert!(build_script.contains("'fgetln.c'")); + assert!(build_script.contains("link_with: [libcompat]")); + assert!(build_script.contains("diff/diff.c > diff/diff.c.new")); + assert!(build_script.contains("diff3/diff3prog.c > diff3/diff3prog.c.new")); + assert!(build_script.contains("diff3/meson.build > diff3/meson.build.new")); + assert!(!build_script.contains("/a char *fgetln(FILE *, size_t *);")); + assert!(!build_script.contains("/a \\ link_with: [libcompat],")); + assert!(build_script.contains("lbi_meson build -Dbuildtype=release")); + assert!(build_script.contains("meson compile -C build -j \"${LWI_MAKE_JOBS}\"")); + assert!(!build_script.contains("$(nproc)")); + } + + fn test_recipe(chapter: u8) -> GeneratedRecipe { + let name = format!("pkg{chapter}"); + GeneratedRecipe { + package: BookPackage { + chapter, + section: format!("{chapter}.1"), + title: format!("Package {chapter} 1.0"), + name: name.clone(), + version: "1.0".to_string(), + layer: layer_for_package(chapter, &name), + page_url: format!("https://example.invalid/chapter{chapter}/{name}.html"), + recipe_id: format!("{chapter}-1-{name}"), + }, + spec_path: PathBuf::from(format!("/tmp/{name}.toml")), + progress_path: PathBuf::from(format!("/tmp/{name}.done")), + } + } + + #[cfg(unix)] + #[test] + fn bootstrap_completion_accepts_recorded_symlink_payloads() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let db_path = config.installed_db_path(sysroot); + let mut recipe = test_recipe(8); + recipe.package.name = "shadow".to_string(); + recipe.progress_path = tmp.path().join("shadow.done"); + + fs::create_dir_all(recipe.progress_path.parent().unwrap()).unwrap(); + fs::write(&recipe.progress_path, "package = \"shadow\"\n").unwrap(); + fs::create_dir_all(sysroot.join("system/systembinaries")).unwrap(); + std::os::unix::fs::symlink("vipw", sysroot.join("system/systembinaries/vigr")).unwrap(); + + fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + drop(rusqlite::Connection::open(&db_path).unwrap()); + crate::db::get_all_replaces(&db_path).unwrap(); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute( + "INSERT INTO packages (name, version, revision) VALUES ('shadow', '1.0', 1)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO files (package_id, path) SELECT id, 'system/systembinaries/vigr' FROM packages WHERE name = 'shadow'", + [], + ) + .unwrap(); + + assert!(bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + } + + #[test] + fn bsddiffutils_prepare_hands_off_sbase_cmp_files() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let db_path = config.installed_db_path(sysroot); + fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + fs::create_dir_all(sysroot.join("system/binaries")).unwrap(); + fs::create_dir_all(sysroot.join("system/documentation/man-pages/man1")).unwrap(); + fs::write(sysroot.join("system/binaries/cmp"), "sbase cmp").unwrap(); + fs::write( + sysroot.join("system/documentation/man-pages/man1/cmp.1"), + "sbase cmp man", + ) + .unwrap(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE packages (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE files (package_id INTEGER NOT NULL, path TEXT NOT NULL); + INSERT INTO packages (id, name) VALUES (1, 'sbase'), (2, 'other'); + INSERT INTO files (package_id, path) VALUES + (1, 'system/binaries/cmp'), + (1, 'system/documentation/man-pages/man1/cmp.1'), + (1, 'system/binaries/cat'), + (2, 'system/binaries/keep'); + "#, + ) + .unwrap(); + + let package = BookPackage { + chapter: 6, + section: "6.8".to_string(), + title: "BSD-Diffutils 0.99.0".to_string(), + name: "bsddiffutils".to_string(), + version: "0.99.0".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/bsddiffutils.html".to_string(), + recipe_id: "6-8-bsddiffutils".to_string(), + }; + + prepare_bootstrap_package_install(sysroot, &config, &package).unwrap(); + + assert!(!sysroot.join("system/binaries/cmp").exists()); + assert!( + !sysroot + .join("system/documentation/man-pages/man1/cmp.1") + .exists() + ); + let remaining_sbase_files: Vec = conn + .prepare( + "SELECT f.path FROM files f JOIN packages p ON p.id = f.package_id WHERE p.name = 'sbase' ORDER BY f.path", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(remaining_sbase_files, vec!["system/binaries/cat"]); + } + + #[test] + fn bsdgrep_prepare_hands_off_sbase_grep_files() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let db_path = config.installed_db_path(sysroot); + fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + fs::create_dir_all(sysroot.join("system/binaries")).unwrap(); + fs::create_dir_all(sysroot.join("system/documentation/man-pages/man1")).unwrap(); + fs::write(sysroot.join("system/binaries/grep"), "sbase grep").unwrap(); + fs::write( + sysroot.join("system/documentation/man-pages/man1/grep.1"), + "sbase grep man", + ) + .unwrap(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE packages (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE files (package_id INTEGER NOT NULL, path TEXT NOT NULL); + INSERT INTO packages (id, name) VALUES (1, 'sbase'), (2, 'other'); + INSERT INTO files (package_id, path) VALUES + (1, 'system/binaries/grep'), + (1, 'system/documentation/man-pages/man1/grep.1'), + (1, 'system/binaries/cat'), + (2, 'system/binaries/keep'); + "#, + ) + .unwrap(); + + let package = BookPackage { + chapter: 6, + section: "6.10".to_string(), + title: "bsdgrep master snapshot".to_string(), + name: "bsdgrep".to_string(), + version: "master".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/bsdgrep.html".to_string(), + recipe_id: "6-10-bsdgrep".to_string(), + }; + + prepare_bootstrap_package_install(sysroot, &config, &package).unwrap(); + + assert!(!sysroot.join("system/binaries/grep").exists()); + assert!( + !sysroot + .join("system/documentation/man-pages/man1/grep.1") + .exists() + ); + let remaining_sbase_files: Vec = conn + .prepare( + "SELECT f.path FROM files f JOIN packages p ON p.id = f.package_id WHERE p.name = 'sbase' ORDER BY f.path", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(remaining_sbase_files, vec!["system/binaries/cat"]); + } + + #[test] + fn bootstrap_package_is_complete_detects_missing_payload_files() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let db_path = config.installed_db_path(sysroot); + fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + + let recipe = GeneratedRecipe { + package: BookPackage { + chapter: 6, + section: "6.17".to_string(), + title: "GNU Make 4.4.1".to_string(), + name: "make".to_string(), + version: "4.4.1".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter6/make.html".to_string(), + recipe_id: "6-17-make".to_string(), + }, + spec_path: tmp.path().join("make.toml"), + progress_path: tmp.path().join("make.done"), + }; + fs::write(&recipe.progress_path, "done").unwrap(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE packages (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE files (package_id INTEGER NOT NULL, path TEXT NOT NULL); + INSERT INTO packages (id, name) VALUES (1, 'make'); + INSERT INTO files (package_id, path) VALUES + (1, 'system/binaries/make'), + (1, 'system/documentation/info/dir'); + "#, + ) + .unwrap(); + + assert!(!bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + + fs::create_dir_all(sysroot.join("system/binaries")).unwrap(); + fs::write(sysroot.join("system/binaries/make"), "binary").unwrap(); + assert!(bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + } + + #[test] + fn bootstrap_package_is_complete_requires_bmake_payload() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let db_path = config.installed_db_path(sysroot); + fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + + let recipe = GeneratedRecipe { + package: BookPackage { + chapter: 8, + section: "8.20".to_string(), + title: "bmake 20260406".to_string(), + name: "bmake".to_string(), + version: "20260406".to_string(), + layer: DEVEL_LAYER.to_string(), + page_url: "https://example.invalid/chapter8/bmake.html".to_string(), + recipe_id: "8-20-bmake".to_string(), + }, + spec_path: tmp.path().join("bmake.toml"), + progress_path: tmp.path().join("bmake.done"), + }; + fs::write(&recipe.progress_path, "recipe_revision = 2\n").unwrap(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE packages (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE files (package_id INTEGER NOT NULL, path TEXT NOT NULL); + INSERT INTO packages (id, name) VALUES (1, 'bmake'); + INSERT INTO files (package_id, path) VALUES + (1, 'system/share/licenses/bmake/COPYING'); + "#, + ) + .unwrap(); + fs::create_dir_all(sysroot.join("system/share/licenses/bmake")).unwrap(); + fs::write( + sysroot.join("system/share/licenses/bmake/COPYING"), + "license", + ) + .unwrap(); + + assert!(!bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + + fs::create_dir_all(sysroot.join("system/binaries")).unwrap(); + fs::create_dir_all(sysroot.join("system/share/mk")).unwrap(); + fs::write(sysroot.join("system/binaries/bmake"), "binary").unwrap(); + fs::write(sysroot.join("system/share/mk/sys.mk"), "mk").unwrap(); + + assert!(bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + } + + #[test] + fn bootstrap_package_is_complete_requires_llvm_pass1_native_tools() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let db_path = config.installed_db_path(sysroot); + fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + + let recipe = GeneratedRecipe { + package: BookPackage { + chapter: 5, + section: "5.3".to_string(), + title: "llvm/clang pass 1 22.1.3".to_string(), + name: "llvm-clang-pass1".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/llvm-clang-pass1.html".to_string(), + recipe_id: "5-3-llvm-clang-pass1".to_string(), + }, + spec_path: tmp.path().join("llvm-clang-pass1.toml"), + progress_path: tmp.path().join("llvm-clang-pass1.done"), + }; + fs::write(&recipe.progress_path, "done\n").unwrap(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE packages (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE files (package_id INTEGER NOT NULL, path TEXT NOT NULL); + INSERT INTO packages (id, name) VALUES (1, 'llvm-clang-pass1'); + INSERT INTO files (package_id, path) VALUES + (1, 'system/tools/bin/clang'); + "#, + ) + .unwrap(); + fs::create_dir_all(sysroot.join("system/tools/bin")).unwrap(); + fs::write(sysroot.join("system/tools/bin/clang"), "binary").unwrap(); + + assert!(!bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + + for tool in ["llvm-config", "llvm-tblgen", "clang-tblgen"] { + fs::write(sysroot.join("system/tools/bin").join(tool), "binary").unwrap(); + } + + assert!(bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + } + + #[test] + fn bootstrap_package_is_complete_reinstalls_stale_bmake_recipe_revision() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let db_path = config.installed_db_path(sysroot); + fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + + let recipe = GeneratedRecipe { + package: BookPackage { + chapter: 8, + section: "8.20".to_string(), + title: "bmake 20260406".to_string(), + name: "bmake".to_string(), + version: "20260406".to_string(), + layer: DEVEL_LAYER.to_string(), + page_url: "https://example.invalid/chapter8/bmake.html".to_string(), + recipe_id: "8-20-bmake".to_string(), + }, + spec_path: tmp.path().join("bmake.toml"), + progress_path: tmp.path().join("bmake.done"), + }; + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE packages (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE files (package_id INTEGER NOT NULL, path TEXT NOT NULL); + INSERT INTO packages (id, name) VALUES (1, 'bmake'); + INSERT INTO files (package_id, path) VALUES + (1, 'system/binaries/bmake'), + (1, 'system/share/mk/sys.mk'); + "#, + ) + .unwrap(); + fs::create_dir_all(sysroot.join("system/binaries")).unwrap(); + fs::create_dir_all(sysroot.join("system/share/mk")).unwrap(); + fs::write(sysroot.join("system/binaries/bmake"), "binary").unwrap(); + fs::write(sysroot.join("system/share/mk/sys.mk"), "mk").unwrap(); + + fs::write(&recipe.progress_path, "done\n").unwrap(); + assert!(!bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + + fs::write(&recipe.progress_path, "recipe_revision = 2\n").unwrap(); + assert!(bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + } + + #[test] + fn bootstrap_package_is_complete_treats_replaced_package_as_done() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let db_path = config.installed_db_path(sysroot); + fs::create_dir_all(db_path.parent().unwrap()).unwrap(); + + let recipe = GeneratedRecipe { + package: BookPackage { + chapter: 5, + section: "5.2".to_string(), + title: "musl libc headers 1.2.6".to_string(), + name: "musl-libc-headers".to_string(), + version: "1.2.6".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/musl-libc-headers.html".to_string(), + recipe_id: "5-2-musl-libc-headers".to_string(), + }, + spec_path: tmp.path().join("musl-libc-headers.toml"), + progress_path: tmp.path().join("musl-libc-headers.done"), + }; + fs::write(&recipe.progress_path, "done").unwrap(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + r#" + CREATE TABLE packages (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE files (package_id INTEGER NOT NULL, path TEXT NOT NULL); + CREATE TABLE replaces ( + id INTEGER PRIMARY KEY, + package_id INTEGER NOT NULL, + replaces_name TEXT NOT NULL, + UNIQUE(package_id, replaces_name) + ); + INSERT INTO packages (id, name) VALUES (1, 'musl-libc-pass2'); + INSERT INTO replaces (package_id, replaces_name) VALUES + (1, 'musl-libc-headers'); + "#, + ) + .unwrap(); + + assert!(bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + } + + #[test] + fn bootstrap_package_is_complete_treats_retired_package_as_done() { + let tmp = tempfile::tempdir().unwrap(); + let sysroot = tmp.path(); + let config = config::Config::for_rootfs(sysroot); + let recipe = GeneratedRecipe { + package: BookPackage { + chapter: 5, + section: "5.3".to_string(), + title: "llvm/clang pass 1 22.1.3".to_string(), + name: "llvm-clang-pass1".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/llvm-clang-pass1.html".to_string(), + recipe_id: "5-3-llvm-clang-pass1".to_string(), + }, + spec_path: tmp.path().join("llvm-clang-pass1.toml"), + progress_path: tmp.path().join("llvm-clang-pass1.done"), + }; + fs::write(&recipe.progress_path, "done").unwrap(); + let retired_path = retired_package_progress_path(&config, &recipe.package.name); + fs::create_dir_all(retired_path.parent().unwrap()).unwrap(); + fs::write(retired_path, "retired = true\n").unwrap(); + + assert!(bootstrap_package_is_complete(sysroot, &config, &recipe).unwrap()); + } + + #[test] + fn chapter7_boundary_is_detected_before_chapter8() { + let chapter7 = test_recipe(7); + let next_chapter7 = test_recipe(7); + let chapter8 = test_recipe(8); + + assert!(!step_completes_chapter( + &BookStep::Package(chapter7.package.clone()), + Some(&BookStep::Package(next_chapter7.package)), + 7, + )); + assert!(step_completes_chapter( + &BookStep::Package(chapter7.package), + Some(&BookStep::Package(chapter8.package)), + 7, + )); + } + + #[test] + fn temp_layer_filter_removes_chapter7_retired_packages() { + let filtered = filter_retired_layer_packages( + TEMP_LAYER, + vec![ + "llvm-clang-pass1".to_string(), + "make".to_string(), + "musl-libc-headers".to_string(), + ], + ); + + assert_eq!( + filtered, + vec!["make".to_string(), "musl-libc-headers".to_string()] + ); + assert_eq!( + filter_retired_layer_packages(BASE_LAYER, vec!["llvm-clang-pass1".to_string()]), + vec!["llvm-clang-pass1".to_string()] + ); + } + + #[test] + fn chapter6_install_invocation_uses_cross_prefix_and_bootstrap_path() { + let recipe = test_recipe(6); + let invocation = bootstrap_install_invocation( + Path::new("/target"), + &recipe, + BootstrapBuildMode::Cross, + "x86_64-unknown-linux-musl", + Path::new("/bin/depot"), + ) + .unwrap(); + + assert!(invocation.args.windows(2).any(|pair| { + pair[0] == OsStr::new("--cross-prefix") + && pair[1] == OsStr::new("x86_64-unknown-linux-musl") + })); + let path = invocation + .env + .iter() + .find(|(key, _)| key == OsStr::new("PATH")) + .map(|(_, value)| value.to_string_lossy().into_owned()) + .unwrap(); + assert!(path.starts_with("/target/system/tools/bin:")); + assert!( + !path + .split(':') + .any(|entry| entry == "/target/system/binaries") + ); + assert!( + invocation + .env + .iter() + .any(|(key, _)| key == OsStr::new("LWI_MAKE_FLAGS")) + ); + assert!( + invocation + .env + .iter() + .any(|(key, _)| key == OsStr::new("LWI_MAKE_JOBS")) + ); + assert!(invocation.env.iter().any(|(key, value)| { + key == OsStr::new(DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS) && value == OsStr::new("1") + })); + assert!(invocation.env.iter().any(|(key, value)| { + key == OsStr::new("DEPOT_LBI_SYSROOT") && value == OsStr::new("/target") + })); + } + + #[test] + fn chapter5_post_pass1_install_invocation_uses_cross_mode() { + let mut recipe = test_recipe(5); + recipe.package.section = "5.4".to_string(); + recipe.package.name = "musl-libc-pass2".to_string(); + recipe.package.recipe_id = "5-4-musl-libc-pass2".to_string(); + + let mode = build_mode_for_package(&recipe.package); + let invocation = bootstrap_install_invocation( + Path::new("/target"), + &recipe, + mode, + "x86_64-unknown-linux-musl", + Path::new("/bin/depot"), + ) + .unwrap(); + + assert_eq!(mode, BootstrapBuildMode::Cross); + assert!(invocation.args.windows(2).any(|pair| { + pair[0] == OsStr::new("--cross-prefix") + && pair[1] == OsStr::new("x86_64-unknown-linux-musl") + })); + let path = invocation + .env + .iter() + .find(|(key, _)| key == OsStr::new("PATH")) + .map(|(_, value)| value.to_string_lossy().into_owned()) + .unwrap(); + assert!(path.starts_with("/target/system/tools/bin:")); + assert!( + !path + .split(':') + .any(|entry| entry == "/target/system/binaries") + ); + } + + #[test] + fn chapter7_and_8_install_invocations_use_chroot_mode() { + let recipe = test_recipe(8); + let invocation = bootstrap_install_invocation( + Path::new("/target"), + &recipe, + BootstrapBuildMode::Chroot, + "x86_64-unknown-linux-musl", + Path::new("/bin/depot"), + ) + .unwrap(); + + assert!( + !invocation + .args + .iter() + .any(|arg| arg == OsStr::new("--cross-prefix")) + ); + assert!(invocation.env.iter().any(|(key, value)| { + key == OsStr::new("DEPOT_LBI_CHROOT") && value == OsStr::new("1") + })); + assert!(invocation.env.iter().any(|(key, value)| { + key == OsStr::new("DEPOT_LBI_CHROOT_ROOT") && value == OsStr::new("/target") + })); + } + + #[test] + fn bootstrap_chroot_tool_env_prefers_target_tools() { + let env = bootstrap_chroot_tool_env(); + assert!( + env.iter() + .any(|(key, value)| *key == "AR" && *value == "ar") + ); + assert!( + env.iter() + .any(|(key, value)| *key == "RANLIB" && *value == "ranlib") + ); + assert!(env.iter().any(|(key, value)| { + *key == "PATH" && value.split(':').any(|entry| entry == "/system/binaries") + })); + assert!(env.iter().any(|(key, value)| { + *key == "PATH" + && value + .split(':') + .any(|entry| entry == BOOTSTRAP_CHROOT_SHIM_DIR) + })); + } + + #[test] + fn bootstrap_chroot_mount_guard_cleans_created_file_targets() { + let tmp = tempfile::tempdir().unwrap(); + let target = tmp.path().join("etc/resolv.conf"); + + { + let mut guard = BootstrapChrootMountGuard::default(); + guard.prepare_file_mount_target(&target).unwrap(); + assert!(target.is_file()); + } + + assert!(!target.exists()); + } + + #[test] + fn bootstrap_makeflags_prefers_explicit_env_override() { + let mut env = TestEnv::new(); + env.set_var("LWI_MAKE_FLAGS", "-j37 --output-sync=target"); + assert_eq!( + bootstrap_parallel_makeflags(), + "-j37 --output-sync=target".to_string() + ); + } + + #[test] + fn bootstrap_make_jobs_prefers_explicit_env_override() { + let mut env = TestEnv::new(); + env.set_var("LWI_MAKE_JOBS", "37"); + assert_eq!(bootstrap_parallel_make_jobs(), "37"); + } + + #[test] + fn generated_recipe_passthrough_exports_lwi_parallel_env() { + let tmp = tempfile::tempdir().unwrap(); + let spec_path = tmp.path().join("pkg.toml"); + let build_path = tmp.path().join("build.sh"); + let package = test_recipe(7).package; + let recipe = PageRecipe { + input_files: vec!["pkg-1.0.tar.gz".to_string()], + source_urls: vec!["https://example.invalid/pkg-1.0.tar.gz".to_string()], + extract_dir: None, + commands: vec!["make".to_string()], + dependencies: Vec::new(), + license: "MIT".to_string(), + description: "Example package".to_string(), + }; + + write_generated_recipe( + &spec_path, + &build_path, + &package, + &recipe, + &SourceManifest::default(), + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + + let spec = fs::read_to_string(spec_path).unwrap(); + assert!(spec.contains("\"LWI_MAKE_FLAGS\"")); + assert!(spec.contains("\"LWI_MAKE_JOBS\"")); + assert!(spec.contains("\"DEPOT_LBI_SYSROOT\"")); + } + + #[test] + fn generated_recipe_preserves_static_archives_only_for_toolchain_packages() { + let tmp = tempfile::tempdir().unwrap(); + let recipe = PageRecipe { + input_files: vec!["pkg-1.0.tar.gz".to_string()], + source_urls: vec!["https://example.invalid/pkg-1.0.tar.gz".to_string()], + extract_dir: None, + commands: vec!["make".to_string()], + dependencies: Vec::new(), + license: "MIT".to_string(), + description: "Example package".to_string(), + }; + + let names = [ + ("llvm-clang-pass2", true), + ("llvm", true), + ("musl-libc-pass2", true), + ("musl", true), + ("rustc", true), + ("bmake", false), + ]; + + for (idx, (name, should_preserve)) in names.into_iter().enumerate() { + let mut package = test_recipe(8).package; + package.name = name.to_string(); + package.title = format!("{name} 1.0"); + package.recipe_id = format!("8-{idx}-{name}"); + let spec_path = tmp.path().join(format!("{name}.toml")); + write_generated_recipe( + &spec_path, + &tmp.path().join(format!("{name}-build.sh")), + &package, + &recipe, + &SourceManifest::default(), + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + + let spec = fs::read_to_string(spec_path).unwrap(); + assert_eq!( + spec.contains("no_delete_static = true"), + should_preserve, + "{name} static archive cleanup policy should match" + ); + } + } + + #[test] + fn generated_recipe_uses_lbi_blake2_source_checksums() { + let tmp = tempfile::tempdir().unwrap(); + let spec_path = tmp.path().join("pkg.toml"); + let build_path = tmp.path().join("build.sh"); + let package = test_recipe(7).package; + let recipe = PageRecipe { + input_files: vec!["pkg-1.0.tar.gz".to_string(), "pkg-fix.patch".to_string()], + source_urls: vec![ + "https://example.invalid/pkg-1.0.tar.gz".to_string(), + "https://example.invalid/patches/pkg-fix.patch".to_string(), + ], + extract_dir: None, + commands: vec!["make".to_string()], + dependencies: Vec::new(), + license: "MIT".to_string(), + description: "Example package".to_string(), + }; + let manifest = SourceManifest { + entries: Vec::new(), + blake2b_512: BTreeMap::from([ + ( + "pkg-1.0.tar.gz".to_string(), + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .to_string(), + ), + ( + "pkg-fix.patch".to_string(), + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .to_string(), + ), + ]), + }; + + write_generated_recipe( + &spec_path, + &build_path, + &package, + &recipe, + &manifest, + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + + let spec = fs::read_to_string(spec_path).unwrap(); + assert!(spec.contains("sha256 = \"b2sum:aaaaaaaa")); + assert!(spec.contains("sha256 = \"b2sum:bbbbbbbb")); + assert!(!spec.contains("sha256 = \"skip\"")); + } + + #[test] + fn generated_specs_record_bootstrap_stage_replacements() { + let tmp = tempfile::tempdir().unwrap(); + let recipe = PageRecipe { + input_files: vec!["pkg-1.0.tar.gz".to_string()], + source_urls: vec!["https://example.invalid/pkg-1.0.tar.gz".to_string()], + extract_dir: None, + commands: vec!["make".to_string()], + dependencies: Vec::new(), + license: "MIT".to_string(), + description: "Example package".to_string(), + }; + + let mut musl_pass2 = test_recipe(5).package; + musl_pass2.name = "musl-libc-pass2".to_string(); + musl_pass2.title = "musl libc pass 2 1.2.6".to_string(); + let musl_pass2_spec = tmp.path().join("musl-pass2.toml"); + write_generated_recipe( + &musl_pass2_spec, + &tmp.path().join("musl-pass2-build.sh"), + &musl_pass2, + &recipe, + &SourceManifest::default(), + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + let musl_pass2_spec = fs::read_to_string(musl_pass2_spec).unwrap(); + assert!(musl_pass2_spec.contains("replaces = [\"musl-libc-headers\"]")); + + let mut musl_final = test_recipe(8).package; + musl_final.name = "musl".to_string(); + musl_final.title = "musl libc final pass 1.2.6".to_string(); + let musl_final_spec = tmp.path().join("musl-final.toml"); + write_generated_recipe( + &musl_final_spec, + &tmp.path().join("musl-final-build.sh"), + &musl_final, + &recipe, + &SourceManifest::default(), + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + let musl_final_spec = fs::read_to_string(musl_final_spec).unwrap(); + assert!( + musl_final_spec + .contains("replaces = [\"musl-libc-pass2\", \"musl-libc-headers\", \"musl-libc\"]") + ); + + let mut llvm_pass2 = test_recipe(6).package; + llvm_pass2.name = "llvm-clang-pass2".to_string(); + llvm_pass2.title = "llvm/clang pass 2 22.1.3".to_string(); + let llvm_pass2_spec = tmp.path().join("llvm-pass2.toml"); + write_generated_recipe( + &llvm_pass2_spec, + &tmp.path().join("llvm-pass2-build.sh"), + &llvm_pass2, + &recipe, + &SourceManifest::default(), + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + let llvm_pass2_spec = fs::read_to_string(llvm_pass2_spec).unwrap(); + assert!(llvm_pass2_spec.contains("replaces = [\"llvm-runtimes\"]")); + assert!(!llvm_pass2_spec.contains("llvm-clang-pass1")); + + let mut llvm_final = test_recipe(8).package; + llvm_final.name = "llvm".to_string(); + llvm_final.title = "llvm final 22.1.3".to_string(); + let llvm_final_spec = tmp.path().join("llvm-final.toml"); + write_generated_recipe( + &llvm_final_spec, + &tmp.path().join("llvm-final-build.sh"), + &llvm_final, + &recipe, + &SourceManifest::default(), + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + let llvm_final_spec = fs::read_to_string(llvm_final_spec).unwrap(); + assert!(llvm_final_spec.contains( + "replaces = [\"llvm-clang-pass2\", \"llvm-clang-pass1\", \"llvm-runtimes\"]" + )); + } + + #[test] + fn generated_python_bootstrap_specs_use_custom_chroot_script() { + let tmp = tempfile::tempdir().unwrap(); + let recipe = PageRecipe { + input_files: vec!["flit_core-3.12.0.tar.gz".to_string()], + source_urls: vec!["https://example.invalid/flit_core-3.12.0.tar.gz".to_string()], + extract_dir: Some("flit_core-3.12.0".to_string()), + commands: vec![ + "pip3 wheel -w dist --no-cache-dir --no-build-isolation --no-deps $PWD" + .to_string(), + "pip3 install --root=\"${DESTDIR:-/}\" --prefix=/system --no-index --find-links dist flit_core" + .to_string(), + ], + dependencies: vec!["python".to_string(), "pip".to_string()], + license: "BSD-3-Clause".to_string(), + description: "Build and install flit_core.".to_string(), + }; + + let mut package = test_recipe(8).package; + package.name = "python-flit-core".to_string(); + package.title = "Python-Flit-Core 3.12.0".to_string(); + package.version = "3.12.0".to_string(); + package.recipe_id = "8-19-python-flit-core".to_string(); + let spec_path = tmp.path().join("python-flit-core.toml"); + let build_path = tmp.path().join("build.sh"); + write_generated_recipe( + &spec_path, + &build_path, + &package, + &recipe, + &SourceManifest::default(), + "x86_64-lbi-linux-musl", + "x86_64", + ) + .unwrap(); + + let spec = fs::read_to_string(spec_path).unwrap(); + let build_script = fs::read_to_string(build_path).unwrap(); + assert!(spec.contains("type = \"custom\"")); + assert!(!spec.contains("type = \"python\"")); + assert!(spec.contains("DEPOT_LBI_CHROOT")); + assert!(build_script.contains("internal bootstrap-chroot")); + assert!(build_script.contains("pip3 wheel -w dist")); + } + + #[test] + fn llvm_clang_pass1_generated_script_preserves_compiler_rt_builtins() { + let tmp = tempfile::tempdir().unwrap(); + let spec_path = tmp.path().join("llvm-clang-pass1.toml"); + let build_path = tmp.path().join("build.sh"); + let package = BookPackage { + chapter: 5, + section: "5.3".to_string(), + title: "llvm/clang pass 1 22.1.3".to_string(), + name: "llvm-clang-pass1".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/llvm-clang-pass1.html".to_string(), + recipe_id: "5-3-llvm-clang-pass1".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["llvm-project-22.1.3.src.tar.xz".to_string()], + source_urls: vec!["https://example.invalid/llvm-project-22.1.3.src.tar.xz".to_string()], + extract_dir: Some("llvm-project-22.1.3.src".to_string()), + commands: vec![ + "mkdir -p build-llvm\ncd build-llvm".to_string(), + r#"cmake -G Ninja "../llvm" \ + -DCMAKE_INSTALL_PREFIX=$LBI_ROOT/system/tools \ + -DLLVM_ENABLE_RUNTIMES="compiler-rt" \ + -DCOMPILER_RT_BUILD_BUILTINS=ON \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ + -DCOMPILER_RT_BUILD_XRAY=OFF \ + -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ + -DCOMPILER_RT_BUILD_PROFILE=OFF \ + -DCLANG_DEFAULT_RTLIB=compiler-rt \ + -DDEFAULT_SYSROOT=$LBI_ROOT"# + .to_string(), + ], + dependencies: Vec::new(), + license: "Apache-2.0".to_string(), + description: "LLVM pass1".to_string(), + }; + + write_generated_recipe( + &spec_path, + &build_path, + &package, + &recipe, + &SourceManifest::default(), + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + + let build_script = fs::read_to_string(build_path).unwrap(); + assert!(build_script.contains("export LBI_SYSROOT=\"${DEPOT_LBI_SYSROOT:-$LBI_ROOT}\"")); + assert!(build_script.contains("export LWI_CXXFLAGS=\"${LWI_CXXFLAGS:-$LWI_CFLAGS}\"")); + assert!(build_script.contains("rm -rf build-llvm")); + assert!(build_script.contains("-DLLVM_CCACHE_BUILD=ON")); + assert!( + build_script.contains("cmake -G Ninja \"../llvm\" \\\n -DLLVM_CCACHE_BUILD=ON \\") + ); + assert!(build_script.contains("LLVM_ENABLE_RUNTIMES=\"compiler-rt\"")); + assert!(build_script.contains("COMPILER_RT_BUILD_BUILTINS=ON")); + assert!(build_script.contains("COMPILER_RT_DEFAULT_TARGET_ONLY=ON")); + assert!( + build_script.contains( + "BUILTINS_CMAKE_ARGS=\"-DCMAKE_C_FLAGS=--sysroot=$LBI_SYSROOT;-DCMAKE_ASM_FLAGS=--sysroot=$LBI_SYSROOT\"" + ) + ); + assert!(build_script.contains("COMPILER_RT_BUILD_SANITIZERS=OFF")); + assert!(build_script.contains("COMPILER_RT_BUILD_XRAY=OFF")); + assert!(build_script.contains("COMPILER_RT_BUILD_LIBFUZZER=OFF")); + assert!(build_script.contains("COMPILER_RT_BUILD_PROFILE=OFF")); + assert!(build_script.contains("COMPILER_RT_BUILD_CRT=OFF")); + assert!(build_script.contains("COMPILER_RT_BUILD_MEMPROF=OFF")); + assert!(build_script.contains("COMPILER_RT_BUILD_ORC=OFF")); + assert!(build_script.contains("COMPILER_RT_BUILD_CTX_PROFILE=OFF")); + assert!(build_script.contains("COMPILER_RT_INCLUDE_TESTS=OFF")); + assert!(build_script.contains("CLANG_DEFAULT_RTLIB=compiler-rt")); + assert!(build_script.contains("-DDEFAULT_SYSROOT=$LBI_SYSROOT")); + assert!(build_script.contains("unset DESTDIR")); + assert!(build_script.contains("-DCMAKE_INSTALL_PREFIX=$LBI_ROOT/system/tools")); + assert!( + !build_script + .contains("export CC=\"${CC:-$LBI_SYSROOT/system/tools/bin/$LBI_TARGET-clang}\"") + ); + } + + #[test] + fn chapter5_post_pass1_scripts_default_to_cross_toolchain() { + let package = BookPackage { + chapter: 5, + section: "5.4".to_string(), + title: "musl libc pass 2 1.2.6".to_string(), + name: "musl-libc-pass2".to_string(), + version: "1.2.6".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/musl-libc-pass2.html".to_string(), + recipe_id: "5-4-musl-libc-pass2".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["musl-1.2.6.tar.gz".to_string()], + source_urls: vec!["https://example.invalid/musl-1.2.6.tar.gz".to_string()], + extract_dir: Some("musl-1.2.6".to_string()), + commands: vec![ + "lbi_configure --target=\"$LBI_TARGET\" --with-malloc=mimalloc\nmake $LWI_MAKE_FLAGS".to_string(), + "ln -snf ./libc.so \\\n \"$LBI_ROOT/system/libraries/ld-musl-${LBI_ARCH}.so.1\"\n\nls -lh \"$LBI_ROOT/usr/lib/ld-musl-${LBI_ARCH}.so.1\"".to_string(), + ], + dependencies: Vec::new(), + license: "MIT".to_string(), + description: "musl pass2".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-unknown-linux-musl", "x86_64"); + + assert_eq!(build_mode_for_package(&package), BootstrapBuildMode::Cross); + assert!(build_script.contains("export CC=\"${CC:-$(lbi_find_cross_tool clang)}\"")); + assert!(build_script.contains("export CXX=\"${CXX:-$(lbi_find_cross_tool clang++)}\"")); + assert!(build_script.contains("libclang_rt.builtins-${compiler_rt_arch}.a")); + assert!(build_script.contains("export LIBCC")); + assert!(build_script.contains("make $LWI_MAKE_FLAGS")); + assert!(!build_script.contains("make ${LWI_MAKE_FLAGS:-} $LWI_MAKE_FLAGS")); + assert!( + build_script.contains("ls -lh \"$LBI_ROOT/system/libraries/ld-musl-${LBI_ARCH}.so.1\"") + ); + assert!(!build_script.contains("$LBI_ROOT/usr/lib/ld-musl-${LBI_ARCH}.so.1")); + assert!(!build_script.contains("ln -snf")); + assert!( + build_script.contains("rm -f \"$LBI_ROOT/system/libraries/ld-musl-${LBI_ARCH}.so.1\"") + ); + assert!(build_script.contains("ln -sf ./libc.so")); + assert!(build_script.contains("rm -f \"$LBI_ROOT/lib/ld-musl-${LBI_ARCH}.so.1\"")); + assert!(build_script.contains("rmdir \"$LBI_ROOT/lib\" 2>/dev/null || true")); + assert!(build_script.contains("export LWI_MAKE_JOBS=\"$jobs\"")); + assert!(build_script.contains("export LWI_MAKE_FLAGS=\"-j${LWI_MAKE_JOBS}\"")); + + let final_package = BookPackage { + chapter: 8, + section: "8.6".to_string(), + title: "musl libc final pass 1.2.6".to_string(), + name: "musl".to_string(), + version: "1.2.6".to_string(), + layer: BASE_LAYER.to_string(), + page_url: "https://example.invalid/chapter8/musl-libc-final-pass.html".to_string(), + recipe_id: "8-6-musl".to_string(), + }; + let final_script = generated_build_script( + &final_package, + &recipe, + "x86_64-unknown-linux-musl", + "x86_64", + ); + assert!(!final_script.contains("ln -snf")); + assert!(final_script.contains("ln -sf ./libc.so")); + } + + #[test] + fn lbi_helpers_default_to_target_tuple() { + let package = BookPackage { + chapter: 8, + section: "8.99".to_string(), + title: "Example 1.0".to_string(), + name: "example".to_string(), + version: "1.0".to_string(), + layer: BASE_LAYER.to_string(), + page_url: "https://example.invalid/chapter8/example.html".to_string(), + recipe_id: "8-99-example".to_string(), + }; + let recipe = PageRecipe { + input_files: vec!["example-1.0.tar.gz".to_string()], + source_urls: vec!["https://example.invalid/example-1.0.tar.gz".to_string()], + extract_dir: Some("example-1.0".to_string()), + commands: vec!["lbi_configure\nmake $LWI_MAKE_FLAGS".to_string()], + dependencies: Vec::new(), + license: "MIT".to_string(), + description: "Example".to_string(), + }; + + let build_script = + generated_build_script(&package, &recipe, "x86_64-unknown-linux-musl", "x86_64"); + + assert!(build_script.contains(" --target=\"$LBI_TARGET\" \\\n")); + assert!(build_script.contains(" --host=\"$LBI_TARGET\" \\\n")); + assert!(build_script.contains("-DCMAKE_C_COMPILER_TARGET=\"$LBI_TARGET\"")); + assert!(build_script.contains("-DCMAKE_CXX_COMPILER_TARGET=\"$LBI_TARGET\"")); + assert!( + build_script + .contains("meson setup \"$build_dir\" --cross-file \"$lbi_meson_cross_file\"") + ); + assert!( + build_script.contains("c_args = ['--target=$LBI_TARGET', '--sysroot=$LBI_SYSROOT']") + ); + assert!( + build_script.contains("cpp_args = ['--target=$LBI_TARGET', '--sysroot=$LBI_SYSROOT']") + ); + } + + #[test] + fn rewrite_make_flags_does_not_duplicate_existing_parallel_flags() { + assert_eq!( + rewrite_make_flags("make $LWI_MAKE_FLAGS\nmake install"), + "make $LWI_MAKE_FLAGS\nmake ${LWI_MAKE_FLAGS:-} install" + ); + assert_eq!( + rewrite_make_flags("make ${MAKEFLAGS:-} all"), + "make ${MAKEFLAGS:-} all" + ); + } + + #[test] + fn rewrite_parallel_job_counts_uses_bootstrap_job_variable() { + assert_eq!( + rewrite_parallel_job_counts("meson compile -C build -j \"$(nproc)\""), + "meson compile -C build -j \"${LWI_MAKE_JOBS}\"" + ); + assert_eq!( + rewrite_parallel_job_counts("ninja -j$(nproc)"), + "ninja -j${LWI_MAKE_JOBS}" + ); + } + + #[test] + fn rewrite_lbi_command_removes_chroot_nproc_dependency() { + assert_eq!( + rewrite_lbi_command("meson compile -C build -j \"$(nproc)\"", None), + "meson compile -C build -j \"${LWI_MAKE_JOBS}\"" + ); + } + + #[test] + fn cross_toolchain_defaults_start_after_pass1() { + let pre_pass1 = BookPackage { + chapter: 5, + section: "5.2".to_string(), + title: "musl libc headers 1.2.6".to_string(), + name: "musl-libc-headers".to_string(), + version: "1.2.6".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/musl-libc-headers.html".to_string(), + recipe_id: "5-2-musl-libc-headers".to_string(), + }; + let pass1 = BookPackage { + chapter: 5, + section: "5.3".to_string(), + title: "llvm/clang pass 1 22.1.3".to_string(), + name: "llvm-clang-pass1".to_string(), + version: "22.1.3".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/llvm-clang-pass1.html".to_string(), + recipe_id: "5-3-llvm-clang-pass1".to_string(), + }; + let post_pass1 = BookPackage { + chapter: 5, + section: "5.4".to_string(), + title: "example package".to_string(), + name: "example".to_string(), + version: "1.0".to_string(), + layer: TEMP_LAYER.to_string(), + page_url: "https://example.invalid/chapter5/example.html".to_string(), + recipe_id: "5-4-example".to_string(), + }; + + assert!(!use_cross_toolchain_by_default(&pre_pass1)); + assert!(!use_cross_toolchain_by_default(&pass1)); + assert!(use_cross_toolchain_by_default(&post_pass1)); + } + + #[cfg(unix)] + #[test] + fn root_transition_skips_reexec_when_already_root() { + assert_eq!( + root_transition(true, Some(OsStr::new(""))).unwrap(), + RootTransition::AlreadyRoot + ); + } + + #[cfg(unix)] + #[test] + fn root_transition_prefers_sudo_then_doas() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::tempdir().unwrap(); + let bin = tmp.path().join("bin"); + fs::create_dir_all(&bin).unwrap(); + for name in ["sudo", "doas"] { + let path = bin.join(name); + fs::write(&path, "#!/bin/sh\n").unwrap(); + let mut perms = fs::metadata(&path).unwrap().permissions(); + perms.set_mode(0o755); + fs::set_permissions(path, perms).unwrap(); + } + let action = root_transition(false, Some(bin.as_os_str())).unwrap(); + + assert_eq!(action, RootTransition::Reexec(bin.join("sudo"))); + } + + #[test] + fn root_transition_errors_without_helper() { + let err = root_transition(false, Some(OsStr::new(""))).unwrap_err(); + assert!(err.to_string().contains("neither sudo nor doas")); + } + + #[test] + fn parses_body_style_section_lines() { + assert_eq!( + parse_section_line("5.5. LLVM runtimes (libunwind, libcxxabi, libcxx) 22.1.3"), + Some(( + "5.5".to_string(), + "LLVM runtimes (libunwind, libcxxabi, libcxx) 22.1.3".to_string() + )) + ); + } + + #[test] + fn parses_bulleted_toc_section_lines() { + assert_eq!( + parse_section_line("\u{c} ▪ 8.17 libffi 3.5.2"), + Some(("8.17".to_string(), "libffi 3.5.2".to_string())) + ); + assert_eq!( + parse_section_line(" • 8.23 Meson 1.11.1"), + Some(("8.23".to_string(), "Meson 1.11.1".to_string())) + ); + } + + #[test] + fn skips_chapter_heading_section_lines() { + assert_eq!(parse_section_line("5. Cross-Compilation Setup"), None); + assert_eq!( + parse_section_line("8. Compiling the Remaining utilities for the system"), + None + ); + } + + #[test] + fn normalizes_known_titles() { + assert_eq!( + package_name_from_title("BSD-Diffutils stage 2 0.99.0").as_deref(), + Some("bsddiffutils") + ); + assert_eq!( + package_name_from_title("patch stage 2 0.99.1").as_deref(), + Some("bsdpatch") + ); + assert_eq!( + package_name_from_title("uutils-coreutils 0.8.0").as_deref(), + Some("uutils-coreutils") + ); + assert_eq!( + package_name_from_title("musl libc final pass 1.2.6").as_deref(), + Some("musl") + ); + assert_eq!( + package_name_from_title("LLVM final 22.1.3").as_deref(), + Some("llvm") + ); + assert_eq!( + package_name_from_title("bzip2 1.0.8").as_deref(), + Some("bzip2") + ); + assert_eq!(package_name_from_title("m4 1.4.20").as_deref(), Some("m4")); + assert_eq!(package_name_from_title("."), None); + } + + #[cfg(unix)] + #[test] + fn initializes_lbi_layout_for_fresh_bootstrap() { + let tmp = tempfile::tempdir().unwrap(); + let config = config::Config::for_rootfs(tmp.path()); + + ensure_lbi_layout_for_fresh_bootstrap( + tmp.path(), + &config, + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + + let state = system_state::load(&config).unwrap(); + assert_eq!(state.stage.as_deref(), Some("layout")); + assert!(tmp.path().join("etc/depot.d/build.toml").exists()); + assert!(tmp.path().join("system/binaries").is_dir()); + } + + #[test] + fn essential_system_files_include_lwi_account_defaults() { + let tmp = tempfile::tempdir().unwrap(); + + create_essential_system_files(tmp.path()).unwrap(); + + let passwd = fs::read_to_string(tmp.path().join("etc/passwd")).unwrap(); + let group = fs::read_to_string(tmp.path().join("etc/group")).unwrap(); + assert!(passwd.contains("root:x:0:0:root:/system/charlie:/bin/oksh")); + assert!(passwd.contains("messagebus:x:18:18:D-Bus Message Daemon User")); + assert!(group.contains("users:x:999:")); + assert!(group.contains("wheel:x:97:")); + } + + #[test] + fn skips_lbi_layout_initialization_when_resuming() { + let tmp = tempfile::tempdir().unwrap(); + let config = config::Config::for_rootfs(tmp.path()); + system_state::set_stage(&config, "bootstrap-layers".to_string()).unwrap(); + + ensure_lbi_layout_for_fresh_bootstrap( + tmp.path(), + &config, + "x86_64-unknown-linux-musl", + "x86_64", + ) + .unwrap(); + + assert!(!tmp.path().join("etc/depot.d/build.toml").exists()); + assert_eq!( + fs::read_link(tmp.path().join("usr/include")).unwrap(), + PathBuf::from("../system/headers") + ); + assert_eq!( + fs::read_link(tmp.path().join("dev")).unwrap(), + PathBuf::from("system/devices") + ); + let state = system_state::load(&config).unwrap(); + assert_eq!(state.stage.as_deref(), Some("bootstrap-layers")); + } +} diff --git a/src/builder/custom.rs b/src/builder/custom.rs index e1880e2..7d9b6f2 100755 --- a/src/builder/custom.rs +++ b/src/builder/custom.rs @@ -54,14 +54,12 @@ pub fn build( crate::builder::apply_build_helper_context_env(&mut env_vars, spec)?; crate::builder::apply_build_helper_dirs_env(&mut env_vars, Some(src_dir), Some(&build_dir)); - // For custom builds, look for a build.sh script in the source directory + // For custom builds, the spec's build.sh is authoritative when present. + // Upstream source archives sometimes ship unrelated helper scripts with the + // same name, and the package spec needs to override them deterministically. let build_script = src_dir.join("build.sh"); - - // If the extracted source doesn't include build.sh but the spec directory does, - // copy it into the source dir (this makes `depot install ` behave - // like the spec's build.sh being part of the package when appropriate). let spec_build = spec.spec_dir.join("build.sh"); - if !build_script.exists() && spec_build.exists() { + if spec_build.exists() { fs::create_dir_all(src_dir)?; fs::copy(&spec_build, &build_script).with_context(|| { format!( @@ -477,6 +475,47 @@ mod tests { Ok(()) } + #[test] + fn test_build_prefers_spec_build_sh_over_source_build_sh() -> Result<()> { + let tmp_src = tempdir()?; + let tmp_dest = tempdir()?; + let spec_dir = tempdir()?; + + let source_build_sh = tmp_src.path().join("build.sh"); + std::fs::write(&source_build_sh, "#!/bin/sh\nexit 77\n")?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&source_build_sh)?.permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&source_build_sh, perms)?; + } + + let spec_build_sh = spec_dir.path().join("build.sh"); + std::fs::write( + &spec_build_sh, + "#!/bin/sh\nprintf 'from spec\\n' > \"$DESTDIR/spec-won\"\n", + )?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&spec_build_sh)?.permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&spec_build_sh, perms)?; + } + + let mut spec = mk_spec("custom-prefer-spec", "1.0"); + spec.spec_dir = spec_dir.path().to_path_buf(); + + build(&spec, tmp_src.path(), tmp_dest.path(), None, true, None)?; + + assert_eq!( + std::fs::read_to_string(tmp_dest.path().join("spec-won"))?, + "from spec\n" + ); + Ok(()) + } + #[test] fn test_build_function_mode_uses_per_output_destdirs() -> Result<()> { let tmp_src = tempdir()?; diff --git a/src/cli.rs b/src/cli.rs index e10eafd..7179405 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -144,6 +144,30 @@ pub struct BuildArgs { pub cleanup_deps: bool, } +/// Arguments for importing the Linux by Intent book package plan into Depot layers. +#[derive(Debug, Clone, Args)] +pub struct BootstrapArgs { + /// Target sysroot whose Depot system state should receive the parsed layers + #[arg(value_name = "SYSROOT")] + pub sysroot: PathBuf, + + /// Target triple used for cross and staged bootstrap builds + #[arg(long)] + pub target: Option, + + /// Target architecture component used by build defaults + #[arg(long)] + pub arch: Option, + + /// URL of the Linux by Intent PDF to parse + #[arg(long, default_value = "https://www.vertexlinux.net/lbi/book.pdf")] + pub book_url: String, + + /// Use a local PDF instead of fetching book-url + #[arg(long, value_name = "PDF")] + pub book_pdf: Option, +} + #[derive(Debug, Clone, Args)] pub struct UpdateArgs { #[command(flatten)] @@ -226,6 +250,70 @@ pub struct ConfigArgs { pub rootfs_args: RootfsArgs, } +/// Arguments for Depot system-build state management commands. +#[derive(Debug, Clone, Args)] +pub struct SystemArgs { + /// Root filesystem whose system state should be inspected or modified. + #[command(flatten)] + pub rootfs_args: RootfsArgs, + + /// Requested system state subcommand. + #[command(subcommand)] + pub command: SystemCommands, +} + +/// System-build state and Linux by Intent layout commands. +#[derive(Subcommand, Debug, Clone)] +pub enum SystemCommands { + /// Show tracked system build stage and package layers + Status, + /// Move the tracked system status to a build stage + Stage { + /// New stage name, such as cross-tools, minimal, chroot, stage2, or bootable + stage: String, + }, + /// Manage package layer membership + Layer { + #[command(subcommand)] + command: SystemLayerCommands, + }, + /// Initialize the Linux by Intent /system layout and Depot build defaults + InitLbi { + /// Target triple used for cross and staged builds + #[arg(long, default_value = "x86_64-unknown-linux-musl")] + target: String, + /// Target architecture component used by build defaults + #[arg(long)] + arch: Option, + /// Replace an existing Depot build config generated for the target rootfs + #[arg(long)] + force: bool, + }, +} + +/// Package layer membership commands. +#[derive(Subcommand, Debug, Clone)] +pub enum SystemLayerCommands { + /// Add packages to a named layer + Add { + /// Layer name, such as base, devel, toolchain, or boot + layer: String, + /// Package names to add to the layer + #[arg(required = true, num_args = 1..)] + packages: Vec, + }, + /// Remove packages from a named layer + Remove { + /// Layer name + layer: String, + /// Package names to remove from the layer + #[arg(required = true, num_args = 1..)] + packages: Vec, + }, + /// List all tracked layers + List, +} + #[derive(Debug, Clone, Args)] pub struct GenerateArtifactsArgs { /// Output directory for generated files @@ -265,6 +353,8 @@ pub enum Commands { Remove(RemoveArgs), /// Build a package without installing Build(BuildArgs), + /// Parse the Linux by Intent book and populate temp/base/devel layers + Bootstrap(BootstrapArgs), /// Update installed packages from configured repositories Update(UpdateArgs), /// Scan package specs for upstream version updates @@ -283,6 +373,8 @@ pub enum Commands { Repo(RepoArgs), /// Show current configuration Config(ConfigArgs), + /// Manage system build stage, package layers, and book-style layout state + System(SystemArgs), /// Generate shell completion scripts and a man page into an output directory. #[command(hide = true)] GenerateArtifacts(GenerateArtifactsArgs), @@ -317,6 +409,19 @@ pub enum InternalCommands { #[command(hide = true)] Clone { repo: String, dest: Option }, #[command(hide = true)] + BootstrapChroot { + #[arg(long)] + rootfs: PathBuf, + #[arg(long)] + sources: PathBuf, + #[arg(long)] + destdir: PathBuf, + #[arg(long)] + workdir: String, + #[arg(long)] + script: String, + }, + #[command(hide = true)] AutotoolsConfigure { #[arg(value_name = "ARG", num_args = 0.., allow_hyphen_values = true)] args: Vec, diff --git a/src/commands.rs b/src/commands.rs index 5eb7677..d003c7d 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -1,10 +1,11 @@ use crate::cli::{ BuildArgs, Cli, Commands, ConfigArgs, ConvertArgs, InfoArgs, InstallArgs, InternalCommands, - ListArgs, OwnsArgs, RemoveArgs, RepoCommands, RepoKindArg, SearchArgs, SignArgs, UpdateArgs, + ListArgs, OwnsArgs, RemoveArgs, RepoCommands, RepoKindArg, SearchArgs, SignArgs, SystemArgs, + UpdateArgs, }; use crate::{ - builder, cli_assets, config, cross, db, deps, index, install, locking, package, planner, - signing, source, staging, ui, + bootstrap, builder, cli_assets, config, cross, db, deps, index, install, locking, package, + planner, signing, source, staging, ui, }; use anyhow::{Context, Result}; use git2::Direction; @@ -19,10 +20,11 @@ use url::Url; use walkdir::WalkDir; use build_cmd::support::{ - automatic_tests_disabled_for_outputs, build_lib32_companion_package, clean_build_workspace, - effective_lib32_only, ensure_requested_development_package_installed, make_lib32_package_spec, - maybe_disable_tests_for_missing_deps, maybe_prompt_to_skip_tests_for_missing_requested_deps, - merge_missing_dependencies, requested_outputs, should_install_test_deps, + automatic_tests_disabled_for_outputs, build_lib32_companion_package, clean_build_source_dirs, + clean_build_workspace, effective_lib32_only, ensure_requested_development_package_installed, + make_lib32_package_spec, maybe_disable_tests_for_missing_deps, + maybe_prompt_to_skip_tests_for_missing_requested_deps, merge_missing_dependencies, + requested_outputs, should_install_test_deps, }; use install_cmd::archive::{ extract_package_archive_to_staging, load_package_archive_into_staging, @@ -89,6 +91,7 @@ fn command_rootfs(command: &Commands) -> Option<&Path> { Commands::Install(args) => Some(&args.rootfs_args.rootfs), Commands::Remove(args) => Some(&args.rootfs_args.rootfs), Commands::Build(args) => Some(&args.rootfs_args.rootfs), + Commands::Bootstrap(args) => Some(&args.sysroot), Commands::Update(args) => Some(&args.rootfs_args.rootfs), Commands::Info(args) => Some(&args.rootfs_args.rootfs), Commands::Search(args) => Some(&args.rootfs_args.rootfs), @@ -97,6 +100,7 @@ fn command_rootfs(command: &Commands) -> Option<&Path> { Commands::Sign(args) => Some(&args.rootfs_args.rootfs), Commands::Repo(args) => Some(repo_command_rootfs(&args.command)), Commands::Config(args) => Some(&args.rootfs_args.rootfs), + Commands::System(args) => Some(&args.rootfs_args.rootfs), Commands::Check(_) | Commands::Convert(_) | Commands::GenerateArtifacts(_) @@ -110,6 +114,7 @@ fn command_assume_yes(command: &Commands) -> bool { Commands::Install(args) => args.prompt_args.yes, Commands::Remove(args) => args.prompt_args.yes, Commands::Build(args) => args.prompt_args.yes, + Commands::Bootstrap(_) => false, Commands::Update(args) => args.prompt_args.yes, Commands::Check(_) | Commands::Convert(_) @@ -120,6 +125,7 @@ fn command_assume_yes(command: &Commands) -> bool { | Commands::Sign(_) | Commands::Repo(_) | Commands::Config(_) + | Commands::System(_) | Commands::GenerateArtifacts(_) | Commands::MakeSpec(_) | Commands::Internal(_) => false, @@ -913,7 +919,7 @@ fn collect_installed_replacement_packages( Ok(replacements) } -fn remove_installed_package_with_hooks( +pub(crate) fn remove_installed_package_with_hooks( package: &str, rootfs: &Path, config: &config::Config, @@ -2054,6 +2060,27 @@ fn run_direct_archive_install_requests( Ok(true) } +struct SourceBuildCleanupGuard<'a> { + config: &'a config::Config, + enabled: bool, +} + +impl<'a> SourceBuildCleanupGuard<'a> { + fn new(config: &'a config::Config, enabled: bool) -> Self { + Self { config, enabled } + } +} + +impl Drop for SourceBuildCleanupGuard<'_> { + fn drop(&mut self) { + if self.enabled + && let Err(err) = clean_build_source_dirs(self.config) + { + crate::log_warn!("Failed to clean build source dirs: {}", err); + } + } +} + fn run_direct_install_request( options: DirectInstallOptions<'_>, config: &config::Config, @@ -2142,6 +2169,8 @@ fn run_direct_install_request( pkg_spec.apply_config(config); (pkg_spec, None) }; + let built_from_source = staging_dir.is_none(); + let _source_cleanup_guard = SourceBuildCleanupGuard::new(config, built_from_source); if options.lib32_only && staging_dir.is_some() { anyhow::bail!("--lib32-only is only supported when installing from a package spec"); @@ -2438,6 +2467,7 @@ fn run_direct_install_request( dir.path().to_path_buf() } else { // 1-2. Fetch + extract sources (supports archives and git URL#rev) + clean_build_source_dirs(config)?; source::preflight_manual_sources(&pkg_spec, &config.cache_dir)?; let src_dir = source::prepare(&pkg_spec, &config.cache_dir, &config.build_dir)?; built_src_dir = Some(src_dir.clone()); @@ -2551,7 +2581,8 @@ pub fn run(cli: Cli) -> Result<()> { Commands::Install(args) => args.build_exec_args.test_deps, Commands::Build(args) => args.build_exec_args.test_deps, Commands::Update(args) => args.build_exec_args.test_deps, - Commands::Check(_) + Commands::Bootstrap(_) + | Commands::Check(_) | Commands::Remove(_) | Commands::Info(_) | Commands::Search(_) @@ -2560,6 +2591,7 @@ pub fn run(cli: Cli) -> Result<()> { | Commands::Sign(_) | Commands::Repo(_) | Commands::Config(_) + | Commands::System(_) | Commands::GenerateArtifacts(_) | Commands::Convert(_) | Commands::MakeSpec(_) @@ -2570,6 +2602,7 @@ pub fn run(cli: Cli) -> Result<()> { Commands::Install(args) => install_cmd::run_install(args, cli_test_deps)?, Commands::Remove(args) => install_cmd::run_remove(args)?, Commands::Build(args) => build_cmd::run_build(args, cli_test_deps)?, + Commands::Bootstrap(args) => bootstrap::run(args)?, Commands::Update(args) => update::run_update(args, cli_test_deps)?, Commands::Check(args) => check::run_check(args)?, Commands::Info(args) => misc::run_info(args)?, @@ -2580,6 +2613,7 @@ pub fn run(cli: Cli) -> Result<()> { Commands::Repo(args) => repo::run_repo(args.command)?, Commands::GenerateArtifacts(args) => misc::run_generate_artifacts(args)?, Commands::Config(args) => misc::run_config(args)?, + Commands::System(args) => misc::run_system(args)?, Commands::MakeSpec(args) => misc::run_make_spec(args)?, Commands::Convert(args) => misc::run_convert(args)?, Commands::Internal(args) => misc::run_internal(args)?, diff --git a/src/commands/build_cmd.rs b/src/commands/build_cmd.rs index d0849f3..8d80b9a 100644 --- a/src/commands/build_cmd.rs +++ b/src/commands/build_cmd.rs @@ -4,11 +4,11 @@ pub(crate) mod support; use self::support::{ RequestedBuildToolPackageInstall, automatic_tests_disabled_for_outputs, - build_lib32_companion_package, clean_build_workspace, effective_lib32_only, - ensure_requested_build_tool_package_installed, ensure_requested_development_package_installed, - maybe_disable_tests_for_missing_deps, maybe_prompt_to_skip_tests_for_missing_requested_deps, - merge_missing_dependencies, requested_outputs, should_install_test_deps, - warn_if_running_as_root_for_build, + build_lib32_companion_package, clean_build_source_dirs, clean_build_workspace, + effective_lib32_only, ensure_requested_build_tool_package_installed, + ensure_requested_development_package_installed, maybe_disable_tests_for_missing_deps, + maybe_prompt_to_skip_tests_for_missing_requested_deps, merge_missing_dependencies, + requested_outputs, should_install_test_deps, warn_if_running_as_root_for_build, }; pub(super) fn run_build(args: BuildArgs, cli_test_deps: bool) -> Result<()> { @@ -303,6 +303,7 @@ pub(super) fn run_build(args: BuildArgs, cli_test_deps: bool) -> Result<()> { watcher.check()?; } + clean_build_source_dirs(&config)?; source::preflight_manual_sources(&pkg_spec, &config.cache_dir)?; let src_dir = source::prepare(&pkg_spec, &config.cache_dir, &config.build_dir)?; if let Some(watcher) = interrupt_watcher.as_ref() { @@ -507,6 +508,7 @@ pub(super) fn run_build(args: BuildArgs, cli_test_deps: bool) -> Result<()> { false, )?; } + clean_build_source_dirs(&config)?; if clean { clean_build_workspace(&config)?; } @@ -529,6 +531,12 @@ pub(super) fn run_build(args: BuildArgs, cli_test_deps: bool) -> Result<()> { )); } } + if let Err(clean_err) = clean_build_source_dirs(&config) { + ui::warn(format!( + "Failed to clean build source dirs after failed build: {}", + clean_err + )); + } if interrupted { anyhow::bail!("Build interrupted by Ctrl-C"); } diff --git a/src/commands/build_cmd/support.rs b/src/commands/build_cmd/support.rs index 6278740..410c918 100644 --- a/src/commands/build_cmd/support.rs +++ b/src/commands/build_cmd/support.rs @@ -115,6 +115,22 @@ pub(crate) fn clean_build_workspace(config: &config::Config) -> Result<()> { Ok(()) } +pub(crate) fn clean_build_source_dirs(config: &config::Config) -> Result<()> { + if config.build_dir.exists() { + fs::remove_dir_all(&config.build_dir).with_context(|| { + format!( + "Failed to clean build source dirs: {}", + config.build_dir.display() + ) + })?; + ui::success(format!( + "Cleaned build source dirs: {}", + config.build_dir.display() + )); + } + Ok(()) +} + pub(crate) fn warn_if_running_as_root_for_build(command: &str, rootfs: &Path) { if crate::fakeroot::is_root() { ui::warn(format!("Running '{}' as root is discouraged.", command)); diff --git a/src/commands/misc.rs b/src/commands/misc.rs index 8e6c4a8..8bec41c 100644 --- a/src/commands/misc.rs +++ b/src/commands/misc.rs @@ -112,6 +112,117 @@ pub(super) fn run_config(args: ConfigArgs) -> Result<()> { Ok(()) } +pub(super) fn run_system(args: SystemArgs) -> Result<()> { + let SystemArgs { + rootfs_args, + command, + } = args; + let rootfs = rootfs_args.rootfs; + let config = config::Config::for_rootfs(&rootfs); + let mut system_lock = locking::open_lock(&config)?; + let system_lock_path = locking::lock_path(&config); + + match command { + crate::cli::SystemCommands::Status => { + let _guard = locking::try_read(&system_lock, &system_lock_path, "system status")?; + let state = crate::system_state::load(&config)?; + print_system_state(&state); + } + crate::cli::SystemCommands::Stage { stage } => { + let _guard = locking::try_write(&mut system_lock, &system_lock_path, "system stage")?; + let state = crate::system_state::set_stage(&config, stage)?; + ui::success(format!( + "Moved system status to stage {}", + state.stage.as_deref().unwrap_or("unknown") + )); + } + crate::cli::SystemCommands::Layer { command } => { + let _guard = locking::try_write(&mut system_lock, &system_lock_path, "system layer")?; + match command { + crate::cli::SystemLayerCommands::Add { layer, packages } => { + let package_count = packages.len(); + crate::system_state::add_packages_to_layer(&config, layer.clone(), &packages)?; + ui::success(format!( + "Added {} package(s) to layer {}", + package_count, layer + )); + } + crate::cli::SystemLayerCommands::Remove { layer, packages } => { + let package_count = packages.len(); + crate::system_state::remove_packages_from_layer( + &config, + layer.clone(), + &packages, + )?; + ui::success(format!( + "Removed {} package(s) from layer {}", + package_count, layer + )); + } + crate::cli::SystemLayerCommands::List => { + drop(_guard); + let _guard = + locking::try_read(&system_lock, &system_lock_path, "system layer list")?; + let state = crate::system_state::load(&config)?; + print_system_layers(&state); + } + } + } + crate::cli::SystemCommands::InitLbi { + target, + arch, + force, + } => { + let _guard = + locking::try_write(&mut system_lock, &system_lock_path, "system init-lbi")?; + let state = crate::system_state::init_lbi_layout( + &rootfs, + &config, + &target, + arch.as_deref(), + force, + )?; + ui::success(format!( + "Initialized Linux by Intent layout for {} ({})", + state.target.as_deref().unwrap_or("unknown"), + state.arch.as_deref().unwrap_or("unknown") + )); + } + } + + Ok(()) +} + +fn print_system_state(state: &crate::system_state::SystemState) { + println!( + "Stage: {}", + state.stage.as_deref().unwrap_or("uninitialized") + ); + if let Some(target) = &state.target { + println!("Target: {target}"); + } + if let Some(arch) = &state.arch { + println!("Arch: {arch}"); + } + print_system_layers(state); +} + +fn print_system_layers(state: &crate::system_state::SystemState) { + if state.layers.is_empty() { + println!("Layers: none"); + return; + } + + println!("Layers:"); + for (layer, packages) in &state.layers { + if packages.is_empty() { + println!(" {layer}:"); + } else { + println!(" {}: {}", layer, packages.join(", ")); + } + } +} + pub(super) fn run_make_spec(args: crate::cli::MakeSpecArgs) -> Result<()> { let output = args.output; let spec = package::create_interactive()?; diff --git a/src/commands/misc/internal.rs b/src/commands/misc/internal.rs index 9e31910..aff15ef 100644 --- a/src/commands/misc/internal.rs +++ b/src/commands/misc/internal.rs @@ -143,6 +143,13 @@ pub(crate) fn run_internal_command(command: InternalCommands) -> Result<()> { &[], ) } + InternalCommands::BootstrapChroot { + rootfs, + sources, + destdir, + workdir, + script, + } => crate::bootstrap::run_bootstrap_chroot(&rootfs, &sources, &destdir, &workdir, &script), InternalCommands::AutotoolsConfigure { args } => { let env_vars = current_process_env_vars(); let context = current_build_helper_context()?; diff --git a/src/commands/tests.rs b/src/commands/tests.rs index e5904a5..1931d01 100644 --- a/src/commands/tests.rs +++ b/src/commands/tests.rs @@ -282,6 +282,42 @@ fn clean_build_workspace_noops_when_dirs_are_missing() -> Result<()> { Ok(()) } +#[test] +fn clean_build_source_dirs_removes_build_dir_only() -> Result<()> { + let rootfs = tempfile::tempdir().context("Failed to create temp rootfs")?; + let mut cfg = config::Config::for_rootfs(rootfs.path()); + cfg.build_dir = rootfs.path().join("tmp/build"); + cfg.cache_dir = rootfs.path().join("tmp/sources"); + + fs::create_dir_all(&cfg.build_dir) + .with_context(|| format!("Failed to create {}", cfg.build_dir.display()))?; + fs::create_dir_all(&cfg.cache_dir) + .with_context(|| format!("Failed to create {}", cfg.cache_dir.display()))?; + + clean_build_source_dirs(&cfg)?; + + assert!(!cfg.build_dir.exists()); + assert!(cfg.cache_dir.exists()); + Ok(()) +} + +#[test] +fn clean_build_source_dirs_noops_when_build_dir_missing() -> Result<()> { + let rootfs = tempfile::tempdir().context("Failed to create temp rootfs")?; + let mut cfg = config::Config::for_rootfs(rootfs.path()); + cfg.build_dir = rootfs.path().join("tmp/build"); + cfg.cache_dir = rootfs.path().join("tmp/sources"); + + fs::create_dir_all(&cfg.cache_dir) + .with_context(|| format!("Failed to create {}", cfg.cache_dir.display()))?; + + clean_build_source_dirs(&cfg)?; + + assert!(!cfg.build_dir.exists()); + assert!(cfg.cache_dir.exists()); + Ok(()) +} + #[test] fn binary_install_path_uses_repo_record_metadata_without_archive_metadata() -> Result<()> { let rootfs = tempfile::tempdir().context("Failed to create temp rootfs")?; diff --git a/src/cross.rs b/src/cross.rs index 21ec38a..68d3207 100755 --- a/src/cross.rs +++ b/src/cross.rs @@ -41,20 +41,27 @@ impl CrossConfig { let cc = find_tool(prefix, &["gcc", "clang"], true)?; let cxx = find_tool(prefix, &["g++", "clang++"], false) .unwrap_or_else(|_| format!("{}-g++", prefix)); - let ar = find_tool(prefix, &["ar", "llvm-ar"], true)?; + let ar = + find_tool_with_unprefixed_fallback(prefix, &["ar", "llvm-ar"], &["llvm-ar"], true)?; let ranlib = find_tool(prefix, &["ranlib", "llvm-ranlib"], false) + .or_else(|_| find_unprefixed_tool(&["llvm-ranlib", "llvm-ar"], false)) .unwrap_or_else(|_| format!("{}-ranlib", prefix)); let strip = find_tool(prefix, &["strip", "llvm-strip"], false) + .or_else(|_| find_unprefixed_tool(&["llvm-strip", "llvm-objcopy"], false)) .unwrap_or_else(|_| format!("{}-strip", prefix)); let ld = find_tool(prefix, &["ld", "ld.lld"], false) .unwrap_or_else(|_| format!("{}-ld", prefix)); let nm = find_tool(prefix, &["nm", "llvm-nm"], false) + .or_else(|_| find_unprefixed_tool(&["llvm-nm"], false)) .unwrap_or_else(|_| format!("{}-nm", prefix)); let objcopy = find_tool(prefix, &["objcopy", "llvm-objcopy"], false) + .or_else(|_| find_unprefixed_tool(&["llvm-objcopy"], false)) .unwrap_or_else(|_| format!("{}-objcopy", prefix)); let objdump = find_tool(prefix, &["objdump", "llvm-objdump"], false) + .or_else(|_| find_unprefixed_tool(&["llvm-objdump"], false)) .unwrap_or_else(|_| format!("{}-objdump", prefix)); let readelf = find_tool(prefix, &["readelf", "llvm-readelf"], false) + .or_else(|_| find_unprefixed_tool(&["llvm-readelf", "llvm-readobj"], false)) .unwrap_or_else(|_| format!("{}-readelf", prefix)); crate::log_info!("Cross-compilation tools discovered:"); @@ -244,6 +251,32 @@ fn find_tool(prefix: &str, suffixes: &[&str], required: bool) -> Result Err(anyhow::anyhow!("Tool not found")) } +fn find_tool_with_unprefixed_fallback( + prefix: &str, + suffixes: &[&str], + unprefixed: &[&str], + required: bool, +) -> Result { + find_tool(prefix, suffixes, false).or_else(|_| find_unprefixed_tool(unprefixed, required)) +} + +fn find_unprefixed_tool(names: &[&str], required: bool) -> Result { + for name in names { + if let Ok(output) = Command::new("which").arg(name).output() + && output.status.success() + { + let path = String::from_utf8_lossy(&output.stdout).trim().to_string(); + return Ok(path); + } + } + + if required { + anyhow::bail!("Could not find tool: {{{}}}", names.join("|")); + } + + Err(anyhow::anyhow!("Tool not found")) +} + #[cfg(test)] mod tests { use super::lib32_target_triple; diff --git a/src/db/mod.rs b/src/db/mod.rs index d2f31cd..10a5452 100755 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -10,6 +10,8 @@ use std::fs; use std::path::Path; use walkdir::WalkDir; +const DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS: &str = "DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS"; + fn format_licenses(licenses: &[String]) -> String { licenses.join(", ") } @@ -18,6 +20,14 @@ fn verbose_remove_output() -> bool { std::env::var_os("DEPOT_VERBOSE_REMOVE").is_some() } +fn should_ignore_sbase_conflicts() -> bool { + std::env::var_os(DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS).is_some() +} + +fn should_auto_clear_conflict(owner: &str, path: &str) -> bool { + (owner == "sbase" && should_ignore_sbase_conflicts()) || is_auto_removable_path(path) +} + /// Installed package row from the local package database. #[derive(Debug, Clone, PartialEq, Eq)] pub struct InstalledPackageRecord { @@ -213,7 +223,7 @@ pub fn register_package_with_replacement( if let Ok(owner) = owner_res && owner != spec.package.name { - if is_auto_removable_path(file) { + if should_auto_clear_conflict(&owner, file) { auto_conflicts.push((file.clone(), owner)); } else { fatal_conflicts.push((file.clone(), owner)); @@ -241,6 +251,15 @@ pub fn register_package_with_replacement( )?; if let Some(rootfs) = &rootfs_opt { + if destdir.join(f).symlink_metadata().is_ok() { + crate::log_info!( + "Auto-cleared DB ownership for path: {} (previously owned by {})", + f, + owner + ); + continue; + } + let disk_path = rootfs.join(f); if disk_path.exists() { let _ = std::fs::remove_file(&disk_path); @@ -1056,6 +1075,7 @@ mod tests { use crate::package::{ Alternatives, Build, BuildFlags, BuildType, Dependencies, PackageInfo, PackageSpec, Source, }; + use crate::test_support::TestEnv; use std::path::PathBuf; fn mk_spec(name: &str, version: &str) -> PackageSpec { @@ -1243,6 +1263,69 @@ mod tests { assert!(files_b.contains(&"usr/share/perl5/shared.pm".to_string())); } + #[test] + fn register_package_auto_clears_sbase_conflicts_when_requested() { + let tmp = tempfile::tempdir().unwrap(); + let rootfs = tmp.path().join("rootfs"); + let db_path = crate::config::Config::for_rootfs(&rootfs).installed_db_path(&rootfs); + std::fs::create_dir_all(rootfs.join("system/binaries")).unwrap(); + std::fs::write(rootfs.join("system/binaries/find"), "sbase find").unwrap(); + + let spec_a = mk_spec("sbase", "1.0"); + let dest_a = tmp.path().join("dest_a"); + std::fs::create_dir_all(dest_a.join("system/binaries")).unwrap(); + std::fs::write(dest_a.join("system/binaries/find"), "sbase find").unwrap(); + register_package(&db_path, &spec_a, &dest_a).unwrap(); + + let spec_b = mk_spec("bfs", "4.1"); + let dest_b = tmp.path().join("dest_b"); + std::fs::create_dir_all(dest_b.join("system/binaries")).unwrap(); + std::fs::write(dest_b.join("system/binaries/find"), "bfs find").unwrap(); + + let mut env = TestEnv::new(); + env.set_var(DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS, "1"); + register_package(&db_path, &spec_b, &dest_b).unwrap(); + + let files_sbase = get_package_files(&db_path, "sbase").unwrap(); + assert!(!files_sbase.contains(&"system/binaries/find".to_string())); + let files_bfs = get_package_files(&db_path, "bfs").unwrap(); + assert!(files_bfs.contains(&"system/binaries/find".to_string())); + } + + #[test] + fn register_package_auto_clear_preserves_new_payload_on_disk() { + let tmp = tempfile::tempdir().unwrap(); + let rootfs = tmp.path().join("rootfs"); + let db_path = crate::config::Config::for_rootfs(&rootfs).installed_db_path(&rootfs); + std::fs::create_dir_all(rootfs.join("system/binaries")).unwrap(); + std::fs::write(rootfs.join("system/binaries/find"), "sbase find").unwrap(); + + let spec_a = mk_spec("sbase", "1.0"); + let dest_a = tmp.path().join("dest_a"); + std::fs::create_dir_all(dest_a.join("system/binaries")).unwrap(); + std::fs::write(dest_a.join("system/binaries/find"), "sbase find").unwrap(); + register_package(&db_path, &spec_a, &dest_a).unwrap(); + + std::fs::write(rootfs.join("system/binaries/find"), "bfs find").unwrap(); + let spec_b = mk_spec("bfs", "4.1"); + let dest_b = tmp.path().join("dest_b"); + std::fs::create_dir_all(dest_b.join("system/binaries")).unwrap(); + std::fs::write(dest_b.join("system/binaries/find"), "bfs find").unwrap(); + + let mut env = TestEnv::new(); + env.set_var(DEPOT_BOOTSTRAP_IGNORE_SBASE_CONFLICTS, "1"); + register_package(&db_path, &spec_b, &dest_b).unwrap(); + + assert_eq!( + std::fs::read_to_string(rootfs.join("system/binaries/find")).unwrap(), + "bfs find" + ); + let files_sbase = get_package_files(&db_path, "sbase").unwrap(); + assert!(!files_sbase.contains(&"system/binaries/find".to_string())); + let files_bfs = get_package_files(&db_path, "bfs").unwrap(); + assert!(files_bfs.contains(&"system/binaries/find".to_string())); + } + #[test] fn get_package_files_missing_package_returns_empty() { let tmp = tempfile::tempdir().unwrap(); diff --git a/src/main.rs b/src/main.rs index 835fac6..b57366c 100755 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ //! Depot - Not Your Average Package Manager //! A source-based package manager for Linux +mod bootstrap; mod build_options; mod builder; mod cli; @@ -24,6 +25,7 @@ mod shell_helpers; mod signing; mod source; mod staging; +mod system_state; #[cfg(test)] mod test_support; mod ui; diff --git a/src/source/fetcher.rs b/src/source/fetcher.rs index 203d928..f6e5fa8 100755 --- a/src/source/fetcher.rs +++ b/src/source/fetcher.rs @@ -8,9 +8,17 @@ use std::collections::HashSet; use std::fs::{self, File}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; +use std::time::Duration; use url::Url; const MAX_MIRROR_RETRIES: usize = 8; +const HTTP_FETCH_TIMEOUT: Duration = Duration::from_secs(120); +const HTTP_FETCH_RETRY_LIMIT: usize = 2; +const MUSL_1_2_6_SNAPSHOT_URL: &str = + "https://git.musl-libc.org/cgit/musl/snapshot/musl-1.2.6.tar.gz"; +const MUSL_1_2_6_RELEASE_URL: &str = "https://musl.libc.org/releases/musl-1.2.6.tar.gz"; +const MUSL_1_2_6_GENTOO_MIRROR_URL: &str = + "https://tw.archive.ubuntu.com/gentoo/distfiles/9d/musl-1.2.6.tar.gz"; fn scheme_uses_http_transport(scheme: &str) -> bool { matches!(scheme, "http" | "https") @@ -32,10 +40,6 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R return Ok(dest_path); } - crate::log_info!("Fetching: {}", url); - - // Parse URL early so we can handle non-HTTP schemes (FTP support) - let parsed_url = Url::parse(&url).with_context(|| format!("Invalid URL: {}", url))?; let pb = ProgressBar::new(0); pb.set_style( ProgressStyle::default_bar() @@ -46,256 +50,46 @@ pub fn fetch_archive(spec: &PackageSpec, source: &Source, cache_dir: &Path) -> R .progress_chars("#>-"), ); - // If this is an FTP URL, fetch via suppaftp and skip the HTTP client path. - if parsed_url.scheme() == "ftp" { - // Connect and login (anonymous fallback) - let host = parsed_url.host_str().context("FTP URL missing host")?; - let port = parsed_url.port_or_known_default().unwrap_or(21); - let addr = format!("{}:{}", host, port); - let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str()) - .with_context(|| format!("Failed to connect to FTP host: {}", addr))?; - let user = if parsed_url.username().is_empty() { - "anonymous" + let candidate_urls = archive_fetch_candidates(&url); + let mut attempt_errors = Vec::new(); + + for (index, candidate_url) in candidate_urls.iter().enumerate() { + if index == 0 { + crate::log_info!("Fetching: {}", candidate_url); } else { - parsed_url.username() - }; - let pass = parsed_url.password().unwrap_or("anonymous@"); - ftp_stream - .login(user, pass) - .with_context(|| format!("FTP login failed for {}", host))?; - - // Retrieve the path (try with and without leading slash) - let path = parsed_url.path(); - let candidates = [path.to_string(), path.trim_start_matches('/').to_string()]; - let mut retrieved = false; - for p in candidates.iter().filter(|s| !s.is_empty()) { - match ftp_stream.retr( - p, - |reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> { - let mut file = - File::create(&dest_path).map_err(suppaftp::FtpError::ConnectionError)?; - let mut buffer = [0u8; 8192]; - loop { - let bytes_read = reader - .read(&mut buffer) - .map_err(suppaftp::FtpError::ConnectionError)?; - if bytes_read == 0 { - break; - } - file.write_all(&buffer[..bytes_read]) - .map_err(suppaftp::FtpError::ConnectionError)?; - } - Ok(()) - }, - ) { - Ok(_) => { - retrieved = true; - break; - } - Err(_) => continue, - } - } - ftp_stream.quit().ok(); - if !retrieved { - bail!("FTP error fetching {}", url); - } - } else { - if !scheme_uses_http_transport(parsed_url.scheme()) { - bail!( - "Unsupported URL scheme for source fetch: {}", - parsed_url.scheme() - ); + crate::log_info!("Primary source failed; trying fallback: {}", candidate_url); } - // Download with progress bar - // Use a sensible default User-Agent so servers that reject empty/unknown agents (e.g. IANA) - // will accept requests. Include package name/version at compile time. - let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); - let client = super::build_blocking_client(&ua, None) - .with_context(|| "Failed to build HTTP client")?; - - let mut response = client - .get(&url) - .send() - .with_context(|| format!("Failed to fetch: {}", url))?; - // If the server returned a non-success status, read a short body preview and fail early. - // This prevents saving HTML error pages (which then fail checksum) and gives a clearer - // diagnostic to the user. - let status = response.status(); - if !status.is_success() { - let mut preview_bytes = Vec::new(); - // read up to 1 KiB for a preview (ignore errors while reading preview) - let _ = response.take(1024).read_to_end(&mut preview_bytes); - let preview = String::from_utf8_lossy(&preview_bytes); - bail!( - "HTTP error fetching {}: {}{}", - url, - status, - if preview.trim().is_empty() { - "".to_string() - } else { - format!(" — preview: {}", preview.trim()) - } - ); - } - let total_size = response.content_length().unwrap_or(0); - pb.set_length(total_size); - - let mut file = File::create(&dest_path) - .with_context(|| format!("Failed to create: {}", dest_path.display()))?; - - let mut buffer = [0u8; 8192]; - let mut downloaded = 0u64; - - loop { - let bytes_read = response.read(&mut buffer)?; - if bytes_read == 0 { - break; - } - file.write_all(&buffer[..bytes_read])?; - downloaded += bytes_read as u64; - pb.set_position(downloaded); - } - - pb.finish_with_message("Download complete"); - } - - // Quick validation: ensure the downloaded file looks like the expected - // archive (detect obvious HTML error pages or wrong formats by magic). - // If validation returns an alternate URL (e.g. SourceForge mirror), follow - // and retry a few times. - let mut next_alt = validate_downloaded_archive(&dest_path, &filename, &url)?; - let mut seen_alts: HashSet = HashSet::new(); - let mut retries = 0usize; - while let Some(alt) = next_alt { - retries += 1; - if retries > MAX_MIRROR_RETRIES { - bail!( - "Exceeded mirror retry limit ({}) while fetching {}", - MAX_MIRROR_RETRIES, - url - ); - } - if !seen_alts.insert(alt.clone()) { - bail!("Mirror retry loop detected for URL: {}", alt); - } - crate::log_info!("Retrying download from mirror: {}", alt); + pb.set_position(0); + pb.set_length(0); fs::remove_file(&dest_path).ok(); - // If mirror URL is FTP -> use suppaftp; otherwise use HTTP retry. - if let Ok(alt_url) = Url::parse(&alt) { - if alt_url.scheme() == "ftp" { - // FTP mirror retrieval - let host = alt_url.host_str().context("FTP mirror URL missing host")?; - let port = alt_url.port_or_known_default().unwrap_or(21); - let addr = format!("{}:{}", host, port); - let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str()) - .with_context(|| format!("Failed to connect to FTP host: {}", addr))?; - let user = if alt_url.username().is_empty() { - "anonymous" - } else { - alt_url.username() - }; - let pass = alt_url.password().unwrap_or("anonymous@"); - ftp_stream - .login(user, pass) - .with_context(|| format!("FTP login failed for {}", host))?; - - let path = alt_url.path(); - let candidates = [path.to_string(), path.trim_start_matches('/').to_string()]; - let mut retrieved = false; - for p in candidates.iter().filter(|s| !s.is_empty()) { - match ftp_stream.retr( - p, - |reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> { - let mut file = File::create(&dest_path) - .map_err(suppaftp::FtpError::ConnectionError)?; - let mut buffer = [0u8; 8192]; - let mut downloaded = 0u64; - loop { - let bytes_read = reader - .read(&mut buffer) - .map_err(suppaftp::FtpError::ConnectionError)?; - if bytes_read == 0 { - break; - } - file.write_all(&buffer[..bytes_read]) - .map_err(suppaftp::FtpError::ConnectionError)?; - downloaded += bytes_read as u64; - pb.set_position(downloaded); - } - Ok(()) - }, - ) { - Ok(_) => { - retrieved = true; - break; - } - Err(_) => continue, - } - } - ftp_stream.quit().ok(); - if !retrieved { - bail!("FTP mirror error fetching {}", alt); - } - } else { - // HTTP(S) mirror retry (recreate client for retry) - let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); - let client = super::build_blocking_client(&ua, None) - .with_context(|| "Failed to build HTTP client")?; - let mut response = client - .get(&alt) - .send() - .with_context(|| format!("Failed to fetch mirror URL: {}", alt))?; - - let status = response.status(); - if !status.is_success() { - let mut preview_bytes = Vec::new(); - let _ = response.take(1024).read_to_end(&mut preview_bytes); - let preview = String::from_utf8_lossy(&preview_bytes); - bail!( - "HTTP error fetching {}: {}{}", - alt, - status, - if preview.trim().is_empty() { - "".to_string() - } else { - format!(" — preview: {}", preview.trim()) - } - ); + match fetch_archive_from_candidate(candidate_url, &dest_path, &filename, &pb) { + Ok(()) => { + if !verify_checksum(&dest_path, &source.sha256)? { + fs::remove_file(&dest_path)?; + attempt_errors.push(format!( + "{}: checksum verification failed for {}", + candidate_url, filename + )); + continue; } - let mut file = File::create(&dest_path) - .with_context(|| format!("Failed to create: {}", dest_path.display()))?; - let mut buffer = [0u8; 8192]; - let mut downloaded = 0u64; - loop { - let bytes_read = response.read(&mut buffer)?; - if bytes_read == 0 { - break; - } - file.write_all(&buffer[..bytes_read])?; - downloaded += bytes_read as u64; - pb.set_position(downloaded); - } + crate::log_info!("Checksum verified: {}", filename); + return Ok(dest_path); + } + Err(err) => { + attempt_errors.push(format!("{}: {err:#}", candidate_url)); + fs::remove_file(&dest_path).ok(); } - } else { - // Fallback: unknown/malformed alt URL -> bail - bail!("Malformed mirror URL: {}", alt); } - - pb.finish_with_message("Download complete (mirror)"); - next_alt = validate_downloaded_archive(&dest_path, &filename, &alt)?; } - // Verify checksum - if !verify_checksum(&dest_path, &source.sha256)? { - fs::remove_file(&dest_path)?; - bail!("Checksum verification failed for {}", filename); - } - - crate::log_info!("Checksum verified: {}", filename); - Ok(dest_path) + bail!( + "Failed to fetch {} from any candidate URL:\n{}", + filename, + attempt_errors.join("\n") + ) } /// Verify checksum of a file supporting optional algorithm prefix. @@ -338,6 +132,238 @@ pub(crate) fn derive_filename_from_url(url: &str) -> String { format!("source-{}.download", &hex[..12]) } +fn archive_fetch_candidates(primary_url: &str) -> Vec { + let mut candidates = Vec::new(); + if matches!( + primary_url, + MUSL_1_2_6_SNAPSHOT_URL | MUSL_1_2_6_RELEASE_URL | MUSL_1_2_6_GENTOO_MIRROR_URL + ) { + candidates.push(MUSL_1_2_6_GENTOO_MIRROR_URL.to_string()); + } + if !candidates.iter().any(|url| url == primary_url) { + candidates.push(primary_url.to_string()); + } + if let Some(mirror_url) = musl_release_mirror_url(primary_url) + && !candidates.contains(&mirror_url) + { + candidates.push(mirror_url); + } + candidates +} + +fn musl_release_mirror_url(url: &str) -> Option { + let parsed = Url::parse(url).ok()?; + if parsed.host_str()? != "git.musl-libc.org" { + return None; + } + + let mut segments = parsed.path_segments()?; + match ( + segments.next(), + segments.next(), + segments.next(), + segments.next(), + segments.next(), + ) { + (Some("cgit"), Some("musl"), Some("snapshot"), Some(filename), None) + if !filename.trim().is_empty() => + { + Some(format!("https://musl.libc.org/releases/{filename}")) + } + _ => None, + } +} + +fn fetch_archive_from_candidate( + candidate_url: &str, + dest_path: &Path, + filename: &str, + pb: &ProgressBar, +) -> Result<()> { + let mut current_url = candidate_url.to_string(); + let mut seen_alts: HashSet = HashSet::new(); + let mut retries = 0usize; + + loop { + download_archive_from_url(¤t_url, dest_path, pb)?; + pb.finish_with_message("Download complete"); + + let Some(next_alt) = validate_downloaded_archive(dest_path, filename, ¤t_url)? else { + return Ok(()); + }; + + retries += 1; + if retries > MAX_MIRROR_RETRIES { + bail!( + "Exceeded mirror retry limit ({}) while fetching {}", + MAX_MIRROR_RETRIES, + candidate_url + ); + } + if !seen_alts.insert(next_alt.clone()) { + bail!("Mirror retry loop detected for URL: {}", next_alt); + } + + crate::log_info!("Retrying download from mirror: {}", next_alt); + fs::remove_file(dest_path).ok(); + pb.set_position(0); + pb.set_length(0); + current_url = next_alt; + } +} + +fn download_archive_from_url(url: &str, dest_path: &Path, pb: &ProgressBar) -> Result<()> { + let parsed_url = Url::parse(url).with_context(|| format!("Invalid URL: {}", url))?; + if parsed_url.scheme() == "ftp" { + return download_ftp_archive(&parsed_url, dest_path, pb); + } + if !scheme_uses_http_transport(parsed_url.scheme()) { + bail!( + "Unsupported URL scheme for source fetch: {}", + parsed_url.scheme() + ); + } + download_http_archive(url, dest_path, pb) +} + +fn download_ftp_archive(parsed_url: &Url, dest_path: &Path, pb: &ProgressBar) -> Result<()> { + let host = parsed_url.host_str().context("FTP URL missing host")?; + let port = parsed_url.port_or_known_default().unwrap_or(21); + let addr = format!("{}:{}", host, port); + let mut ftp_stream = suppaftp::FtpStream::connect(addr.as_str()) + .with_context(|| format!("Failed to connect to FTP host: {}", addr))?; + let user = if parsed_url.username().is_empty() { + "anonymous" + } else { + parsed_url.username() + }; + let pass = parsed_url.password().unwrap_or("anonymous@"); + ftp_stream + .login(user, pass) + .with_context(|| format!("FTP login failed for {}", host))?; + + let path = parsed_url.path(); + let candidates = [path.to_string(), path.trim_start_matches('/').to_string()]; + let mut retrieved = false; + for candidate in candidates.iter().filter(|path| !path.is_empty()) { + match ftp_stream.retr( + candidate, + |reader: &mut dyn Read| -> std::result::Result<(), suppaftp::FtpError> { + let mut file = + File::create(dest_path).map_err(suppaftp::FtpError::ConnectionError)?; + let mut buffer = [0u8; 8192]; + let mut downloaded = 0u64; + loop { + let bytes_read = reader + .read(&mut buffer) + .map_err(suppaftp::FtpError::ConnectionError)?; + if bytes_read == 0 { + break; + } + file.write_all(&buffer[..bytes_read]) + .map_err(suppaftp::FtpError::ConnectionError)?; + downloaded += bytes_read as u64; + pb.set_position(downloaded); + } + Ok(()) + }, + ) { + Ok(_) => { + retrieved = true; + break; + } + Err(_) => continue, + } + } + ftp_stream.quit().ok(); + if !retrieved { + bail!("FTP error fetching {}", parsed_url); + } + Ok(()) +} + +fn download_http_archive(url: &str, dest_path: &Path, pb: &ProgressBar) -> Result<()> { + let ua = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); + let mut last_err = None; + + for attempt in 1..=HTTP_FETCH_RETRY_LIMIT { + match download_http_archive_once(url, dest_path, pb, &ua) { + Ok(()) => return Ok(()), + Err(err) if attempt < HTTP_FETCH_RETRY_LIMIT && is_transient_http_error(&err) => { + crate::log_info!( + "Fetch attempt {} for {} failed with a transient network error; retrying", + attempt, + url + ); + fs::remove_file(dest_path).ok(); + pb.set_position(0); + pb.set_length(0); + last_err = Some(err); + } + Err(err) => return Err(err), + } + } + + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("Failed to fetch: {}", url))) +} + +fn download_http_archive_once( + url: &str, + dest_path: &Path, + pb: &ProgressBar, + ua: &str, +) -> Result<()> { + let client = super::build_blocking_client(ua, Some(HTTP_FETCH_TIMEOUT)) + .with_context(|| "Failed to build HTTP client")?; + let mut response = client + .get(url) + .send() + .with_context(|| format!("Failed to fetch: {}", url))?; + let status = response.status(); + if !status.is_success() { + let mut preview_bytes = Vec::new(); + let _ = response.take(1024).read_to_end(&mut preview_bytes); + let preview = String::from_utf8_lossy(&preview_bytes); + bail!( + "HTTP error fetching {}: {}{}", + url, + status, + if preview.trim().is_empty() { + "".to_string() + } else { + format!(" — preview: {}", preview.trim()) + } + ); + } + + let total_size = response.content_length().unwrap_or(0); + pb.set_length(total_size); + + let mut file = File::create(dest_path) + .with_context(|| format!("Failed to create: {}", dest_path.display()))?; + let mut buffer = [0u8; 8192]; + let mut downloaded = 0u64; + loop { + let bytes_read = response.read(&mut buffer)?; + if bytes_read == 0 { + break; + } + file.write_all(&buffer[..bytes_read])?; + downloaded += bytes_read as u64; + pb.set_position(downloaded); + } + + Ok(()) +} + +fn is_transient_http_error(err: &anyhow::Error) -> bool { + err.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|inner| inner.is_timeout() || inner.is_connect() || inner.is_request()) + }) +} + /// Validate downloaded file's magic header to make sure it is the expected /// archive format (avoids saving HTML pages or other unexpected content). fn validate_downloaded_archive( @@ -647,6 +673,45 @@ mod tests { assert!(name.starts_with("source-") && name.ends_with(".download")); } + #[test] + fn musl_snapshot_candidates_prefer_known_gentoo_mirror() { + let candidates = archive_fetch_candidates(MUSL_1_2_6_SNAPSHOT_URL); + assert_eq!( + candidates, + vec![ + MUSL_1_2_6_GENTOO_MIRROR_URL.to_string(), + MUSL_1_2_6_SNAPSHOT_URL.to_string(), + "https://musl.libc.org/releases/musl-1.2.6.tar.gz".to_string(), + ] + ); + } + + #[test] + fn musl_release_candidates_prefer_known_gentoo_mirror() { + let candidates = archive_fetch_candidates(MUSL_1_2_6_RELEASE_URL); + assert_eq!( + candidates, + vec![ + MUSL_1_2_6_GENTOO_MIRROR_URL.to_string(), + MUSL_1_2_6_RELEASE_URL.to_string(), + ] + ); + } + + #[test] + fn musl_release_mirror_only_matches_snapshot_urls() { + assert_eq!( + musl_release_mirror_url( + "https://git.musl-libc.org/cgit/musl/snapshot/musl-1.2.5.tar.gz" + ), + Some("https://musl.libc.org/releases/musl-1.2.5.tar.gz".to_string()) + ); + assert_eq!( + musl_release_mirror_url("https://musl.libc.org/releases/musl-1.2.5.tar.gz"), + None + ); + } + #[test] fn sourceforge_html_no_link_falls_back_to_download_suffix() { use std::io::Write; diff --git a/src/staging/mod.rs b/src/staging/mod.rs index c1c5fce..640f189 100755 --- a/src/staging/mod.rs +++ b/src/staging/mod.rs @@ -19,9 +19,12 @@ pub const INTERNAL_DEPOT_DIR: &str = ".depot"; pub const INTERNAL_OUTPUTS_DIR: &str = ".depot/outputs"; fn is_info_dir_index_path(rel_path: &str) -> bool { - matches!(rel_path, "usr/info/dir" | "usr/share/info/dir") - || rel_path.starts_with("usr/info/dir.") + matches!( + rel_path, + "usr/info/dir" | "usr/share/info/dir" | "system/documentation/info/dir" + ) || rel_path.starts_with("usr/info/dir.") || rel_path.starts_with("usr/share/info/dir.") + || rel_path.starts_with("system/documentation/info/dir.") } fn is_purged_install_basename(rel_path: &str) -> bool { @@ -43,6 +46,8 @@ fn is_skipped_install_path(rel_path: &str) -> bool { || p == INTERNAL_DEPOT_DIR || p.strip_prefix(INTERNAL_DEPOT_DIR) .is_some_and(|rest| rest.starts_with('/')) + || p == "destdir" + || p.starts_with("destdir/") || p == "scripts" || p.starts_with("scripts/") || is_purged_payload_path(p) @@ -340,12 +345,26 @@ fn move_tree_preserving_layout(src: &Path, dst: &Path) -> Result<()> { fs::create_dir_all(parent) .with_context(|| format!("Failed to create {}", parent.display()))?; } - if dst.symlink_metadata().is_ok() { - anyhow::bail!( - "Failed to move {} into {}: destination already exists", - src.display(), - dst.display() - ); + match dst.symlink_metadata() { + Ok(dst_metadata) + if duplicate_staged_path_is_equivalent(src, &metadata, dst, &dst_metadata)? => + { + fs::remove_file(src).with_context(|| { + format!("Failed to remove duplicate staged path {}", src.display()) + })?; + return Ok(()); + } + Ok(_) => { + anyhow::bail!( + "Failed to move {} into {}: destination already exists", + src.display(), + dst.display() + ); + } + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => { + return Err(err).with_context(|| format!("Failed to inspect {}", dst.display())); + } } fs::rename(src, dst).with_context(|| { format!( @@ -359,6 +378,59 @@ fn move_tree_preserving_layout(src: &Path, dst: &Path) -> Result<()> { Ok(()) } +fn duplicate_staged_path_is_equivalent( + src: &Path, + src_metadata: &fs::Metadata, + dst: &Path, + dst_metadata: &fs::Metadata, +) -> Result { + let src_type = src_metadata.file_type(); + let dst_type = dst_metadata.file_type(); + + if src_type.is_symlink() || dst_type.is_symlink() { + if !(src_type.is_symlink() && dst_type.is_symlink()) { + return Ok(false); + } + let src_target = fs::read_link(src) + .with_context(|| format!("Failed to read symlink {}", src.display()))?; + let dst_target = fs::read_link(dst) + .with_context(|| format!("Failed to read symlink {}", dst.display()))?; + return Ok(src_target == dst_target); + } + + if !src_metadata.is_file() || !dst_metadata.is_file() { + return Ok(false); + } + if src_metadata.len() != dst_metadata.len() { + return Ok(false); + } + + files_have_same_contents(src, dst) +} + +fn files_have_same_contents(left: &Path, right: &Path) -> Result { + let mut left = fs::File::open(left) + .with_context(|| format!("Failed to open staged file {}", left.display()))?; + let mut right = fs::File::open(right) + .with_context(|| format!("Failed to open staged file {}", right.display()))?; + let mut left_buf = [0u8; 8192]; + let mut right_buf = [0u8; 8192]; + + loop { + let left_read = left.read(&mut left_buf)?; + let right_read = right.read(&mut right_buf)?; + if left_read != right_read { + return Ok(false); + } + if left_read == 0 { + return Ok(true); + } + if left_buf[..left_read] != right_buf[..right_read] { + return Ok(false); + } + } +} + fn split_docs_for_output( output_destdir: &Path, docs_destdir: &Path, @@ -829,9 +901,182 @@ pub fn process(destdir: &Path, spec: &PackageSpec) -> Result<()> { ); } + let normalized = normalize_lbi_layout(destdir)?; + if normalized > 0 { + crate::log_info!("Normalized {} path(s) into the /system layout", normalized); + } + Ok(()) } +fn normalize_lbi_layout(destdir: &Path) -> Result { + let mut roots = vec![destdir.to_path_buf()]; + let outputs = output_staging_root(destdir); + if outputs.exists() { + let mut output_dirs = fs::read_dir(&outputs) + .with_context(|| format!("Failed to read {}", outputs.display()))? + .collect::, _>>() + .with_context(|| { + format!( + "Failed to read output staging entry from {}", + outputs.display() + ) + })?; + output_dirs.sort_by_key(|entry| entry.file_name()); + for entry in output_dirs { + let path = entry.path(); + if path.is_dir() { + roots.push(path); + } + } + } + + let mut changed = 0usize; + for root in roots { + changed += normalize_lbi_tree_paths(&root)?; + changed += rewrite_lbi_pkgconfig_files(&root)?; + } + Ok(changed) +} + +fn normalize_lbi_tree_paths(root: &Path) -> Result { + let mappings = [ + ("usr/share/man", "system/documentation/man-pages"), + ("usr/share/info", "system/documentation/info"), + ("usr/bin", "system/binaries"), + ("usr/sbin", "system/systembinaries"), + ("usr/lib64", "system/libraries"), + ("usr/lib", "system/libraries"), + ("usr/include", "system/headers"), + ("usr/share", "system/share"), + ("bin", "system/binaries"), + ("sbin", "system/systembinaries"), + ("lib64", "system/libraries"), + ("lib", "system/libraries"), + ("include", "system/headers"), + ("etc", "system/configuration"), + ("var", "system/variable"), + ("system/bin", "system/binaries"), + ("system/sbin", "system/systembinaries"), + ("system/lib64", "system/libraries"), + ("system/lib", "system/libraries"), + ("system/include", "system/headers"), + ("system/share/man", "system/documentation/man-pages"), + ("system/share/info", "system/documentation/info"), + ]; + + let mut changed = 0usize; + for (from, to) in mappings { + let src = root.join(from); + if !src.exists() { + continue; + } + let dst = root.join(to); + move_lbi_path(&src, &dst)?; + changed += 1; + } + prune_empty_dirs(root, &["usr", "system"])?; + Ok(changed) +} + +fn move_lbi_path(src: &Path, dst: &Path) -> Result<()> { + if src == dst { + return Ok(()); + } + if let Some(parent) = dst.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + if !dst.exists() { + fs::rename(src, dst) + .with_context(|| format!("Failed to move {} to {}", src.display(), dst.display()))?; + return Ok(()); + } + + let src_meta = fs::symlink_metadata(src) + .with_context(|| format!("Failed to inspect {}", src.display()))?; + let dst_meta = fs::symlink_metadata(dst) + .with_context(|| format!("Failed to inspect {}", dst.display()))?; + if src_meta.is_dir() && dst_meta.is_dir() { + move_directory_contents(src, dst)?; + return Ok(()); + } + + anyhow::bail!( + "Refusing to overwrite existing path while normalizing /system layout: {}", + dst.display() + ); +} + +fn prune_empty_dirs(root: &Path, dirs: &[&str]) -> Result<()> { + for dir in dirs { + let path = root.join(dir); + if path.exists() && path.is_dir() && is_directory_empty(&path)? { + fs::remove_dir(&path) + .with_context(|| format!("Failed to remove empty directory {}", path.display()))?; + } + } + Ok(()) +} + +fn rewrite_lbi_pkgconfig_files(root: &Path) -> Result { + let mut changed = 0usize; + if !root.exists() { + return Ok(0); + } + + for entry in WalkDir::new(root).follow_links(false) { + let entry = entry.with_context(|| format!("Failed to walk {}", root.display()))?; + if !entry.file_type().is_file() { + continue; + } + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("pc") { + continue; + } + let Ok(original) = fs::read_to_string(path) else { + continue; + }; + let rewritten = rewrite_lbi_pkgconfig_text(&original); + if rewritten != original { + fs::write(path, rewritten) + .with_context(|| format!("Failed to rewrite pkg-config file {}", path.display()))?; + changed += 1; + } + } + Ok(changed) +} + +fn rewrite_lbi_pkgconfig_text(input: &str) -> String { + input + .replace("prefix=/usr", "prefix=/system") + .replace("exec_prefix=/usr", "exec_prefix=/system") + .replace("bindir=${prefix}/bin", "bindir=${prefix}/binaries") + .replace("sbindir=${prefix}/sbin", "sbindir=${prefix}/systembinaries") + .replace("libdir=${prefix}/lib64", "libdir=${prefix}/libraries") + .replace("libdir=${prefix}/lib", "libdir=${prefix}/libraries") + .replace( + "includedir=${prefix}/include", + "includedir=${prefix}/headers", + ) + .replace( + "mandir=${prefix}/share/man", + "mandir=${prefix}/documentation/man-pages", + ) + .replace( + "infodir=${prefix}/share/info", + "infodir=${prefix}/documentation/info", + ) + .replace("/usr/sbin", "/system/systembinaries") + .replace("/usr/bin", "/system/binaries") + .replace("/usr/lib64", "/system/libraries") + .replace("/usr/lib", "/system/libraries") + .replace("/usr/include", "/system/headers") + .replace("/usr/share/man", "/system/documentation/man-pages") + .replace("/usr/share/info", "/system/documentation/info") + .replace("/usr/share", "/system/share") +} + /// Copy license files into the staged tree. /// /// Copies common license file patterns from the source directory root into: @@ -1539,9 +1784,9 @@ mod tests { let spec = mk_spec_for_stage_processing(); process(&destdir, &spec).unwrap(); - assert!(!destdir.join("usr/lib/libfoo.a").exists()); - assert!(!destdir.join("usr/lib/libfoo.la").exists()); - assert!(destdir.join("usr/lib/libfoo.so").exists()); + assert!(!destdir.join("system/libraries/libfoo.a").exists()); + assert!(!destdir.join("system/libraries/libfoo.la").exists()); + assert!(destdir.join("system/libraries/libfoo.so").exists()); } #[test] @@ -1556,8 +1801,8 @@ mod tests { spec.build.flags.no_delete_static = true; process(&destdir, &spec).unwrap(); - assert!(destdir.join("usr/lib/libfoo.a").exists()); - assert!(!destdir.join("usr/lib/libfoo.la").exists()); + assert!(destdir.join("system/libraries/libfoo.a").exists()); + assert!(!destdir.join("system/libraries/libfoo.la").exists()); } #[test] @@ -1580,18 +1825,18 @@ mod tests { process(&destdir, &spec).unwrap(); let docs_destdir = output_staging_dir(&destdir, "foo-docs"); - assert!(docs_destdir.join("usr/share/doc/foo/README").exists()); + assert!(docs_destdir.join("system/share/doc/foo/README").exists()); assert!( docs_destdir - .join("usr/share/gtk-doc/html/foo/index.html") + .join("system/share/gtk-doc/html/foo/index.html") .exists() ); assert!(docs_destdir.join("opt/foo-docs/guide.txt").exists()); - assert!(destdir.join("usr/bin/foo").exists()); - assert!(!destdir.join("usr/share/doc/foo/README").exists()); + assert!(destdir.join("system/binaries/foo").exists()); + assert!(!destdir.join("system/share/doc/foo/README").exists()); assert!( !destdir - .join("usr/share/gtk-doc/html/foo/index.html") + .join("system/share/gtk-doc/html/foo/index.html") .exists() ); assert!(!destdir.join("opt/foo-docs/guide.txt").exists()); @@ -1623,9 +1868,129 @@ mod tests { process(&destdir, &spec).unwrap(); let docs_destdir = output_staging_dir(&destdir, "foo-dev-docs"); - assert!(docs_destdir.join("usr/share/doc/foo-dev/README").exists()); - assert!(dev_destdir.join("usr/include/foo.h").exists()); - assert!(!dev_destdir.join("usr/share/doc/foo-dev/README").exists()); + assert!( + docs_destdir + .join("system/share/doc/foo-dev/README") + .exists() + ); + assert!(dev_destdir.join("system/headers/foo.h").exists()); + assert!(!dev_destdir.join("system/share/doc/foo-dev/README").exists()); + } + + #[test] + fn process_normalizes_usr_paths_into_system_layout() { + let tmp = tempfile::tempdir().unwrap(); + let destdir = tmp.path().join("dest"); + std::fs::create_dir_all(destdir.join("usr/bin")).unwrap(); + std::fs::create_dir_all(destdir.join("usr/lib/pkgconfig")).unwrap(); + std::fs::create_dir_all(destdir.join("usr/include/foo")).unwrap(); + std::fs::create_dir_all(destdir.join("usr/share/man/man1")).unwrap(); + std::fs::create_dir_all(destdir.join("usr/share/info")).unwrap(); + std::fs::write(destdir.join("usr/bin/foo"), "bin").unwrap(); + std::fs::write(destdir.join("usr/share/man/man1/foo.1"), "man").unwrap(); + std::fs::write(destdir.join("usr/share/info/foo.info"), "info").unwrap(); + std::fs::write( + destdir.join("usr/lib/pkgconfig/foo.pc"), + "prefix=/usr\nbindir=${prefix}/bin\nlibdir=${prefix}/lib\nincludedir=${prefix}/include\nmandir=${prefix}/share/man\ninfodir=${prefix}/share/info\n", + ) + .unwrap(); + + let spec = mk_spec_for_stage_processing(); + process(&destdir, &spec).unwrap(); + + assert!(destdir.join("system/binaries/foo").exists()); + assert!(destdir.join("system/headers/foo").is_dir()); + assert!( + destdir + .join("system/documentation/man-pages/man1/foo.1") + .exists() + ); + assert!(destdir.join("system/documentation/info/foo.info").exists()); + let pc = + std::fs::read_to_string(destdir.join("system/libraries/pkgconfig/foo.pc")).unwrap(); + assert!(pc.contains("prefix=/system")); + assert!(pc.contains("bindir=${prefix}/binaries")); + assert!(pc.contains("libdir=${prefix}/libraries")); + assert!(pc.contains("includedir=${prefix}/headers")); + assert!(pc.contains("mandir=${prefix}/documentation/man-pages")); + assert!(pc.contains("infodir=${prefix}/documentation/info")); + assert!(!destdir.join("usr").exists()); + } + + #[test] + fn process_normalizes_duplicate_identical_lbi_paths() { + let tmp = tempfile::tempdir().unwrap(); + let destdir = tmp.path().join("dest"); + std::fs::create_dir_all(destdir.join("system/lib/clang/22/include")).unwrap(); + std::fs::create_dir_all(destdir.join("system/libraries/clang/22/include")).unwrap(); + std::fs::write( + destdir.join("system/lib/clang/22/include/builtins.h"), + "same header\n", + ) + .unwrap(); + std::fs::write( + destdir.join("system/libraries/clang/22/include/builtins.h"), + "same header\n", + ) + .unwrap(); + + let spec = mk_spec_for_stage_processing(); + process(&destdir, &spec).unwrap(); + + assert!( + !destdir + .join("system/lib/clang/22/include/builtins.h") + .exists() + ); + assert_eq!( + std::fs::read_to_string(destdir.join("system/libraries/clang/22/include/builtins.h")) + .unwrap(), + "same header\n" + ); + } + + #[test] + fn process_rejects_conflicting_lbi_normalized_paths() { + let tmp = tempfile::tempdir().unwrap(); + let destdir = tmp.path().join("dest"); + std::fs::create_dir_all(destdir.join("system/lib/clang/22/include")).unwrap(); + std::fs::create_dir_all(destdir.join("system/libraries/clang/22/include")).unwrap(); + std::fs::write( + destdir.join("system/lib/clang/22/include/builtins.h"), + "left header\n", + ) + .unwrap(); + std::fs::write( + destdir.join("system/libraries/clang/22/include/builtins.h"), + "right header\n", + ) + .unwrap(); + + let spec = mk_spec_for_stage_processing(); + let err = process(&destdir, &spec).unwrap_err(); + + assert!( + err.to_string().contains("destination already exists"), + "{err:#}" + ); + } + + #[test] + fn process_normalizes_split_output_usr_paths() { + let tmp = tempfile::tempdir().unwrap(); + let destdir = tmp.path().join("dest"); + let out = destdir.join(".depot/outputs/foo-devel/usr/lib/pkgconfig"); + std::fs::create_dir_all(&out).unwrap(); + std::fs::write(out.join("foo.pc"), "prefix=/usr\nlibdir=/usr/lib\n").unwrap(); + + let spec = mk_spec_for_stage_processing(); + process(&destdir, &spec).unwrap(); + + let pc_path = destdir.join(".depot/outputs/foo-devel/system/libraries/pkgconfig/foo.pc"); + assert!(pc_path.exists()); + let pc = std::fs::read_to_string(pc_path).unwrap(); + assert!(pc.contains("prefix=/system")); + assert!(pc.contains("libdir=/system/libraries")); } #[test] @@ -2428,6 +2793,34 @@ mod tests { .contains(&".depot/outputs/clang/usr/bin/clang".to_string()) ); } + + #[test] + fn generate_manifest_skips_bootstrap_chroot_destdir_mountpoint() { + let tmp = tempfile::tempdir().unwrap(); + let destdir = tmp.path().join("dest"); + std::fs::create_dir_all(destdir.join("system/binaries")).unwrap(); + std::fs::create_dir_all(destdir.join("destdir/system/documentation/xz")).unwrap(); + std::fs::write(destdir.join("system/binaries/xz"), "ok").unwrap(); + std::fs::write( + destdir.join("destdir/system/documentation/xz/README"), + "stale staging", + ) + .unwrap(); + + let manifest = generate_manifest_with_dirs(&destdir).unwrap(); + + assert!(manifest.files.contains(&"system/binaries/xz".to_string())); + assert!( + !manifest + .files + .contains(&"destdir/system/documentation/xz/README".to_string()) + ); + assert!( + !manifest + .directories + .contains(&"destdir/system/documentation/xz".to_string()) + ); + } } /// Manifest containing files and directories for a package diff --git a/src/system_state.rs b/src/system_state.rs new file mode 100644 index 0000000..1efd1ad --- /dev/null +++ b/src/system_state.rs @@ -0,0 +1,504 @@ +use crate::config::Config; +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +const STATE_FILENAME: &str = "system.toml"; + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct SystemState { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) stage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) target: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) arch: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) layers: BTreeMap>, +} + +pub(crate) fn state_path(config: &Config) -> PathBuf { + config.db_dir.join(STATE_FILENAME) +} + +pub(crate) fn load(config: &Config) -> Result { + let path = state_path(config); + if !path.exists() { + return Ok(SystemState::default()); + } + let content = + fs::read_to_string(&path).with_context(|| format!("Failed to read {}", path.display()))?; + toml::from_str(&content).with_context(|| format!("Failed to parse {}", path.display())) +} + +pub(crate) fn save(config: &Config, state: &SystemState) -> Result { + let path = state_path(config); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + let content = toml::to_string_pretty(state).context("Failed to serialize system state")?; + fs::write(&path, content).with_context(|| format!("Failed to write {}", path.display()))?; + Ok(path) +} + +pub(crate) fn set_stage(config: &Config, stage: String) -> Result { + let stage = normalize_token("stage", &stage)?; + let mut state = load(config)?; + state.stage = Some(stage); + save(config, &state)?; + Ok(state) +} + +pub(crate) fn add_packages_to_layer( + config: &Config, + layer: String, + packages: &[String], +) -> Result { + let layer = normalize_token("layer", &layer)?; + let mut state = load(config)?; + let existing = state.layers.remove(&layer).unwrap_or_default(); + let mut merged = existing.into_iter().collect::>(); + for package in packages { + merged.insert(normalize_token("package", package)?); + } + state.layers.insert(layer, merged.into_iter().collect()); + save(config, &state)?; + Ok(state) +} + +pub(crate) fn set_layer_packages( + config: &Config, + layer: String, + packages: &[String], +) -> Result { + let layer = normalize_token("layer", &layer)?; + let mut state = load(config)?; + let mut seen = BTreeSet::new(); + let mut normalized = Vec::new(); + for package in packages { + let package = normalize_token("package", package)?; + if seen.insert(package.clone()) { + normalized.push(package); + } + } + state.layers.insert(layer, normalized); + save(config, &state)?; + Ok(state) +} + +pub(crate) fn remove_packages_from_layer( + config: &Config, + layer: String, + packages: &[String], +) -> Result { + let layer = normalize_token("layer", &layer)?; + let remove = packages + .iter() + .map(|package| normalize_token("package", package)) + .collect::>>()?; + let mut state = load(config)?; + if let Some(existing) = state.layers.get_mut(&layer) { + existing.retain(|package| !remove.contains(package)); + if existing.is_empty() { + state.layers.remove(&layer); + } + } + save(config, &state)?; + Ok(state) +} + +pub(crate) fn init_lbi_layout( + rootfs: &Path, + config: &Config, + target: &str, + arch: Option<&str>, + force: bool, +) -> Result { + let target = normalize_token("target", target)?; + let arch = match arch { + Some(arch) => normalize_token("arch", arch)?, + None => crate::cross::target_arch_from_triple(&target).to_string(), + }; + + ensure_lbi_layout_paths(rootfs)?; + write_lbi_build_config(rootfs, &target, &arch, force)?; + + let mut state = load(config)?; + state.target = Some(target); + state.arch = Some(arch); + state.stage.get_or_insert_with(|| "layout".to_string()); + save(config, &state)?; + Ok(state) +} + +pub(crate) fn ensure_lbi_layout_paths(rootfs: &Path) -> Result<()> { + create_lbi_directories(rootfs)?; + create_lbi_links(rootfs)?; + Ok(()) +} + +fn normalize_token(kind: &str, value: &str) -> Result { + let trimmed = value.trim(); + if trimmed.is_empty() { + bail!("{kind} must not be empty"); + } + if trimmed.contains('/') || trimmed.contains('\0') { + bail!("{kind} must not contain '/' or NUL bytes: {trimmed}"); + } + Ok(trimmed.to_string()) +} + +fn create_lbi_directories(rootfs: &Path) -> Result<()> { + let dirs = [ + "system", + "system/configuration", + "system/binaries", + "system/systembinaries", + "system/libraries", + "system/headers", + "system/share", + "system/documentation", + "system/documentation/man-pages", + "system/documentation/info", + "system/tools", + "system/variable", + "system/variable/lib", + "system/users", + "system/charlie", + "system/devices", + "system/devices/pts", + "system/devices/shm", + "system/processes", + "system/run", + "system/system", + "system/temporary", + "usr", + ]; + + for dir in dirs { + let path = rootfs.join(dir); + fs::create_dir_all(&path) + .with_context(|| format!("Failed to create {}", path.display()))?; + } + Ok(()) +} + +fn create_lbi_links(rootfs: &Path) -> Result<()> { + let links = [ + ("etc", "system/configuration"), + ("bin", "system/binaries"), + ("sbin", "system/systembinaries"), + ("lib", "system/libraries"), + ("var", "system/variable"), + ("home", "system/users"), + ("root", "system/charlie"), + ("dev", "system/devices"), + ("proc", "system/processes"), + ("run", "system/run"), + ("sys", "system/system"), + ("usr/bin", "../system/binaries"), + ("usr/sbin", "../system/systembinaries"), + ("usr/lib", "../system/libraries"), + ("usr/include", "../system/headers"), + ("usr/share", "../system/share"), + ("system/lib", "libraries"), + ]; + + for (link, target) in links { + ensure_relative_symlink(rootfs, Path::new(link), Path::new(target))?; + } + Ok(()) +} + +#[cfg(unix)] +fn ensure_relative_symlink(rootfs: &Path, link: &Path, target: &Path) -> Result<()> { + use std::os::unix::fs as unix_fs; + + let link_path = rootfs.join(link); + if let Ok(metadata) = fs::symlink_metadata(&link_path) { + if !metadata.file_type().is_symlink() { + if metadata.is_dir() { + let target_path = link_path + .parent() + .unwrap_or(rootfs) + .join(target) + .components() + .collect::(); + merge_dir_contents(&link_path, &target_path).with_context(|| { + format!( + "Failed to merge existing directory {} into {}", + link_path.display(), + target_path.display() + ) + })?; + fs::remove_dir(&link_path) + .with_context(|| format!("Failed to remove {}", link_path.display()))?; + } else { + bail!( + "Refusing to replace non-directory path while creating LBI layout: {}", + link_path.display() + ); + } + } else { + let existing = fs::read_link(&link_path) + .with_context(|| format!("Failed to read symlink {}", link_path.display()))?; + if existing == target { + return Ok(()); + } + fs::remove_file(&link_path) + .with_context(|| format!("Failed to replace symlink {}", link_path.display()))?; + } + } + if let Some(parent) = link_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + unix_fs::symlink(target, &link_path) + .with_context(|| format!("Failed to create symlink {}", link_path.display()))?; + Ok(()) +} + +fn merge_dir_contents(src: &Path, dest: &Path) -> Result<()> { + fs::create_dir_all(dest).with_context(|| format!("Failed to create {}", dest.display()))?; + let mut entries = fs::read_dir(src) + .with_context(|| format!("Failed to read {}", src.display()))? + .collect::, _>>() + .with_context(|| format!("Failed to read entry from {}", src.display()))?; + entries.sort_by_key(|entry| entry.file_name()); + + for entry in entries { + let source_path = entry.path(); + let dest_path = dest.join(entry.file_name()); + if dest_path.exists() { + let source_type = entry + .file_type() + .with_context(|| format!("Failed to inspect {}", source_path.display()))?; + let dest_type = fs::symlink_metadata(&dest_path) + .with_context(|| format!("Failed to inspect {}", dest_path.display()))? + .file_type(); + if source_type.is_dir() && dest_type.is_dir() { + merge_dir_contents(&source_path, &dest_path)?; + fs::remove_dir(&source_path) + .with_context(|| format!("Failed to remove {}", source_path.display()))?; + continue; + } + bail!( + "Refusing to overwrite existing path while merging LBI directory: {}", + dest_path.display() + ); + } + fs::rename(&source_path, &dest_path).with_context(|| { + format!( + "Failed to move {} to {}", + source_path.display(), + dest_path.display() + ) + })?; + } + Ok(()) +} + +#[cfg(not(unix))] +fn ensure_relative_symlink(_rootfs: &Path, _link: &Path, _target: &Path) -> Result<()> { + bail!("LBI layout initialization requires Unix symlink support") +} + +fn write_lbi_build_config(rootfs: &Path, target: &str, arch: &str, force: bool) -> Result { + let path = rootfs.join("etc/depot.d/build.toml"); + if path.exists() && !force { + bail!( + "{} already exists; re-run with --force to replace it", + path.display() + ); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("Failed to create {}", parent.display()))?; + } + let content = lbi_build_config_toml(target, arch); + fs::write(&path, content).with_context(|| format!("Failed to write {}", path.display()))?; + Ok(path) +} + +fn lbi_build_config_toml(target: &str, arch: &str) -> String { + let makeflags = format!( + "-j{}", + std::thread::available_parallelism() + .map(|parallelism| parallelism.get()) + .unwrap_or(1) + ); + format!( + r#"# Generated by `depot system init-lbi`. +# These defaults mirror the Linux by Intent /system layout. + +[flags] +prefix = "/system" +bindir = "/system/binaries" +sbindir = "/system/systembinaries" +libdir = "/system/libraries" +libexecdir = "/system/libraries" +sysconfdir = "/system/configuration" +localstatedir = "/system/variable" +sharedstatedir = "/system/variable/lib" +includedir = "/system/headers" +datarootdir = "/system/share" +datadir = "/system/share" +mandir = "/system/documentation/man-pages" +infodir = "/system/documentation/info" +cc = "clang" +cxx = "clang++" +ar = "llvm-ar" +ld = "ld.lld" +carch = "{arch}" +chost = "{target}" +target = "{target}" +cflags = ["-O2", "-pipe"] +cxxflags = ["-O2", "-pipe"] +ldflags = ["-Wl,-rpath,/system/libraries"] +makeflags = "{makeflags}" +"#, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn layer_add_sorts_and_deduplicates_packages() { + let tmp = tempfile::tempdir().unwrap(); + let config = Config::for_rootfs(tmp.path()); + let state = add_packages_to_layer( + &config, + "base".to_string(), + &["zlib".to_string(), "musl".to_string(), "zlib".to_string()], + ) + .unwrap(); + assert_eq!(state.layers["base"], vec!["musl", "zlib"]); + } + + #[test] + fn set_layer_packages_replaces_only_named_layer() { + let tmp = tempfile::tempdir().unwrap(); + let config = Config::for_rootfs(tmp.path()); + add_packages_to_layer(&config, "custom".to_string(), &["kept".to_string()]).unwrap(); + add_packages_to_layer( + &config, + "base".to_string(), + &["old".to_string(), "zlib".to_string()], + ) + .unwrap(); + let state = set_layer_packages( + &config, + "base".to_string(), + &["zlib".to_string(), "musl".to_string(), "zlib".to_string()], + ) + .unwrap(); + assert_eq!(state.layers["base"], vec!["zlib", "musl"]); + assert_eq!(state.layers["custom"], vec!["kept"]); + } + + #[test] + fn stage_is_persisted() { + let tmp = tempfile::tempdir().unwrap(); + let config = Config::for_rootfs(tmp.path()); + set_stage(&config, "cross-tools".to_string()).unwrap(); + let loaded = load(&config).unwrap(); + assert_eq!(loaded.stage.as_deref(), Some("cross-tools")); + } + + #[cfg(unix)] + #[test] + fn init_lbi_creates_layout_and_build_defaults() { + let tmp = tempfile::tempdir().unwrap(); + let config = Config::for_rootfs(tmp.path()); + let state = init_lbi_layout( + tmp.path(), + &config, + "x86_64-unknown-linux-musl", + None, + false, + ) + .unwrap(); + assert_eq!(state.stage.as_deref(), Some("layout")); + assert_eq!(state.arch.as_deref(), Some("x86_64")); + assert!(tmp.path().join("system/binaries").is_dir()); + assert!(tmp.path().join("system/devices/pts").is_dir()); + assert!(tmp.path().join("system/devices/shm").is_dir()); + assert_eq!( + fs::read_link(tmp.path().join("usr/bin")).unwrap(), + PathBuf::from("../system/binaries") + ); + assert_eq!( + fs::read_link(tmp.path().join("dev")).unwrap(), + PathBuf::from("system/devices") + ); + assert_eq!( + fs::read_link(tmp.path().join("proc")).unwrap(), + PathBuf::from("system/processes") + ); + assert_eq!( + fs::read_link(tmp.path().join("sys")).unwrap(), + PathBuf::from("system/system") + ); + assert_eq!( + fs::read_link(tmp.path().join("run")).unwrap(), + PathBuf::from("system/run") + ); + let build_toml = fs::read_to_string(tmp.path().join("etc/depot.d/build.toml")).unwrap(); + assert!(build_toml.contains("prefix = \"/system\"")); + assert!(build_toml.contains("chost = \"x86_64-unknown-linux-musl\"")); + let makeflags_line = build_toml + .lines() + .find(|line| line.trim_start().starts_with("makeflags = \"-j")) + .expect("expected makeflags default in generated build.toml"); + let jobs = makeflags_line + .trim() + .trim_start_matches("makeflags = \"-j") + .trim_end_matches('"') + .parse::() + .expect("expected numeric makeflags job count"); + assert!(jobs >= 1); + } + + #[cfg(unix)] + #[test] + fn ensure_lbi_layout_paths_reconciles_existing_usr_include_directory() { + let tmp = tempfile::tempdir().unwrap(); + fs::create_dir_all(tmp.path().join("usr/include")).unwrap(); + fs::write(tmp.path().join("usr/include/marker.h"), "/* marker */").unwrap(); + + ensure_lbi_layout_paths(tmp.path()).unwrap(); + + assert_eq!( + fs::read_link(tmp.path().join("usr/include")).unwrap(), + PathBuf::from("../system/headers") + ); + assert!(tmp.path().join("system/headers/marker.h").exists()); + } + + #[cfg(unix)] + #[test] + fn init_lbi_migrates_existing_var_contents_before_linking() { + let tmp = tempfile::tempdir().unwrap(); + fs::create_dir_all(tmp.path().join("var/lib/depot")).unwrap(); + fs::write(tmp.path().join("var/lib/depot/lock"), "").unwrap(); + let config = Config::for_rootfs(tmp.path()); + init_lbi_layout( + tmp.path(), + &config, + "x86_64-unknown-linux-musl", + None, + false, + ) + .unwrap(); + assert_eq!( + fs::read_link(tmp.path().join("var")).unwrap(), + PathBuf::from("system/variable") + ); + assert!(tmp.path().join("system/variable/lib/depot/lock").exists()); + } +} diff --git a/src/ui.rs b/src/ui.rs index 0c26dee..f0ae8e0 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -50,6 +50,16 @@ pub fn success(message: impl AsRef) { println!("{} {}", label(Stream::Stdout, "OK", "32"), message.as_ref()); } +pub fn merge_package(layer: &str, package: &str) { + println!( + "{} {} {} into layer {}", + paint(Stream::Stdout, ">>>", "32;1"), + paint(Stream::Stdout, "merging package", "36;1"), + paint(Stream::Stdout, package, "32;1"), + layer + ); +} + pub fn warn(message: impl AsRef) { eprintln!( "{} {}",