diff --git a/Cargo.toml b/Cargo.toml index c5456eb..320f71a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,9 @@ tar = "0.4" flate2 = "1" sha2 = "0.10" hex = "0.4" +# The macOS OpenSSL libraries come from a conda-forge .conda package, a zip +# archive wrapping a zstd-compressed tar. +zip = { version = "2", default-features = false } [profile.release] strip = true diff --git a/build.rs b/build.rs index b676a8e..ad9f9de 100644 --- a/build.rs +++ b/build.rs @@ -42,12 +42,12 @@ fn main() { let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - bundle_postgresql(&pg_version, &out_dir); + bundle_postgresql(&pg_version, &versions, &out_dir); bundle_pgvector(&pg_version, &pgvector_tag, &pgvector_repo, &out_dir); bundle_runtime_libs(&versions, &out_dir); } -fn bundle_postgresql(pg_version: &str, out_dir: &PathBuf) { +fn bundle_postgresql(pg_version: &str, versions: &HashMap, out_dir: &PathBuf) { let target = env::var("TARGET").unwrap(); // Map Rust target to theseus-rs binary name @@ -100,6 +100,13 @@ fn bundle_postgresql(pg_version: &str, out_dir: &PathBuf) { eprintln!("Using cached PostgreSQL bundle: {}", bundle_path.display()); } + let bundle_path = match target.as_str() { + "aarch64-apple-darwin" | "x86_64-apple-darwin" => { + bundle_macos_openssl(&bundle_path, versions, out_dir) + } + _ => bundle_path, + }; + println!( "cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}", bundle_path.display() @@ -375,3 +382,255 @@ fn write_tar_gz(out_path: &Path, entries: &[(String, Vec)]) -> io::Result<() gz.finish()?.flush()?; Ok(()) } + +/// The OpenSSL libraries the theseus-rs macOS build links against. +const MACOS_OPENSSL_LIBS: [&str; 2] = ["libssl.3.dylib", "libcrypto.3.dylib"]; + +/// Make the theseus-rs macOS PostgreSQL bundle self-contained. +/// +/// The theseus-rs macOS binaries (postgres, pg_dump, libpq, pgcrypto, ...) +/// load OpenSSL from the Homebrew prefix of the machine that built them +/// (`/opt/homebrew/opt/openssl@3/lib/...`), so PostgreSQL cannot start on a +/// Mac without Homebrew's openssl@3 (theseus-rs/postgresql-binaries#30). +/// +/// We do what Postgres.app does: ship libssl/libcrypto inside the bundle's +/// `lib/` directory and point every Mach-O at them with an +/// `@loader_path`-relative install name, then re-sign ad-hoc because +/// install_name_tool invalidates the signature. OpenSSL comes from +/// conda-forge's pinned, checksummed package (versions.env), which has +/// builds for both macOS architectures with a macOS 11 deployment target. +/// +/// Returns the path of the repacked bundle; the build fails if any Mach-O in +/// it still references a library under /opt/homebrew or /usr/local. +fn bundle_macos_openssl( + upstream_bundle: &Path, + versions: &HashMap, + out_dir: &Path, +) -> PathBuf { + let target = env::var("TARGET").unwrap(); + let arch = if target.starts_with("aarch64") { + "ARM64" + } else { + "X86_64" + }; + let get = |k: String| -> String { + versions + .get(&k) + .unwrap_or_else(|| panic!("Missing {} in versions.env", k)) + .clone() + }; + let openssl_version = get("OPENSSL_CONDA_VERSION".to_string()); + let url = get(format!("OPENSSL_CONDA_URL_{}", arch)); + let sha256 = get(format!("OPENSSL_CONDA_SHA256_{}", arch)); + + let stem = upstream_bundle + .file_name() + .and_then(|s| s.to_str()) + .and_then(|s| s.strip_suffix(".tar.gz")) + .expect("PostgreSQL bundle name must end in .tar.gz"); + let portable = out_dir.join(format!("{}-openssl-{}.tar.gz", stem, openssl_version)); + if portable.exists() { + eprintln!( + "Using cached portable PostgreSQL bundle: {}", + portable.display() + ); + return portable; + } + + let conda_path = out_dir.join(format!("{}.conda", sha256_short(&url))); + if !conda_path.exists() { + eprintln!("Downloading {}...", url); + download_file(&url, &conda_path.to_path_buf()).expect("Failed to download OpenSSL package"); + } + verify_sha256(&conda_path, &sha256); + + let staging = out_dir.join("macos-portable"); + if staging.exists() { + fs::remove_dir_all(&staging).expect("Failed to clear macOS staging directory"); + } + fs::create_dir_all(&staging).expect("Failed to create macOS staging directory"); + let upstream = File::open(upstream_bundle).expect("open PostgreSQL bundle"); + tar::Archive::new(flate2::read::GzDecoder::new(upstream)) + .unpack(&staging) + .expect("Failed to unpack PostgreSQL bundle"); + let root_name = fs::read_dir(&staging) + .expect("read staging directory") + .next() + .expect("PostgreSQL bundle is empty") + .expect("read staging entry") + .file_name(); + let root = staging.join(&root_name); + + for lib in MACOS_OPENSSL_LIBS { + let bytes = extract_from_conda(&conda_path, "pkg-", &format!("lib/{}", lib)) + .unwrap_or_else(|e| panic!("Failed to extract {} from {}: {}", lib, url, e)); + fs::write(root.join("lib").join(lib), bytes).expect("Failed to write OpenSSL library"); + } + let license = extract_from_conda(&conda_path, "info-", "info/licenses/LICENSE.txt") + .unwrap_or_else(|e| panic!("Failed to extract OpenSSL licence from {}: {}", url, e)); + let license_dir = root.join("share").join("openssl"); + fs::create_dir_all(&license_dir).expect("Failed to create OpenSSL licence directory"); + fs::write(license_dir.join("LICENSE.txt"), license).expect("Failed to write OpenSSL licence"); + + for path in mach_o_files(&root) { + // `@loader_path//lib/` from wherever this file sits. + let depth = path + .parent() + .unwrap() + .strip_prefix(&root) + .unwrap() + .components() + .count(); + let lib_ref = |lib: &str| format!("@loader_path/{}lib/{}", "../".repeat(depth), lib); + + let mut args: Vec = Vec::new(); + let name = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); + if MACOS_OPENSSL_LIBS.contains(&name) { + args.extend(["-id".to_string(), lib_ref(name)]); + } + for dep in mach_o_dependencies(&path) { + let dep_name = dep.rsplit('/').next().unwrap_or(""); + if MACOS_OPENSSL_LIBS.contains(&dep_name) && dep != lib_ref(dep_name) { + args.extend(["-change".to_string(), dep.clone(), lib_ref(dep_name)]); + } + } + if args.is_empty() { + continue; + } + run_tool("install_name_tool", &args, &path); + run_tool( + "codesign", + &["-f".to_string(), "-s".to_string(), "-".to_string()], + &path, + ); + } + + for path in mach_o_files(&root) { + for dep in mach_o_dependencies(&path) { + assert!( + !dep.starts_with("/opt/homebrew/") && !dep.starts_with("/usr/local/"), + "{} still references {} after bundling OpenSSL", + path.display(), + dep + ); + } + } + + let tmp = out_dir.join(format!( + "{}.partial", + portable.file_name().unwrap().to_str().unwrap() + )); + { + let file = File::create(&tmp).expect("create portable bundle"); + let gz = flate2::write::GzEncoder::new(file, flate2::Compression::default()); + let mut builder = tar::Builder::new(gz); + builder.follow_symlinks(false); + builder + .append_dir_all(&root_name, &root) + .expect("Failed to repack PostgreSQL bundle"); + builder + .into_inner() + .and_then(|gz| gz.finish()) + .and_then(|mut f| f.flush()) + .expect("Failed to write portable bundle"); + } + fs::rename(&tmp, &portable).expect("Failed to move portable bundle into place"); + eprintln!( + "Bundled OpenSSL {} into {}", + openssl_version, + portable.display() + ); + portable +} + +/// Read `wanted` from the zstd tar member whose name starts with `member` +/// (`pkg-` or `info-`) of a conda `.conda` package, which is a zip archive. +fn extract_from_conda(conda_path: &Path, member: &str, wanted: &str) -> io::Result> { + let mut zip = zip::ZipArchive::new(File::open(conda_path)?)?; + let index = (0..zip.len()) + .find(|&i| { + zip.by_index(i) + .map(|e| e.name().starts_with(member) && e.name().ends_with(".tar.zst")) + .unwrap_or(false) + }) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, format!("no {}*.tar.zst", member)) + })?; + let entry = zip.by_index(index)?; + let mut tar = tar::Archive::new(zstd::Decoder::new(entry)?); + for tentry in tar.entries()? { + let mut tentry = tentry?; + if tentry.path()?.to_str() == Some(wanted) { + let mut buf = Vec::new(); + tentry.read_to_end(&mut buf)?; + return Ok(buf); + } + } + Err(io::Error::new( + io::ErrorKind::NotFound, + format!("{} not found", wanted), + )) +} + +/// Every regular Mach-O file under `dir`, found by its magic number. +fn mach_o_files(dir: &Path) -> Vec { + let mut found = Vec::new(); + for entry in fs::read_dir(dir).expect("read bundle directory") { + let path = entry.expect("read bundle entry").path(); + let meta = fs::symlink_metadata(&path).expect("stat bundle entry"); + if meta.is_dir() { + found.extend(mach_o_files(&path)); + } else if meta.is_file() { + let mut magic = [0u8; 4]; + let is_mach_o = File::open(&path) + .and_then(|mut f| f.read_exact(&mut magic)) + .is_ok() + && matches!( + u32::from_be_bytes(magic), + 0xfeedface | 0xfeedfacf | 0xcefaedfe | 0xcffaedfe | 0xcafebabe + ); + if is_mach_o { + found.push(path); + } + } + } + found +} + +/// The install names a Mach-O file loads, as `otool -L` lists them (for a +/// dylib this includes its own install name). +fn mach_o_dependencies(path: &Path) -> Vec { + let output = std::process::Command::new("otool") + .arg("-L") + .arg(path) + .output() + .expect("Failed to run otool (install the Xcode Command Line Tools)"); + assert!( + output.status.success(), + "otool -L {} failed", + path.display() + ); + String::from_utf8_lossy(&output.stdout) + .lines() + .skip(1) + .filter_map(|line| line.trim().split(" (compatibility").next()) + .filter(|dep| !dep.is_empty()) + .map(str::to_string) + .collect() +} + +fn run_tool(tool: &str, args: &[String], path: &Path) { + let output = std::process::Command::new(tool) + .args(args) + .arg(path) + .output() + .unwrap_or_else(|e| panic!("Failed to run {}: {}", tool, e)); + assert!( + output.status.success(), + "{} {:?} {} failed: {}", + tool, + args, + path.display(), + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/src/main.rs b/src/main.rs index 617f36d..85a77ae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -492,6 +492,10 @@ fn extract_bundled_postgresql(installation_dir: &PathBuf, pg_version: &str) -> R // Check if already extracted let bin_dir = version_dir.join("bin"); let already_extracted = bin_dir.exists() && bin_dir.join(POSTGRES_BINARY).exists(); + // Earlier macOS builds shipped a bundle that loads OpenSSL from Homebrew; + // re-extract over it so the bundled OpenSSL replaces those binaries. + #[cfg(target_os = "macos")] + let already_extracted = already_extracted && version_dir.join("lib/libcrypto.3.dylib").exists(); if !already_extracted { if POSTGRESQL_BUNDLE.is_empty() { diff --git a/versions.env b/versions.env index e250ad8..2daf757 100644 --- a/versions.env +++ b/versions.env @@ -19,13 +19,29 @@ PGVECTOR_COMPILED_REPO=nicoloboschi/pgvector_compiled # This works on every supported distro: 22.04 has the same SONAMEs natively; # 24.04+ keeps libxml2.so.2 around but in a newer ICU lineage; 25.10+ has # neither, but the bundled libxml2.so.2 + libicu70 fills both gaps. -LIBXML2_DEB_VERSION=2.9.13+dfsg-1ubuntu0.12 -LIBXML2_DEB_URL_AMD64=https://archive.ubuntu.com/ubuntu/pool/main/libx/libxml2/libxml2_2.9.13+dfsg-1ubuntu0.12_amd64.deb -LIBXML2_DEB_SHA256_AMD64=b3678e6e4b166bc0e4226fb118d489ab51802c772914421397c9dcb2dd0e0d2b -LIBXML2_DEB_URL_ARM64=https://ports.ubuntu.com/ubuntu-ports/pool/main/libx/libxml2/libxml2_2.9.13+dfsg-1ubuntu0.12_arm64.deb -LIBXML2_DEB_SHA256_ARM64=6feb8502740e786e7d6a8d89e3d8d1060aaf6f27ad48bf279cb3c19b451eb16f +# Launchpad keeps every published .deb; the archive pool drops superseded +# point releases, which broke the previous pin. +LIBXML2_DEB_VERSION=2.9.13+dfsg-1ubuntu0.13 +LIBXML2_DEB_URL_AMD64=https://launchpad.net/ubuntu/+archive/primary/+files/libxml2_2.9.13+dfsg-1ubuntu0.13_amd64.deb +LIBXML2_DEB_SHA256_AMD64=825df4a2c852b4cec058ed3e95648f5c7b41f923af85e15d464ec80c14302edc +LIBXML2_DEB_URL_ARM64=https://launchpad.net/ubuntu/+archive/primary/+files/libxml2_2.9.13+dfsg-1ubuntu0.13_arm64.deb +LIBXML2_DEB_SHA256_ARM64=62c61132b49b893392fca7ae8e4889b181c55d071cd030d891275db5237e10d7 LIBICU_DEB_VERSION=70.1-2ubuntu1 LIBICU_DEB_URL_AMD64=http://archive.ubuntu.com/ubuntu/pool/main/i/icu/libicu70_70.1-2ubuntu1_amd64.deb LIBICU_DEB_SHA256_AMD64=223c1e6cebf2a6748637303227d8b84337c14a2d30eacbc46a68d02557dcff3e LIBICU_DEB_URL_ARM64=http://ports.ubuntu.com/pool/main/i/icu/libicu70_70.1-2ubuntu1_arm64.deb LIBICU_DEB_SHA256_ARM64=a06f86489e364f555a0655f476e6b2693760aff9d8dec0011daf9b17957da3c5 + +# OpenSSL bundled with the macOS binaries. +# +# The theseus-rs macOS PostgreSQL build loads libssl/libcrypto from +# /opt/homebrew/opt/openssl@3, so it cannot start on a Mac without Homebrew. +# build.rs copies these libraries into the bundle's lib/ and rewrites every +# reference to @loader_path. conda-forge is used because it publishes +# checksummed OpenSSL builds for both macOS architectures (deployment target +# macOS 11); 3.5 is OpenSSL's current LTS line. +OPENSSL_CONDA_VERSION=3.5.9 +OPENSSL_CONDA_URL_ARM64=https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.5.9-h55eecbc_0.conda +OPENSSL_CONDA_SHA256_ARM64=a26829f8d4ed2631216a8543ef7704d2cff90ce43e381015530ab21cf762b681 +OPENSSL_CONDA_URL_X86_64=https://conda.anaconda.org/conda-forge/osx-64/openssl-3.5.9-h332eb6d_0.conda +OPENSSL_CONDA_SHA256_X86_64=4747221abec91a82a8ebd7e8b36eac90c21c74435f9473c86c0e41a9305f2183