From f7069c9202c9214c3f005b49c9a1b7340d3c6cc2 Mon Sep 17 00:00:00 2001 From: Bret Mogilefsky Date: Thu, 29 Oct 2026 00:07:08 +0000 Subject: [PATCH] test(bats): add ACQ_BATS_SHARD to run one shard of the offline suite Add scripts/bats-shard, which selects shard I of N of test/bats/*.bats deterministically (LPT balancing by @test count, derived from the files on disk each run, so a new test file joins a shard automatically). It fails closed (exit 2) on a malformed or out-of-range spec, a missing directory, no .bats files, or an empty shard. scripts/test-acq-bats honors ACQ_BATS_SHARD=I/N on the GNU parallel, xargs fallback and serial paths; explicit file arguments still pass straight through. acq_setup_stubs clears ACQ_BATS_SHARD so a CI leg's shard spec cannot leak into the runner contract tests. Tests: test/bats/02-bats-shard.bats (partition completeness, determinism, add/remove stability, weighting, fail-closed specs) and runner tests in 01-test-runner.bats. Co-authored-by: OpenCode Agent --- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 8 ++ scripts/bats-shard | 93 +++++++++++++++++++++++ scripts/test-acq-bats | 39 ++++++++-- test/bats/01-test-runner.bats | 53 ++++++++++++++ test/bats/02-bats-shard.bats | 134 ++++++++++++++++++++++++++++++++++ test/bats/helper.bash | 2 +- 7 files changed, 324 insertions(+), 7 deletions(-) create mode 100755 scripts/bats-shard create mode 100644 test/bats/02-bats-shard.bats diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 580ddfa7..f4b77f01 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -111,7 +111,7 @@ repos: entry: bash scripts/test-acq-bats language: system pass_filenames: false - files: '^(acq|acq\.backends/.*|scripts/(test-acq-lib\.sh|test-acq-bats|run-shellcheck|probe-msb-node-startup|test-probe-msb-node-startup\.py|fix-exec-bits|verify-backends)|test/bats/.*)$' + files: '^(acq|acq\.backends/.*|scripts/(test-acq-lib\.sh|test-acq-bats|bats-shard|run-shellcheck|probe-msb-node-startup|test-probe-msb-node-startup\.py|fix-exec-bits|verify-backends)|test/bats/.*)$' - id: test-probe-msb-startup name: MSB startup probe offline tests diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2e35518..e36f5f0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,6 +81,7 @@ This project operates under professional standards of conduct. All contributors: | `install.sh` | One-line `curl \| sh` installer (auto-selects brew/npm/clone; direct `sh install.sh` still installs the default release tag from the repo, not the current checkout) | | `scripts/rotate-apikey` | Rotate your USAi API key secret (`acq usai-rotate-api-key`) | | `scripts/test-acq-bats` | Offline unit suite for acq (bats-core) | +| `scripts/bats-shard` | Picks the `test/bats` files for shard `I/N` (CI matrix; `ACQ_BATS_SHARD=I/N`) | | `scripts/verify-backends` | Live end-to-end backend verification (needs Docker or KVM) | | `.pre-commit-config.yaml` | Optional pre-commit hooks (secret detection, file hygiene) | | `AGENTS.md` | Rules for working **on this quickstart repo** | @@ -199,6 +200,13 @@ network), built on [bats-core](https://github.com/bats-core/bats-core) (ADR-0025 ./scripts/test-acq-bats --timing test/bats/90-sbx-startup-kit.bats ``` + To run only part of the suite (this is how the bash 3.2 CI job splits the work + across a matrix), set `ACQ_BATS_SHARD=I/N`, e.g. `ACQ_BATS_SHARD=2/3 + ./scripts/test-acq-bats`. `scripts/bats-shard` derives each shard from the + `.bats` files on disk (balanced by `@test` count; no checked-in table), so a new + test file joins a shard automatically, and an invalid spec or empty shard + fails closed with exit 2. + Timing mode reports the worker count and Bats' per-test elapsed time, including setup and teardown. Explicit file/flag arguments still pass directly to Bats; they do not implicitly enable parallelism. CI enables timing and verbose diff --git a/scripts/bats-shard b/scripts/bats-shard new file mode 100755 index 00000000..c7a9bfb7 --- /dev/null +++ b/scripts/bats-shard @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# +# bats-shard — print the .bats files that belong to shard I of N. +# +# Used by scripts/test-acq-bats (ACQ_BATS_SHARD=I/N) so a CI matrix can split +# the offline suite across runners. Selection is derived from the files on disk +# every time — there is no checked-in table that can go stale: a new .bats file +# lands in a shard automatically. +# +# Algorithm (deterministic, balanced): weight each file by its number of @test +# blocks (minimum 1; a good, free proxy for wall time), sort by weight +# descending then path ascending (C locale), and greedily assign each file to +# the currently lightest shard (lowest index wins ties). Every file is assigned +# to exactly one shard. Adding or removing a file may move other files between +# shards, which is harmless: every run re-derives the full partition. +# +# Fails closed (exit 2) on a malformed spec, I > N, a missing directory, no +# .bats files, or an empty shard — an empty shard would otherwise pass vacuously. +# +# Usage: +# scripts/bats-shard I/N [DIR] # DIR defaults to test/bats; one path per line +# +# Portable to bash 3.2 / macOS (no mapfile, no associative arrays, POSIX awk). +set -uo pipefail + +_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_spec="${1:-}" +_bats_dir="${2:-${_dir}/../test/bats}" + +case "$_spec" in + [1-9]*/[1-9]*) ;; + *) echo "bats-shard: spec must be I/N with positive integers (got '${_spec}')" >&2; exit 2 ;; +esac +_i="${_spec%%/*}" +_n="${_spec#*/}" +case "${_i}${_n}" in + *[!0-9]*) echo "bats-shard: spec must be I/N with positive integers (got '${_spec}')" >&2; exit 2 ;; +esac +if [ "${#_i}" -gt 4 ] || [ "${#_n}" -gt 4 ] || [ "$_i" -gt "$_n" ]; then + echo "bats-shard: need 1 <= I <= N <= 9999 (got '${_spec}')" >&2 + exit 2 +fi + +if [ ! -d "$_bats_dir" ]; then + echo "bats-shard: not a directory: ${_bats_dir}" >&2 + exit 2 +fi +_bats_dir="$(cd "$_bats_dir" && pwd)" + +export LC_ALL=C +_files=() +for _f in "${_bats_dir}"/*.bats; do + [ -f "$_f" ] && _files+=("$_f") +done +if [ "${#_files[@]}" -eq 0 ]; then + echo "bats-shard: no .bats files in ${_bats_dir}" >&2 + exit 2 +fi + +_picked="$(awk -v shard="$_i" -v total="$_n" ' + /^[ \t]*@test[ \t]/ { c[FILENAME]++ } + END { + nf = ARGC - 1 + for (k = 1; k <= nf; k++) { + name[k] = ARGV[k] "" + w[k] = (c[ARGV[k]] > 0) ? c[ARGV[k]] : 1 + ord[k] = k + } + for (a = 2; a <= nf; a++) { + cur = ord[a] + b = a - 1 + while (b >= 1 && (w[ord[b]] < w[cur] || (w[ord[b]] == w[cur] && name[ord[b]] > name[cur]))) { + ord[b + 1] = ord[b] + b-- + } + ord[b + 1] = cur + } + for (s = 1; s <= total; s++) load[s] = 0 + for (a = 1; a <= nf; a++) { + k = ord[a] + best = 1 + for (s = 2; s <= total; s++) if (load[s] < load[best]) best = s + load[best] += w[k] + if (best == shard) print name[k] + } + } +' "${_files[@]}")" || { echo "bats-shard: failed to read test files" >&2; exit 2; } + +if [ -z "$_picked" ]; then + echo "bats-shard: shard ${_i}/${_n} is empty (only ${#_files[@]} .bats files); refusing to pass vacuously" >&2 + exit 2 +fi +printf '%s\n' "$_picked" | sort diff --git a/scripts/test-acq-bats b/scripts/test-acq-bats index 10aacead..e45db1b3 100755 --- a/scripts/test-acq-bats +++ b/scripts/test-acq-bats @@ -30,6 +30,13 @@ # ./scripts/test-acq-bats test/bats/X.bats # a specific file (passed through) # ACQ_BATS_JOBS=1 ./scripts/test-acq-bats # force serial # ACQ_BATS_TIMING=1 ./scripts/test-acq-bats # per-test timings, also in fallback +# ACQ_BATS_SHARD=1/2 ./scripts/test-acq-bats # only shard 1 of 2 (see below) +# +# SHARDING: ACQ_BATS_SHARD=I/N runs only the files scripts/bats-shard assigns to +# shard I of N, so CI can split the suite across a job matrix. Selection is +# recomputed from test/bats/*.bats on every run (no checked-in table), so a new +# file joins a shard automatically; an invalid spec or empty shard fails closed +# (exit 2). Ignored when explicit file arguments are given. set -uo pipefail _dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -69,7 +76,27 @@ if [ "$#" -gt 0 ]; then exec "$BATS" ${_timing[@]+"${_timing[@]}"} "$@" fi -# Whole-suite run: opt into across-files parallelism when we can. +# Whole-suite run (optionally one shard): the targets are the whole directory, +# or the explicit file list scripts/bats-shard selects. A bad shard spec must +# stop the run here rather than silently running nothing or everything. +_targets=("${REPO_ROOT}/test/bats") +_shard_files=() +if [ -n "${ACQ_BATS_SHARD:-}" ]; then + _shard_out="$(bash "${_dir}/bats-shard" "$ACQ_BATS_SHARD")" || { + echo "test-acq-bats: invalid ACQ_BATS_SHARD='${ACQ_BATS_SHARD}'" >&2 + exit 2 + } + while IFS= read -r _f; do + _shard_files+=("$_f") + done <&2 + printf 'test-acq-bats: %s\n' "${_shard_files[@]##*/}" >&2 +fi + +# Opt into across-files parallelism when we can. # - ACQ_BATS_JOBS overrides the job count; =1 forces serial. # - bats --jobs needs GNU parallel or shenwei356/rush on PATH; without one it # errors out, so we only pass --jobs when a parallelizer is present. @@ -85,13 +112,15 @@ if [ "${ACQ_BATS_TIMING:-0}" = 1 ]; then echo "test-acq-bats: workers=$_jobs; per-test timings include setup and teardown" >&2 fi if [ "$_jobs" -gt 1 ] 2>/dev/null && { command -v parallel >/dev/null 2>&1 || command -v rush >/dev/null 2>&1; }; then - exec "$BATS" ${_timing[@]+"${_timing[@]}"} --jobs "$_jobs" --no-parallelize-within-files "${REPO_ROOT}/test/bats" + exec "$BATS" ${_timing[@]+"${_timing[@]}"} --jobs "$_jobs" --no-parallelize-within-files "${_targets[@]}" fi if [ "$_jobs" -gt 1 ] 2>/dev/null && command -v xargs >/dev/null 2>&1 && printf 'x\0' | xargs -0 -n1 -P 1 true 2>/dev/null; then _tmp="$(mktemp -d "${TMPDIR:-/tmp}/acq-bats.XXXXXX")" trap 'rm -rf "$_tmp"' EXIT HUP INT TERM - printf '%s\0' "${REPO_ROOT}"/test/bats/*.bats | + _xargs_files=("${REPO_ROOT}"/test/bats/*.bats) + if [ "${#_shard_files[@]}" -gt 0 ]; then _xargs_files=("${_shard_files[@]}"); fi + printf '%s\0' "${_xargs_files[@]}" | xargs -0 -n1 -P "$_jobs" sh -c ' _bats=$1 _outdir=$2 _file=$3 _name=$(basename "$_file") @@ -105,7 +134,7 @@ if [ "$_jobs" -gt 1 ] 2>/dev/null && command -v xargs >/dev/null 2>&1 && printf ' sh "$BATS" "$_tmp" _rc=0 - for _file in "${REPO_ROOT}"/test/bats/*.bats; do + for _file in "${_xargs_files[@]}"; do _out="${_tmp}/$(basename "$_file").out" [ -f "$_out" ] && cat "$_out" _status="${_out}.status" @@ -116,4 +145,4 @@ if [ "$_jobs" -gt 1 ] 2>/dev/null && command -v xargs >/dev/null 2>&1 && printf exit "$_rc" fi -exec "$BATS" ${_timing[@]+"${_timing[@]}"} "${REPO_ROOT}/test/bats" +exec "$BATS" ${_timing[@]+"${_timing[@]}"} "${_targets[@]}" diff --git a/test/bats/01-test-runner.bats b/test/bats/01-test-runner.bats index adba30a0..8c521b42 100644 --- a/test/bats/01-test-runner.bats +++ b/test/bats/01-test-runner.bats @@ -9,6 +9,7 @@ setup() { mkdir -p "$RUNNER_ROOT/scripts" "$RUNNER_ROOT/test/vendor/bats-core/bin" "$RUNNER_ROOT/test/bats" cp "$REPO_ROOT/scripts/test-acq-bats" "$RUNNER_ROOT/scripts/test-acq-bats" cp "$REPO_ROOT/scripts/run-shellcheck" "$RUNNER_ROOT/scripts/run-shellcheck" + cp "$REPO_ROOT/scripts/bats-shard" "$RUNNER_ROOT/scripts/bats-shard" cat > "$RUNNER_ROOT/test/vendor/bats-core/bin/bats" <<'STUB' #!/bin/sh printf 'bats-arg:%s\n' "$@" @@ -80,6 +81,58 @@ STUB assert_line 'bats-arg:--timing' } +@test "runner: ACQ_BATS_SHARD runs only the selected files (serial path)" { + printf '@test "x" {\n true\n}\n' > "$RUNNER_ROOT/test/bats/a.bats" + printf '@test "y" {\n true\n}\n@test "z" {\n true\n}\n' > "$RUNNER_ROOT/test/bats/b.bats" + run env ACQ_BATS_JOBS=1 ACQ_BATS_SHARD=1/2 bash "$RUNNER_ROOT/scripts/test-acq-bats" + assert_success + assert_line "bats-arg:$RUNNER_ROOT/test/bats/b.bats" + refute_line "bats-arg:$RUNNER_ROOT/test/bats/a.bats" + refute_line "bats-arg:$RUNNER_ROOT/test/bats" + run env ACQ_BATS_JOBS=1 ACQ_BATS_SHARD=2/2 bash "$RUNNER_ROOT/scripts/test-acq-bats" + assert_success + assert_line "bats-arg:$RUNNER_ROOT/test/bats/a.bats" + refute_line "bats-arg:$RUNNER_ROOT/test/bats/b.bats" +} + +@test "runner: ACQ_BATS_SHARD xargs fallback runs only the selected files" { + printf '@test "x" {\n true\n}\n' > "$RUNNER_ROOT/test/bats/a.bats" + printf '@test "y" {\n true\n}\n@test "z" {\n true\n}\n' > "$RUNNER_ROOT/test/bats/b.bats" + run env ACQ_BATS_JOBS=2 ACQ_BATS_SHARD=1/2 bash -c ' + command() { + case "$*" in + "-v parallel"|"-v rush") return 1 ;; + *) builtin command "$@" ;; + esac + } + export -f command + exec bash "$1/scripts/test-acq-bats" + ' _ "$RUNNER_ROOT" + assert_success + assert_line "bats-arg:$RUNNER_ROOT/test/bats/b.bats" + refute_line "bats-arg:$RUNNER_ROOT/test/bats/a.bats" +} + +@test "runner: invalid ACQ_BATS_SHARD fails closed before running any test" { + local spec + for spec in 0/2 3/2 x 1/2/3 9/9; do + run env ACQ_BATS_JOBS=1 ACQ_BATS_SHARD="$spec" bash "$RUNNER_ROOT/scripts/test-acq-bats" + assert_failure 2 + refute_output --partial 'bats-arg:' + done +} + +@test "runner: explicit file arguments ignore ACQ_BATS_SHARD" { + run env ACQ_BATS_SHARD=1/2 bash "$RUNNER_ROOT/scripts/test-acq-bats" 'only.bats' + assert_success + assert_line 'bats-arg:only.bats' +} + +@test "runner: an ACQ_BATS_SHARD exported by the CI leg does not leak into runner tests" { + # acq_setup_stubs (this test's setup) must have cleared it. + [ -z "${ACQ_BATS_SHARD:-}" ] +} + @test "shellcheck runner: validates workers and preserves a failing batch" { cat > "$STUBDIR/shellcheck" <<'STUB' #!/bin/sh diff --git a/test/bats/02-bats-shard.bats b/test/bats/02-bats-shard.bats new file mode 100644 index 00000000..0b6d2b50 --- /dev/null +++ b/test/bats/02-bats-shard.bats @@ -0,0 +1,134 @@ +#!/usr/bin/env bats +# scripts/bats-shard: deterministic, balanced, fail-closed selection of shard I/N. + +load 'helper' + +setup() { + acq_setup_stubs + SHARD="$REPO_ROOT/scripts/bats-shard" + FAKE="$STUBDIR/fake-bats" + mkdir -p "$FAKE" + # Uneven sizes so weighting matters: big=5 tests, mid=3, small*=1, empty=0 (weight floor 1). + make_fake big 5 + make_fake mid 3 + make_fake small-a 1 + make_fake small-b 1 + make_fake small-c 1 + : > "$FAKE/empty.bats" +} + +teardown() { acq_teardown_stubs; } + +make_fake() { + local name="$1" count="$2" i + : > "$FAKE/$name.bats" + for ((i = 1; i <= count; i++)); do + printf '@test "%s %d" {\n true\n}\n' "$name" "$i" >> "$FAKE/$name.bats" + done +} + +# Collect the shard selections for N shards into $STUBDIR/sel.I files. +select_all() { + local n="$1" dir="${2:-$FAKE}" i + for ((i = 1; i <= n; i++)); do + bash "$SHARD" "$i/$n" "$dir" > "$STUBDIR/sel.$i" || return 1 + done +} + +@test "bats-shard: every file lands in exactly one shard for N=1..6" { + local n + for n in 1 2 3 4 5 6; do + select_all "$n" + cat "$STUBDIR"/sel.* | sort > "$STUBDIR/union" + ls "$FAKE"/*.bats | sort > "$STUBDIR/all" + diff "$STUBDIR/union" "$STUBDIR/all" + [ -z "$(sort "$STUBDIR/union" | uniq -d)" ] + rm -f "$STUBDIR"/sel.* + done +} + +@test "bats-shard: selection is deterministic across runs" { + run bash "$SHARD" 2/3 "$FAKE" + assert_success + local first="$output" + run bash "$SHARD" 2/3 "$FAKE" + assert_success + assert_output "$first" +} + +@test "bats-shard: balances by @test count, not by file count" { + # 5+3+1+1+1+(1 floor) = 12 tests -> two shards of 6. + select_all 2 + local i total + for i in 1 2; do + total=0 + while IFS= read -r f; do + c=$(grep -c '^@test' "$f" || true); [ "$c" -ge 1 ] || c=1 + total=$((total + c)) + done < "$STUBDIR/sel.$i" + [ "$total" -eq 6 ] + done +} + +@test "bats-shard: a newly added file is picked up with no table to update" { + make_fake brand-new 2 + select_all 3 + cat "$STUBDIR"/sel.* | grep -c 'brand-new.bats$' | grep -qx 1 + [ "$(cat "$STUBDIR"/sel.* | wc -l | tr -d ' ')" -eq 7 ] +} + +@test "bats-shard: removing a file leaves a complete, duplicate-free partition" { + rm "$FAKE/mid.bats" + select_all 3 + cat "$STUBDIR"/sel.* | sort > "$STUBDIR/union" + ls "$FAKE"/*.bats | sort > "$STUBDIR/all" + diff "$STUBDIR/union" "$STUBDIR/all" + [ -z "$(uniq -d "$STUBDIR/union")" ] +} + +@test "bats-shard: paths are absolute and handle spaces" { + make_fake 'has space' 1 + run bash "$SHARD" 1/1 "$FAKE" + assert_success + assert_line "$FAKE/has space.bats" +} + +@test "bats-shard: malformed specs fail closed with status 2 and no output" { + local spec + for spec in '' 0/2 3/2 a/b 1/0 1/ /2 01/2 1/02 1/2/3 ' 1/2' '1/2;x' -1/2 1/99999 '1 /2'; do + run bash "$SHARD" "$spec" "$FAKE" + assert_failure 2 + refute_output --regexp '\.bats' + done +} + +@test "bats-shard: no spec argument fails closed" { + run bash "$SHARD" + assert_failure 2 +} + +@test "bats-shard: an empty shard fails closed rather than passing vacuously" { + run bash "$SHARD" 7/7 "$FAKE" + assert_failure 2 + assert_output --partial 'empty' +} + +@test "bats-shard: missing directory and directory without .bats files fail closed" { + run bash "$SHARD" 1/2 "$STUBDIR/nope" + assert_failure 2 + mkdir "$STUBDIR/none" + run bash "$SHARD" 1/2 "$STUBDIR/none" + assert_failure 2 +} + +@test "bats-shard: real suite partitions completely for N=2 and N=3" { + local n + for n in 2 3; do + select_all "$n" "$REPO_ROOT/test/bats" + cat "$STUBDIR"/sel.* | sort > "$STUBDIR/union" + ls "$REPO_ROOT"/test/bats/*.bats | sort > "$STUBDIR/all" + diff "$STUBDIR/union" "$STUBDIR/all" + [ -z "$(uniq -d "$STUBDIR/union")" ] + rm -f "$STUBDIR"/sel.* + done +} diff --git a/test/bats/helper.bash b/test/bats/helper.bash index b6d0c6f4..827e67f1 100644 --- a/test/bats/helper.bash +++ b/test/bats/helper.bash @@ -43,7 +43,7 @@ acq_setup_stubs() { ACQ_PROVIDER_KITS_FROM_ENV ACQ_PROVIDER_KITS_FROM_CLI ACQ_UPDATE_CHECK \ ACQ_PROVIDER_CREDENTIAL_BINDINGS ACQ_PROVIDER_INPUT_PYTHON \ ACQ_PROVIDER_BINDINGS_GENERATION ACQ_KIT_SOURCES \ - ACQ_STATE_DIR ACQ_SECRET_STORE_DIR ACQ_NETWORK_TIER \ + ACQ_STATE_DIR ACQ_SECRET_STORE_DIR ACQ_NETWORK_TIER ACQ_BATS_SHARD \ ACQ_NETWORK_TIER_CONFIRM_OPEN ACQ_MSB_BALANCED_EGRESS ACQ_MSB_IMAGE \ ACQ_MSB_PULL ACQ_MSB_WORKSPACE ACQ_MSB_CLONES_DIR \ EMAIL GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME \