diff --git a/.github/scripts/junit-totals.sh b/.github/scripts/junit-totals.sh new file mode 100644 index 0000000000..616ac72c81 --- /dev/null +++ b/.github/scripts/junit-totals.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Prints " " over all JUnit XML in an extracted report bundle - from both +# Gradle (test-results) and AGP (androidTest-results): one per file with +# tests/failures/errors/skipped attributes. Failed counts failures and errors. +# +# Usage: junit-totals.sh +set -euo pipefail + +# the AGP file names contain spaces ("...(AVD) - 11-_app-.xml"), so the files are read through +# find -exec instead of being passed around as a word list +find "$1" -path '*-results/*' -name '*.xml' -exec cat {} + | awk ' + function attr(name) { return match($0, name "=\"[0-9]+\"") ? substr($0, RSTART + length(name) + 2, RLENGTH - length(name) - 3) + 0 : 0 } + /]/ { t += attr("tests"); f += attr("failures") + attr("errors"); s += attr("skipped") } + END { print t + 0, f + 0, s + 0 }' diff --git a/.github/scripts/publish-reports.sh b/.github/scripts/publish-reports.sh index 52b970ac3e..5536cbf2be 100644 --- a/.github/scripts/publish-reports.sh +++ b/.github/scripts/publish-reports.sh @@ -8,9 +8,15 @@ # Needs the gh CLI on the runner and GH_TOKEN with contents: write (to create the release/tag and # upload/delete its assets). # -# With REPORTS_TAG set (release-test-report.yml), the bundle goes to that version's pre-release -# instead, under its plain name and without the retention cleanup - that release is created by the -# calling workflow and kept until deleted by hand. +# With REPORTS_TAG set (release-test-report.yml), the bundle also goes to that version's +# pre-release, under its plain name - that release is created by the calling workflow and kept +# until deleted by hand. +# +# The name on ci-reports holds the git tree of the tested code, so a later release report run on +# the same code can reuse a passing bundle instead of running the tests again (see +# reuse-verified-reports.sh). The tree, not the commit: a PR run tests GitHub's merge preview +# commit, and the merge commit made when merging the PR has the same tree as long as the base +# branch hasn't changed in between. # # Usage: publish-reports.sh set -euo pipefail @@ -28,7 +34,6 @@ if [ -n "${REPORTS_TAG:-}" ]; then echo "## Reports: $name" echo "[Download $name]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$REPORTS_TAG/$name) - attached to the [$REPORTS_TAG]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/tag/$REPORTS_TAG) pre-release" } >> "$GITHUB_STEP_SUMMARY" - exit 0 fi # unit-test.yml and android-test.yml run at the same time, so both may try to create the release on @@ -41,7 +46,8 @@ fi # run id + attempt in the name, since all runs share this one release (a re-run gets its own file # instead of replacing the reports of the attempt before it) -name="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(basename "$file")" +tree=$(git rev-parse 'HEAD^{tree}') +name="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-tree-${tree}-$(basename "$file")" staged="$RUNNER_TEMP/$name" cp "$file" "$staged" gh release upload "$tag" "$staged" --clobber diff --git a/.github/scripts/release-report-summary.sh b/.github/scripts/release-report-summary.sh index 4f2344baa9..2082ff16ca 100644 --- a/.github/scripts/release-report-summary.sh +++ b/.github/scripts/release-report-summary.sh @@ -3,6 +3,9 @@ # results (and unit test line coverage) per suite, and writes them as the release notes and the # job summary. Used by release-test-report.yml after the test jobs have uploaded their bundles. # +# REUSED_UNIT, REUSED_API30 and REUSED_API34 hold the id of the earlier run whose bundle was reused +# for that suite (see reuse-verified-reports.sh), empty if this run tested it. +# # Needs the gh CLI on the runner and GH_TOKEN with contents: write. # # Usage: release-report-summary.sh @@ -26,53 +29,52 @@ run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" echo "- Run: [$GITHUB_RUN_ID]($run_url)" echo "- Date: $(date -u +%Y-%m-%d) (UTC)" echo - echo "| Suite | Result | Tests | Passed | Failed | Skipped | Report |" - echo "|---|---|---:|---:|---:|---:|---|" + echo "| Suite | Result | Tests | Passed | Failed | Skipped | Tested in | Report |" + echo "|---|---|---:|---:|---:|---:|---|---|" } > "$notes" failures="$work/failures.md" : > "$failures" coverage="" -for suite in "Unit tests:unit-test-reports.tar.gz" \ - "Instrumentation tests (API 30, small screen):android-test-reports-api30.tar.gz" \ - "Instrumentation tests (API 34, Pixel 6):android-test-reports-api34.tar.gz"; do - label="${suite%%:*}" - bundle="${suite#*:}" +for suite in "${REUSED_UNIT:-}:Unit tests:unit-test-reports.tar.gz" \ + "${REUSED_API30:-}:Instrumentation tests (API 30, small screen):android-test-reports-api30.tar.gz" \ + "${REUSED_API34:-}:Instrumentation tests (API 34, Pixel 6):android-test-reports-api34.tar.gz"; do + reused_from="${suite%%:*}" + rest="${suite#*:}" + label="${rest%%:*}" + bundle="${rest#*:}" + tested_in="this run" + if [ -n "$reused_from" ]; then + tested_in="[run $reused_from]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$reused_from), same code" + fi if [ ! -f "$work/bundles/$bundle" ]; then - echo "| $label | ⚠️ no report | | | | | |" >> "$notes" + echo "| $label | ⚠️ no report | | | | | $tested_in | |" >> "$notes" continue fi dir="$work/extract/${bundle%.tar.gz}" mkdir -p "$dir" tar -xzf "$work/bundles/$bundle" -C "$dir" - # JUnit XML from both Gradle (test-results) and AGP (androidTest-results): one per - # file with tests/failures/errors/skipped attributes, failing s followed by - # or - xmls=$(find "$dir" -path '*-results/*' -name '*.xml' | sort) - if [ -z "$xmls" ]; then - echo "| $label | ⚠️ no results | | | | | |" >> "$notes" + if [ -z "$(find "$dir" -path '*-results/*' -name '*.xml')" ]; then + echo "| $label | ⚠️ no results | | | | | $tested_in | |" >> "$notes" continue fi - # shellcheck disable=SC2086 - read -r tests failed skipped < <(awk ' - function attr(name, m) { return match($0, name "=\"[0-9]+\"") ? substr($0, RSTART + length(name) + 2, RLENGTH - length(name) - 3) + 0 : 0 } - /]/ { t += attr("tests"); f += attr("failures") + attr("errors"); s += attr("skipped") } - END { print t + 0, f + 0, s + 0 }' $xmls) + read -r tests failed skipped < <(bash "$(dirname "$0")/junit-totals.sh" "$dir") passed=$((tests - failed - skipped)) result="✅ passed" [ "$failed" -gt 0 ] && result="❌ failed" url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$tag/$bundle" - echo "| $label | $result | $tests | $passed | $failed | $skipped | [$bundle]($url) |" >> "$notes" + echo "| $label | $result | $tests | $passed | $failed | $skipped | $tested_in | [$bundle]($url) |" >> "$notes" if [ "$failed" -gt 0 ]; then echo "**$label**" >> "$failures" - # shellcheck disable=SC2086 - awk ' + # failing s are followed by or ; the AGP file names contain spaces, + # see junit-totals.sh + find "$dir" -path '*-results/*' -name '*.xml' -exec cat {} + | awk ' function attr(name, m) { return match($0, name "=\"[^\"]*\"") ? substr($0, RSTART + length(name) + 2, RLENGTH - length(name) - 3) : "" } /]/ { tc = attr("classname") "." attr("name") } - /]|]/ { if (tc != "") print "- `" tc "`"; tc = "" }' $xmls | sort -u >> "$failures" + /]|]/ { if (tc != "") print "- `" tc "`"; tc = "" }' | sort -u >> "$failures" echo >> "$failures" fi diff --git a/.github/scripts/reuse-verified-reports.sh b/.github/scripts/reuse-verified-reports.sh new file mode 100644 index 0000000000..a588c83739 --- /dev/null +++ b/.github/scripts/reuse-verified-reports.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Looks on the ci-reports pre-release for the newest bundle of the given suite that was made from +# exactly the checked-out code (same git tree, see publish-reports.sh) - e.g. by the PR run before +# merging, or by an earlier release report run whose summary failed. If it passed (tests ran, none +# failed), attaches it to the version's pre-release under its plain name, as if this run had +# tested it, and prints the id of the run that made it. Prints nothing if there's none to reuse, +# then the suite has to be tested again. +# +# Only the newest bundle counts: if the newest run on that code failed, an older passing one +# doesn't make it verified. +# +# Needs the gh CLI on the runner and GH_TOKEN with contents: write. +# +# Usage: reuse-verified-reports.sh +set -euo pipefail + +tag="$1" +bundle="$2" +tree=$(git rev-parse 'HEAD^{tree}') +suffix="-tree-$tree-$bundle" + +release_id=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/ci-reports" --jq .id 2>/dev/null) || { + echo "No ci-reports pre-release, testing $bundle" >&2 + exit 0 +} +name=$(gh api --paginate "repos/$GITHUB_REPOSITORY/releases/$release_id/assets" \ + --jq ".[] | select(.name | endswith(\"$suffix\")) | [.created_at, .name] | @tsv" | + sort | tail -1 | cut -f2) +if [ -z "$name" ]; then + echo "No $bundle tested on tree $tree, testing it" >&2 + exit 0 +fi + +work="$RUNNER_TEMP/reuse-$bundle" +rm -rf "$work" +mkdir -p "$work/extract" +gh release download ci-reports --pattern "$name" --dir "$work" +tar -xzf "$work/$name" -C "$work/extract" +read -r tests failed skipped < <(bash "$(dirname "$0")/junit-totals.sh" "$work/extract") +if [ "$tests" -eq 0 ] || [ "$failed" -gt 0 ]; then + echo "$name has $failed of $tests tests failed, testing $bundle again" >&2 + exit 0 +fi + +cp "$work/$name" "$work/$bundle" +gh release upload "$tag" "$work/$bundle" --clobber +rm -rf "$work" +echo "Reusing $name: $tests tests, none failed, $skipped skipped" >&2 +# --tree-... +echo "${name%%-*}" diff --git a/.github/workflows/android-test.yml b/.github/workflows/android-test.yml index 0c2a782607..37fcf1c08d 100644 --- a/.github/workflows/android-test.yml +++ b/.github/workflows/android-test.yml @@ -12,6 +12,10 @@ on: release-tag: type: string default: "" + # JSON list of the API levels to test, so it can leave out the ones that already passed + api-levels: + type: string + default: "[30, 34]" permissions: contents: write # attaching the report bundle to the ci-reports pre-release (publish-reports.sh) @@ -23,28 +27,27 @@ jobs: # not for PRs from forks - see unit-test.yml if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: [self-hosted, macOS] + # screen per API level: API 30 keeps the emulator's default small screen (320x640 @ 160dpi + # = 320x640dp, smaller than nearly any real phone) as the small-screen check; API 34 runs + # on a normal Full HD phone (pixel_6: 1080x2400 @ 420dpi, ~411x914dp). + # AVD_SUFFIX goes into the AVD name and cache key - the runner's disk keeps AVDs between + # runs and reuses one by name (force-avd-creation: false), so a new screen needs a new name. + # Set here rather than with matrix include entries, since an include entry for an API level + # left out by the api-levels input would be added as a job of its own + env: + AVD_PROFILE: ${{ matrix.api-level == 34 && 'pixel_6' || '' }} + AVD_SUFFIX: ${{ matrix.api-level == 34 && '-pixel6' || '' }} strategy: max-parallel: 1 # multiple runners are registered on the same physical Mac - without this, # matrix legs can run concurrently and contend for the same real GPU, which is what was # causing "Failed to find ColorBuffer" emulator errors regardless of the -gpu backend used matrix: - api-level: - - 30 # min API with arm64-v8a system images - - 34 # close to targetSdk, well-supported arm64-v8a image + # 30: min API with arm64-v8a system images; 34: close to targetSdk, well-supported + # arm64-v8a image. When not called by release-test-report.yml (pull_request, + # workflow_dispatch), there's no api-levels input, so both are tested + api-level: ${{ fromJSON(inputs.api-levels || '[30, 34]') }} arch: - arm64-v8a - # screen per API level: API 30 keeps the emulator's default small screen (320x640 @ 160dpi - # = 320x640dp, smaller than nearly any real phone) as the small-screen check; API 34 runs - # on a normal Full HD phone (pixel_6: 1080x2400 @ 420dpi, ~411x914dp). - # avd-suffix goes into the AVD name and cache key - the runner's disk keeps AVDs between - # runs and reuses one by name (force-avd-creation: false), so a new screen needs a new name - include: - - api-level: 30 - profile: "" - avd-suffix: "" - - api-level: 34 - profile: pixel_6 - avd-suffix: "-pixel6" steps: - uses: actions/checkout@v4 @@ -57,7 +60,7 @@ jobs: path: | ~/.android/avd/* ~/.android/adb* - key: avd-${{ matrix.arch }}-${{ matrix.api-level }}-google-apis-swiftshader${{ matrix.avd-suffix }} + key: avd-${{ matrix.arch }}-${{ matrix.api-level }}-google-apis-swiftshader${{ env.AVD_SUFFIX }} - name: Create AVD and generate snapshot for caching if: steps.avd-cache.outputs.cache-hit != 'true' @@ -66,13 +69,13 @@ jobs: api-level: ${{ matrix.api-level }} arch: ${{ matrix.arch }} target: google_apis - profile: ${{ matrix.profile }} + profile: ${{ env.AVD_PROFILE }} # this is a self-hosted runner with a persistent disk (unlike GitHub-hosted runners, # which start every job with an empty disk) - without a name derived from the actual # config, the action's default fixed "test" AVD name means any future change to # api-level/arch/target/profile here would silently keep reusing a stale AVD built under # the OLD config instead of picking up the new one, since force-avd-creation stays false - avd-name: ci-${{ matrix.arch }}-api${{ matrix.api-level }}-google-apis${{ matrix.avd-suffix }} + avd-name: ci-${{ matrix.arch }}-api${{ matrix.api-level }}-google-apis${{ env.AVD_SUFFIX }} force-avd-creation: false emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disable-animations: false @@ -84,8 +87,8 @@ jobs: api-level: ${{ matrix.api-level }} arch: ${{ matrix.arch }} target: google_apis - profile: ${{ matrix.profile }} - avd-name: ci-${{ matrix.arch }}-api${{ matrix.api-level }}-google-apis${{ matrix.avd-suffix }} + profile: ${{ env.AVD_PROFILE }} + avd-name: ci-${{ matrix.arch }}-api${{ matrix.api-level }}-google-apis${{ env.AVD_SUFFIX }} force-avd-creation: false emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disable-animations: true diff --git a/.github/workflows/release-test-report.yml b/.github/workflows/release-test-report.yml index b644382e18..edb3e7a2c4 100644 --- a/.github/workflows/release-test-report.yml +++ b/.github/workflows/release-test-report.yml @@ -4,6 +4,12 @@ name: Release test report # the unit + instrumentation test reports for a release build. They're attached to a # "v-test-reports" pre-release titled " release test reports summary", whose # notes hold the pass/fail summary. Running again for the same version replaces that pre-release. +# +# A suite that already passed on exactly this code (same git tree) isn't tested again: its bundle +# from that run (usually the PR's, or an earlier run of this workflow) is reused, see +# reuse-verified-reports.sh. So after a merged PR whose tests passed, or a run where only some +# suites failed, only what's missing runs. The "tested in" column of the summary says which run +# each suite's reports come from. on: workflow_dispatch: @@ -12,6 +18,10 @@ on: description: "Version name (leave empty to use appVersionName from app/build.gradle.kts)" type: string default: "" + retest: + description: "Run all tests again, even if they already passed on this code" + type: boolean + default: false permissions: contents: write # creating the pre-release and attaching the report bundles @@ -27,6 +37,10 @@ jobs: outputs: version: ${{ steps.release.outputs.version }} tag: ${{ steps.release.outputs.tag }} + reused-unit: ${{ steps.reuse.outputs.unit }} + reused-api30: ${{ steps.reuse.outputs.api30 }} + reused-api34: ${{ steps.reuse.outputs.api34 }} + api-levels: ${{ steps.reuse.outputs.api-levels }} steps: - uses: actions/checkout@v4 @@ -53,17 +67,44 @@ jobs: echo "version=$version" >> "$GITHUB_OUTPUT" echo "tag=$tag" >> "$GITHUB_OUTPUT" + - name: Reuse reports of suites that already passed on this code + id: reuse + env: + GH_TOKEN: ${{ github.token }} + RETEST: ${{ inputs.retest }} + TAG: ${{ steps.release.outputs.tag }} + run: | + unit="" api30="" api34="" + # a failing lookup only means testing again + if [ "$RETEST" != "true" ]; then + unit=$(bash .github/scripts/reuse-verified-reports.sh "$TAG" unit-test-reports.tar.gz || true) + api30=$(bash .github/scripts/reuse-verified-reports.sh "$TAG" android-test-reports-api30.tar.gz || true) + api34=$(bash .github/scripts/reuse-verified-reports.sh "$TAG" android-test-reports-api34.tar.gz || true) + fi + levels="" + [ -z "$api30" ] && levels="$levels,30" + [ -z "$api34" ] && levels="$levels,34" + { + echo "unit=$unit" + echo "api30=$api30" + echo "api34=$api34" + echo "api-levels=[${levels#,}]" + } >> "$GITHUB_OUTPUT" + unit-test: needs: prepare + if: needs.prepare.outputs.reused-unit == '' uses: ./.github/workflows/unit-test.yml with: release-tag: ${{ needs.prepare.outputs.tag }} android-test: needs: prepare + if: needs.prepare.outputs.api-levels != '[]' uses: ./.github/workflows/android-test.yml with: release-tag: ${{ needs.prepare.outputs.tag }} + api-levels: ${{ needs.prepare.outputs.api-levels }} summary: needs: [prepare, unit-test, android-test] @@ -76,4 +117,7 @@ jobs: - name: Write release notes env: GH_TOKEN: ${{ github.token }} + REUSED_UNIT: ${{ needs.prepare.outputs.reused-unit }} + REUSED_API30: ${{ needs.prepare.outputs.reused-api30 }} + REUSED_API34: ${{ needs.prepare.outputs.reused-api34 }} run: bash .github/scripts/release-report-summary.sh "${{ needs.prepare.outputs.tag }}" "${{ needs.prepare.outputs.version }}"