From b5887139696920f015a92200267143ba3e265f38 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 02:16:25 +0100 Subject: [PATCH 01/13] Size CI test jobs on RunsOn runners to the parallelism they can use Jobs that run tests with --run-in-parallel start one serial pytest per test type, so they never use more CPUs than they have test types. On RunsOn, splitting core test types into two jobs only doubles the jobs: each half still lasts as long as its slowest test type, and the out-of-memory risk the split guards against on public runners does not apply to the larger RunsOn runners. The lowest-dependencies provider jobs left their 32 CPUs partly idle, and the integration-core and system tests ran a single pytest on 32-CPU runners. Generated-by: Claude Opus 5 --- .../workflows/integration-system-tests.yml | 2 +- .github/workflows/special-tests.yml | 2 +- dev/breeze/doc/ci/04_selective_checks.md | 14 +++++- .../src/airflow_breeze/global_constants.py | 7 +++ .../airflow_breeze/utils/selective_checks.py | 27 +++++++----- dev/breeze/tests/test_selective_checks.py | 44 +++++++++++++++++++ 6 files changed, 81 insertions(+), 15 deletions(-) diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index 8b24223a55f93..e61e4070e0c1b 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -75,7 +75,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-core-integrations != '[]' name: "Integration core ${{ matrix.integration }}" - runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} strategy: fail-fast: false max-parallel: 20 diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 09658246df9a6..5eca561ba59c4 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -352,7 +352,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.runners, '32+64', '128+256') }} + runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} test-name: "SystemTest" test-scope: "System" diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 4bf5fffa28457..4646442c0d0d5 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -651,7 +651,7 @@ GitHub Actions to pass the list of parameters to a command to execute | all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | | basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | | ci-image-build | Whether CI image build is needed | true | | -| core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core | API Always Providers | * | +| core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core (2 slices; 1 on RunsOn) | API Always Providers | * | | debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | | default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | | default-constraints-branch | Which branch is default for the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | @@ -675,7 +675,7 @@ GitHub Actions to pass the list of parameters to a command to execute | helm-version | Which Helm version to use for tests | v3.15.3 | | | in-memory-runner-type | Like runner-type, but the RunsOn template puts docker in RAM (`extras=tmpfs`) unless disabled | \["ubuntu-22.04"\] | | | include-success-outputs | Whether to include outputs of successful parallel tests ("true"/"false") | false | | -| individual-providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (individually listed) | Providers[\amazon\] Providers\[google\] | * | +| individual-providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (individually listed; 5 slices, 3 on RunsOn)| Providers[\amazon\] Providers\[google\] | * | | is-committer-build | Whether the build is triggered by a committer | false | | | is-legacy-ui-api-labeled | Whether the PR is labeled as legacy UI/API | false | | | java-sdk-version | JDK version used to build the lang-SDK Java artifacts natively in CI | 17 | | @@ -767,6 +767,16 @@ picks its size class with `format(fromJSON(inputs.runners)[0], , )` (` `'8+16', '32+64'` or `'32+64', '128+256'`), or, for a single-job reusable workflow, its caller passes `format(, , )`. Each class is a range up to twice its size, so Spot can come from the larger instance types too; xdist follows the host's CPUs, so a larger runner runs more test workers. +Jobs using `--run-in-parallel` (the DB, lowest-dependencies and special tests) run one serial pytest per +test type instead, so they never use more CPUs than they have test types, and jobs running a single pytest +(integration and system tests) use one. The size classes follow that: the xdist jobs and the +lowest-dependencies provider tests take `'32+64', '128+256'`, the others `'8+16', '32+64'` or smaller. +The matrix slices follow it too. On public runners the core test types are split into +`NUMBER_OF_CORE_SLICES` (2) jobs, to keep the Non-DB xdist collection inside the runner's memory, and +the individually listed providers into `NUMBER_OF_LOW_DEP_SLICES` (5). On RunsOn the core test types +stay in one job (`NUMBER_OF_CORE_SLICES_ON_RUNS_ON`), because a core DB job lasts as long as its +slowest test type whichever slice it is in, and the providers are split into 3 jobs +(`NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON`), so each job has about one provider per CPU of its runner. `spot=co` takes Spot from the pools with the most capacity regardless of price, which is cheaper than the on-demand launch and whole-job rerun a capacity miss or interruption costs; on a public label the same call returns the label unchanged. Steps that only make sense on RunsOn are diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index efa1c3425b0d0..f4436496fa162 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -968,6 +968,13 @@ def get_legacy_base_image(debian_version: str = ALLOWED_DEBIAN_VERSIONS[0]) -> s # Number of slices for core test types (splitting reduces memory per xdist collection) NUMBER_OF_CORE_SLICES = 2 +# Slice counts on the self-hosted RunsOn runners. Jobs using `--run-in-parallel` run one serial pytest per +# test type, so a job never uses more CPUs than it has test types. The core jobs are bound by their slowest +# test type and the Non-DB xdist collection gets enough memory on RunsOn, so one slice only removes jobs. +# Low dep providers run on 32-CPU runners: a third of the providers per job keeps those CPUs busy. +NUMBER_OF_CORE_SLICES_ON_RUNS_ON = 1 +NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON = 3 + # Milestone Tag Assistant configuration # Labels indicating a bug fix PR that should have a milestone MILESTONE_BUG_LABELS: frozenset[str] = frozenset({"kind:bug", "type:bug-fix"}) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index b5415d504d32b..8ca2802074860 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -52,7 +52,9 @@ JAVA_SDK_VERSION, KIND_VERSION, NUMBER_OF_CORE_SLICES, + NUMBER_OF_CORE_SLICES_ON_RUNS_ON, NUMBER_OF_LOW_DEP_SLICES, + NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, PUBLIC_AMD_RUNNERS, PUBLIC_ARM_RUNNERS, @@ -1877,9 +1879,10 @@ def core_test_types_list_as_strings_in_json(self) -> str | None: if not self.run_unit_tests: return None current_test_types = sorted(set(self._get_core_test_types_to_run())) - if len(current_test_types) <= NUMBER_OF_CORE_SLICES: + number_of_slices = NUMBER_OF_CORE_SLICES_ON_RUNS_ON if self._is_on_runs_on else NUMBER_OF_CORE_SLICES + if len(current_test_types) <= number_of_slices: return json.dumps(_get_test_list_as_json([current_test_types])) - list_of_list_of_types = _split_list(current_test_types, NUMBER_OF_CORE_SLICES) + list_of_list_of_types = _split_list(current_test_types, number_of_slices) return json.dumps(_get_test_list_as_json(list_of_list_of_types)) @cached_property @@ -1966,11 +1969,12 @@ def individual_providers_test_types_list_as_strings_in_json(self) -> str | None: current_test_types = sorted(self._get_individual_providers_list()) if not current_test_types: return None - # We are hard-coding the number of lists as reasonable starting point to split the - # list of test types - and we can modify it in the future - if len(current_test_types) < NUMBER_OF_LOW_DEP_SLICES: + number_of_slices = ( + NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON if self._is_on_runs_on else NUMBER_OF_LOW_DEP_SLICES + ) + if len(current_test_types) < number_of_slices: return json.dumps(_get_test_list_as_json([current_test_types])) - list_of_list_of_types = _split_list(current_test_types, NUMBER_OF_LOW_DEP_SLICES) + list_of_list_of_types = _split_list(current_test_types, number_of_slices) return json.dumps(_get_test_list_as_json(list_of_list_of_types)) @cached_property @@ -2322,17 +2326,18 @@ def runner_type(self) -> str: ) return json.dumps([template.format(run_id=run_id)]) + @cached_property + def _is_on_runs_on(self) -> bool: + # Reuses the routing decision of runner_type, which may have asked the GitHub API. + return self.runner_type not in (PUBLIC_AMD_RUNNERS, PUBLIC_ARM_RUNNERS) + @cached_property def in_memory_runner_type(self) -> str: """Runner labels like ``runner_type``, with docker on a RAM disk when the run is on RunsOn. The repository variable ``USE_IN_MEMORY_DOCKER=false`` keeps docker on disk. """ - # Reuses the routing decision of runner_type, which may have asked the GitHub API. - if ( - self.runner_type in (PUBLIC_AMD_RUNNERS, PUBLIC_ARM_RUNNERS) - or os.environ.get("USE_IN_MEMORY_DOCKER", "").casefold() == "false" - ): + if not self._is_on_runs_on or os.environ.get("USE_IN_MEMORY_DOCKER", "").casefold() == "false": return self.runner_type template = ( RUNS_ON_ARM_IN_MEMORY_RUNNER_TEMPLATE diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index ddce03204adf8..00b133b40dc79 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -37,7 +37,9 @@ DEFAULT_PYTHON_MAJOR_MINOR_VERSION, JAVA_SDK_VERSION, NUMBER_OF_CORE_SLICES, + NUMBER_OF_CORE_SLICES_ON_RUNS_ON, NUMBER_OF_LOW_DEP_SLICES, + NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, GithubEvents, get_airflow_version, @@ -2250,6 +2252,48 @@ def test_list_splitting(): ) +@pytest.mark.parametrize( + ("github_event", "github_context_dict", "expected_core_slices", "expected_low_dep_slices"), + [ + pytest.param( + GithubEvents.PULL_REQUEST, + {}, + NUMBER_OF_CORE_SLICES, + NUMBER_OF_LOW_DEP_SLICES, + id="public runners", + ), + pytest.param( + GithubEvents.PUSH, + {"run_id": 42}, + NUMBER_OF_CORE_SLICES_ON_RUNS_ON, + NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON, + id="RunsOn runners", + ), + ], +) +def test_number_of_test_type_slices_follows_runners( + github_event: GithubEvents, + github_context_dict: dict[str, Any], + expected_core_slices: int, + expected_low_dep_slices: int, +): + selective_checks = SelectiveChecks( + pr_labels=("full tests needed",), + default_branch="main", + github_event=github_event, + github_repository="apache/airflow", + github_context_dict=github_context_dict, + ) + core_slices = json.loads(selective_checks.core_test_types_list_as_strings_in_json) + low_dep_slices = json.loads(selective_checks.individual_providers_test_types_list_as_strings_in_json) + assert len(core_slices) == expected_core_slices + assert len(low_dep_slices) == expected_low_dep_slices + assert sorted(" ".join(s["test_types"] for s in core_slices).split()) == sorted( + ALL_CI_SELECTIVE_TEST_TYPES.split() + ) + assert sum(len(s["test_types"].split()) for s in low_dep_slices) == len(LIST_OF_ALL_PROVIDER_TESTS) + + def test_excluded_providers(): stderr = SelectiveChecks( files=(), From 5a905fadfc88e869e07ddcd89749b74cb0661c9d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 02:34:24 +0100 Subject: [PATCH 02/13] Run provider tests one provider per test type on RunsOn runners Grouping all providers but the long-running ones into one test type left the provider jobs waiting on that group's single serial pytest while the other CPUs idled. The Non-DB jobs run every provider in one xdist pytest, where splitting test types only adds entries, so they get the providers unsplit. Generated-by: Claude Opus 5 --- .github/workflows/ci-amd.yml | 10 ++-- .github/workflows/ci-arm.yml | 10 ++-- .github/workflows/special-tests.yml | 8 +-- .github/workflows/test-providers.yml | 2 +- dev/breeze/doc/ci/04_selective_checks.md | 11 +++- .../airflow_breeze/utils/selective_checks.py | 28 +++++++-- dev/breeze/tests/test_selective_checks.py | 59 +++++++++++++++++++ 7 files changed, 108 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 75c3ff8edc4f1..27d8a07fd7c42 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -319,6 +319,8 @@ jobs: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + providers-xdist-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-xdist-test-types-list-as-strings-in-json }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} @@ -797,7 +799,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -856,7 +858,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -916,7 +918,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -990,7 +992,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-xdist-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 341e1e021603e..6c9f36e4f43cc 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -200,6 +200,8 @@ jobs: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + providers-xdist-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-xdist-test-types-list-as-strings-in-json }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} @@ -678,7 +680,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -737,7 +739,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -797,7 +799,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -871,7 +873,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-xdist-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 5eca561ba59c4..f508ca2224fee 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -123,7 +123,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} + runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" @@ -173,7 +173,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} + runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" test-name: "LatestSQLAlchemy-Postgres" @@ -224,7 +224,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} + runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" @@ -276,7 +276,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} + runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index ccf9dc1230498..b1fefd3385297 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -178,7 +178,7 @@ jobs: timeout-minutes: 80 # yamllint disable rule:line-length name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.test-types.description }} - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }} strategy: fail-fast: false matrix: diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 4646442c0d0d5..734ca8ac5412e 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -691,7 +691,8 @@ GitHub Actions to pass the list of parameters to a command to execute | postgres-versions | Which versions of Postgres to use for tests as JSON array | \['12'\] | | | prod-image-build | Whether PROD image build is needed | true | | | providers-compatibility-tests-matrix | Matrix of providers compatibility tests: (python_version, airflow_version, removed_providers). Excludes unbuilt python versions and the entry whose airflow_version equals the current branch version (unreleased on a vX-Y-test branch until GA) | \[{}\] | | -| providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers | Providers Providers\[-google\] | * | +| providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (one per provider on RunsOn) | Providers Providers\[-google\] | * | +| providers-xdist-test-types-list-as-strings-in-json | Which test types the Non-DB (xdist) provider tests run, with no provider split out | Providers | * | | pyproject-toml-changed | When pyproject.toml changed in the PR. | false | | | python-versions | List of python versions to use for that build | \['3.11'\] | | | python-versions-list-as-string | Which versions of MySQL to use for tests as space-separated string | 3.11 | * | @@ -769,8 +770,12 @@ picks its size class with `format(fromJSON(inputs.runners)[0], , )` (` the larger instance types too; xdist follows the host's CPUs, so a larger runner runs more test workers. Jobs using `--run-in-parallel` (the DB, lowest-dependencies and special tests) run one serial pytest per test type instead, so they never use more CPUs than they have test types, and jobs running a single pytest -(integration and system tests) use one. The size classes follow that: the xdist jobs and the -lowest-dependencies provider tests take `'32+64', '128+256'`, the others `'8+16', '32+64'` or smaller. +(integration and system tests) use one. The size classes follow that: the xdist jobs and every job +running provider tests in parallel take `'32+64', '128+256'`, the others `'8+16', '32+64'` or smaller. +On RunsOn `providers-test-types-list-as-strings-in-json` lists each provider as its own test type, in +one job, instead of the long-running providers plus one group of all the others, so the job is not left +waiting on that group. The Non-DB xdist jobs read `providers-xdist-test-types-list-as-strings-in-json` +instead, which never splits the providers: xdist runs them all in one pytest anyway. The matrix slices follow it too. On public runners the core test types are split into `NUMBER_OF_CORE_SLICES` (2) jobs, to keep the Non-DB xdist collection inside the runner's memory, and the individually listed providers into `NUMBER_OF_LOW_DEP_SLICES` (5). On RunsOn the core test types diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 8ca2802074860..6fd832f2ee664 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1932,10 +1932,7 @@ def _filter_platform_excluded_test_types(self, current_test_types: set[str]) -> if kept: current_test_types.add(f"Providers[{','.join(kept)}]") - @cached_property - def providers_test_types_list_as_strings_in_json(self) -> str: - if not self.run_unit_tests: - return "[]" + def _get_providers_test_types_for_branch(self) -> set[str]: current_test_types = set(self._get_providers_test_types_to_run()) if self._default_branch != "main": test_types_to_remove: set[str] = set() @@ -1947,10 +1944,33 @@ def providers_test_types_list_as_strings_in_json(self) -> str: ) test_types_to_remove.add(test_type) current_test_types = current_test_types - test_types_to_remove + return current_test_types + + @cached_property + def providers_test_types_list_as_strings_in_json(self) -> str: + if not self.run_unit_tests: + return "[]" + current_test_types = self._get_providers_test_types_for_branch() + if current_test_types and self._is_on_runs_on: + # One serial pytest per provider keeps the 32 CPUs of a RunsOn runner busy; grouped + # providers would leave the job waiting on the large "all other providers" group. + return json.dumps(_get_test_list_as_json([sorted(self._get_individual_providers_list())])) self._extract_long_provider_tests(current_test_types) self._filter_platform_excluded_test_types(current_test_types) return json.dumps(_get_test_list_as_json([sorted(current_test_types)])) + @cached_property + def providers_xdist_test_types_list_as_strings_in_json(self) -> str: + """Provider test types for the xdist jobs, which run them all in one pytest, so none is split out.""" + if not self.run_unit_tests: + return "[]" + current_test_types = self._get_providers_test_types_for_branch() + if "Providers" in current_test_types and self._platform_excluded_providers: + current_test_types.remove("Providers") + current_test_types.add(f"Providers[-{','.join(sorted(self._platform_excluded_providers))}]") + self._filter_platform_excluded_test_types(current_test_types) + return json.dumps(_get_test_list_as_json([sorted(current_test_types)])) + def _get_individual_providers_list(self): current_test_types = set(self._get_providers_test_types_to_run(split_to_individual_providers=True)) if "Providers" in current_test_types: diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 00b133b40dc79..311f3658a62f6 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2294,6 +2294,65 @@ def test_number_of_test_type_slices_follows_runners( assert sum(len(s["test_types"].split()) for s in low_dep_slices) == len(LIST_OF_ALL_PROVIDER_TESTS) +@pytest.mark.parametrize( + ("github_event", "github_context_dict", "expected_test_types"), + [ + pytest.param( + GithubEvents.PULL_REQUEST, + {}, + [ + "Providers[-amazon,celery,google,standard]", + "Providers[amazon]", + "Providers[celery]", + "Providers[google]", + "Providers[standard]", + ], + id="public runners group the providers", + ), + pytest.param( + GithubEvents.PUSH, + {"run_id": 42}, + sorted(LIST_OF_ALL_PROVIDER_TESTS), + id="RunsOn runners list each provider", + ), + ], +) +def test_providers_test_types_follow_runners( + github_event: GithubEvents, github_context_dict: dict[str, Any], expected_test_types: list[str] +): + selective_checks = SelectiveChecks( + pr_labels=("full tests needed",), + default_branch="main", + github_event=github_event, + github_repository="apache/airflow", + github_context_dict=github_context_dict, + ) + slices = json.loads(selective_checks.providers_test_types_list_as_strings_in_json) + assert [s["test_types"].split() for s in slices] == [expected_test_types] + + +@pytest.mark.parametrize( + ("platform", "expected_test_types"), + [ + pytest.param(CI_AMD_PLATFORM, "Providers", id="AMD"), + pytest.param(CI_ARM_PLATFORM, "Providers[-ibm.db2,ibm.mq]", id="ARM skips platform-excluded"), + ], +) +def test_providers_xdist_test_types_are_not_split(platform: str, expected_test_types: str): + selective_checks = SelectiveChecks( + files=("airflow-core/tests/test_example.py",), + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.PUSH, + github_repository="apache/airflow", + github_context_dict={"run_id": 42}, + default_branch="main", + pr_labels=("full tests needed",), + platform=platform, + ) + slices = json.loads(selective_checks.providers_xdist_test_types_list_as_strings_in_json) + assert [s["test_types"] for s in slices] == [expected_test_types] + + def test_excluded_providers(): stderr = SelectiveChecks( files=(), From 80bc577757730ea0b87995f5ee3e7b91ee6c8c5d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 02:50:31 +0100 Subject: [PATCH 03/13] Give docker on RunsOn runners enough address pools for parallel test runs Each test type of a parallel test run is a docker compose project with its own network. Docker's default address pools fit about 30 networks, so provider jobs running a test type per CPU on 32-CPU and larger runners failed with "all predefined address pools have been fully subnetted". Generated-by: Claude Opus 5 --- .../prepare_breeze_and_image/action.yml | 5 +++ scripts/ci/configure_docker_address_pools.sh | 40 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 scripts/ci/configure_docker_address_pools.sh diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 263db0346d444..939d057e0f61b 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -71,6 +71,11 @@ runs: else ./scripts/tools/free_up_disk_space.sh fi + - name: "Configure docker address pools" + shell: bash + run: ./scripts/ci/configure_docker_address_pools.sh + # Only self-hosted runners are large enough to run more test types at once than the default pools fit. + if: runner.environment == 'self-hosted' - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze diff --git a/scripts/ci/configure_docker_address_pools.sh b/scripts/ci/configure_docker_address_pools.sh new file mode 100755 index 0000000000000..15cac414070d1 --- /dev/null +++ b/scripts/ci/configure_docker_address_pools.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Every test type of a parallel test run is a docker compose project with a network of its own. Docker's +# default address pools hand out /16 and /20 subnets, which is about 30 networks per host, so large runners +# running a test type per CPU fail with "all predefined address pools have been fully subnetted". +# Splitting the same 172.17.0.0/12 range into /24 subnets gives 4096 networks without claiming new addresses. +set -euo pipefail + +DAEMON_JSON=/etc/docker/daemon.json + +sudo mkdir -p "$(dirname "${DAEMON_JSON}")" +sudo python3 - "${DAEMON_JSON}" <<'EOF' +import json +import pathlib +import sys + +daemon_json = pathlib.Path(sys.argv[1]) +config = json.loads(daemon_json.read_text()) if daemon_json.exists() and daemon_json.read_text().strip() else {} +config["default-address-pools"] = [{"base": "172.17.0.0/12", "size": 24}] +daemon_json.write_text(json.dumps(config, indent=2) + "\n") +print(daemon_json.read_text()) +EOF +sudo systemctl restart docker +docker info --format '{{.ServerVersion}}' > /dev/null From 6aec22514f02225ddd45d938c72702c47626f6f7 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 02:51:54 +0100 Subject: [PATCH 04/13] Split the longest-running provider tests into several test types on RunsOn Once every provider runs as its own test type, a job lasts as long as its slowest provider, and amazon, google and standard each take far longer than the rest as a single serial pytest. Splitting them lets the large RunsOn runners work on their parts at once. Lowest-dependencies jobs are sized by test-type count, so a change touching a few providers keeps one job. Generated-by: Claude Opus 5 --- Dockerfile.ci | 2 + dev/breeze/doc/ci/04_selective_checks.md | 12 ++++- .../commands/testing_commands.py | 7 +++ .../src/airflow_breeze/global_constants.py | 33 +++++++++++++- .../utils/ci_minutes_estimate.py | 6 ++- .../src/airflow_breeze/utils/run_tests.py | 36 +++++++++++++-- .../airflow_breeze/utils/selective_checks.py | 15 ++++++- .../tests/test_pytest_args_for_test_types.py | 39 ++++++++++++++++ dev/breeze/tests/test_run_test_args.py | 29 ++++++++++++ dev/breeze/tests/test_selective_checks.py | 44 ++++++++++++++++--- scripts/docker/entrypoint_ci.sh | 2 + 11 files changed, 208 insertions(+), 17 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index 46435c08f484c..fd9c2012ca946 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1779,6 +1779,8 @@ function check_force_lowest_dependencies() { local provider_id # shellcheck disable=SC2001 provider_id=$(echo "${TEST_TYPE}" | sed 's/Providers\[\(.*\)\]/\1/') + # A split provider's test type names one part of its tests: Providers[:] + provider_id=${provider_id%%:*} echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${provider_id}${COLOR_RESET}" echo diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 734ca8ac5412e..6e74b2fff5f3d 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -780,14 +780,22 @@ The matrix slices follow it too. On public runners the core test types are split `NUMBER_OF_CORE_SLICES` (2) jobs, to keep the Non-DB xdist collection inside the runner's memory, and the individually listed providers into `NUMBER_OF_LOW_DEP_SLICES` (5). On RunsOn the core test types stay in one job (`NUMBER_OF_CORE_SLICES_ON_RUNS_ON`), because a core DB job lasts as long as its -slowest test type whichever slice it is in, and the providers are split into 3 jobs -(`NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON`), so each job has about one provider per CPU of its runner. +slowest test type whichever slice it is in, and the providers are split into jobs of about 40 test types +(`LOW_DEP_TEST_TYPES_PER_JOB_ON_RUNS_ON`), which the heaviest-first ordering keeps the 32 CPUs busy with: +3 jobs for all providers, one for a change touching a few. On RunsOn the longest-running providers +(`PROVIDER_TEST_SPLITS`: amazon, google and standard) are also split into several test types such as +`Providers[amazon:hooks]`, with `Providers[amazon:other]` running the rest of the provider's tests, so +none of them leaves the job waiting on one long serial pytest. `spot=co` takes Spot from the pools with the most capacity regardless of price, which is cheaper than the on-demand launch and whole-job rerun a capacity miss or interruption costs; on a public label the same call returns the label unchanged. Steps that only make sense on RunsOn are gated on `runner.environment == 'self-hosted'`: * The EC2 cost summary of each job. +* The docker address pools (`scripts/ci/configure_docker_address_pools.sh`). Every test type of a parallel + run is a docker compose project with a network of its own, and Docker's default pools fit about 30 + networks, fewer than a RunsOn runner runs test types at once; the script splits the same range into + `/24` subnets. * The image stashes (`.github/actions/stash-restore` / `stash-save`). On RunsOn the image tarballs go to the stack's S3 cache bucket, named by their sha256, and only the digest is stashed as a GitHub artifact. Any RunsOn job can write to the bucket, so a tarball is used only if it matches the digest stashed by diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 279f05d2a86d0..cbfb6377736bf 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -90,9 +90,11 @@ ALL_TEST_TYPE, ALLOWED_KUBERNETES_VERSIONS, ALLOWED_TEST_TYPE_CHOICES, + PROVIDER_TEST_SPLITS, TESTCONTAINERS_IMAGES_BY_PROVIDER, GroupOfTests, all_selective_core_test_types, + get_provider_split_test_types, providers_test_type, ) from airflow_breeze.params.build_prod_params import BuildProdParams @@ -391,6 +393,11 @@ def _run_tests_in_pool( # tests are still running. We are only adding here test types that take more than 2 minutes to run # on a fast machine in parallel sorting_order = [ + *( + test_type + for provider_id in PROVIDER_TEST_SPLITS + for test_type in get_provider_split_test_types(provider_id) + ), "Providers[standard]", "Providers[amazon]", "Providers[google]", diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index f4436496fa162..70146552743bc 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -971,9 +971,38 @@ def get_legacy_base_image(debian_version: str = ALLOWED_DEBIAN_VERSIONS[0]) -> s # Slice counts on the self-hosted RunsOn runners. Jobs using `--run-in-parallel` run one serial pytest per # test type, so a job never uses more CPUs than it has test types. The core jobs are bound by their slowest # test type and the Non-DB xdist collection gets enough memory on RunsOn, so one slice only removes jobs. -# Low dep providers run on 32-CPU runners: a third of the providers per job keeps those CPUs busy. +# Low dep providers run on 32-CPU runners and the heaviest test types start first, so about 40 test types per +# job keep those CPUs busy while the short ones fill in around the long ones. NUMBER_OF_CORE_SLICES_ON_RUNS_ON = 1 -NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON = 3 +LOW_DEP_TEST_TYPES_PER_JOB_ON_RUNS_ON = 40 + +# On RunsOn the longest-running providers are split further in the individually listed provider test types, +# so every part runs in its own pytest process: `Providers[:]` runs the listed paths (relative to the +# provider's `tests` folder), and `Providers[:other]` runs the rest of the provider's tests. +PROVIDER_TEST_SPLITS: dict[str, dict[str, list[str]]] = { + "amazon": { + "hooks": ["unit/amazon/aws/hooks"], + "operators": ["unit/amazon/aws/operators"], + }, + "google": { + "cloud-hooks": ["unit/google/cloud/hooks"], + "cloud-operators": ["unit/google/cloud/operators"], + }, + "standard": { + "decorators": ["unit/standard/decorators"], + "python": ["unit/standard/operators/test_python.py"], + }, +} +PROVIDER_TEST_SPLIT_OTHER_PART = "other" + + +def get_provider_split_test_types(provider_id: str) -> list[str]: + """Test types running the tests of a split provider, or the provider's single test type.""" + parts = PROVIDER_TEST_SPLITS.get(provider_id) + if not parts: + return [f"Providers[{provider_id}]"] + return [f"Providers[{provider_id}:{part}]" for part in [*parts, PROVIDER_TEST_SPLIT_OTHER_PART]] + # Milestone Tag Assistant configuration # Labels indicating a bug fix PR that should have a milestone diff --git a/dev/breeze/src/airflow_breeze/utils/ci_minutes_estimate.py b/dev/breeze/src/airflow_breeze/utils/ci_minutes_estimate.py index fd51a2140b597..5a3470091f83a 100644 --- a/dev/breeze/src/airflow_breeze/utils/ci_minutes_estimate.py +++ b/dev/breeze/src/airflow_breeze/utils/ci_minutes_estimate.py @@ -125,11 +125,13 @@ def count_provider_test_files(provider_id: str) -> int: def find_tested_provider_ids(provider_test_types: list[str]) -> list[str] | None: """Return the ids of individually tested providers, or None when all providers are tested.""" - provider_ids = [] + provider_ids: list[str] = [] for test_type in provider_test_types: if not test_type.startswith("Providers[") or test_type.startswith("Providers[-"): return None - provider_ids.extend(test_type[len("Providers[") : -1].split(",")) + provider_ids.extend( + provider_id.partition(":")[0] for provider_id in test_type[len("Providers[") : -1].split(",") + ) return provider_ids diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index ef87dd1f6e3a7..f2e2f39f20bc3 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -26,6 +26,8 @@ ALL_TEST_SUITES, ALL_TEST_TYPE, NONE_TEST_TYPE, + PROVIDER_TEST_SPLIT_OTHER_PART, + PROVIDER_TEST_SPLITS, GroupOfTests, SelectiveCoreTestType, all_helm_test_packages, @@ -274,9 +276,14 @@ def are_all_test_paths_excluded( test_dir_paths = [p for p in initial_paths if not p.startswith("-")] if not test_dir_paths: return False - excluded = set(get_excluded_test_provider_folders(python_version)) - suspended = set(get_suspended_test_provider_folders()) - return all(p in excluded | suspended for p in test_dir_paths) + excluded_or_suspended = [ + *get_excluded_test_provider_folders(python_version), + *get_suspended_test_provider_folders(), + ] + return all( + any(p == folder or p.startswith(f"{folder}/") for folder in excluded_or_suspended) + for p in test_dir_paths + ) TEST_TYPE_CORE_MAP_TO_PYTEST_ARGS: dict[str, list[str]] = { @@ -382,7 +389,10 @@ def is_provider_selected_in_test_types(provider_id: str, test_types: list[str]) if provider_id not in excluded: return True elif test_type.startswith(PROVIDERS_LIST_PREFIX): - included = test_type[len(PROVIDERS_LIST_PREFIX) : -1].split(",") + included = [ + included_id.partition(":")[0] + for included_id in test_type[len(PROVIDERS_LIST_PREFIX) : -1].split(",") + ] if provider_id in included: return True else: @@ -447,6 +457,8 @@ def convert_test_type_to_pytest_args( ) providers_with_exclusions.remove(provider_test_to_exclude) return providers_with_exclusions + if test_type.startswith(PROVIDERS_LIST_PREFIX) and ":" in test_type: + return convert_provider_part_to_pytest_args(test_type[len(PROVIDERS_LIST_PREFIX) : -1]) if test_type.startswith(PROVIDERS_LIST_PREFIX): provider_list = test_type[len(PROVIDERS_LIST_PREFIX) : -1].split(",") providers_to_test = [] @@ -482,6 +494,22 @@ def convert_test_type_to_pytest_args( sys.exit(1) +def convert_provider_part_to_pytest_args(provider_part: str) -> list[str]: + """Pytest args for one part of a split provider, given as ``:``.""" + provider_id, _, part = provider_part.partition(":") + parts = PROVIDER_TEST_SPLITS.get(provider_id, {}) + tests_folder = f"providers/{provider_id.replace('.', '/')}/tests" + if part == PROVIDER_TEST_SPLIT_OTHER_PART and parts: + return [ + tests_folder, + *(f"--ignore={tests_folder}/{path}" for paths in parts.values() for path in paths), + ] + if part not in parts: + console_print(f"[error]Unknown part {part!r} of the {provider_id!r} provider tests[/]") + sys.exit(1) + return [f"{tests_folder}/{path}" for path in parts[part]] + + def generate_args_for_pytest( *, test_group: GroupOfTests, diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 6fd832f2ee664..622c5042ca6cf 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -20,6 +20,7 @@ import difflib import itertools import json +import math import os import re import sys @@ -51,10 +52,11 @@ HELM_VERSION, JAVA_SDK_VERSION, KIND_VERSION, + LOW_DEP_TEST_TYPES_PER_JOB_ON_RUNS_ON, NUMBER_OF_CORE_SLICES, NUMBER_OF_CORE_SLICES_ON_RUNS_ON, NUMBER_OF_LOW_DEP_SLICES, - NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON, + PROVIDER_TEST_SPLITS, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, PUBLIC_AMD_RUNNERS, PUBLIC_ARM_RUNNERS, @@ -74,6 +76,7 @@ all_helm_test_packages, all_selective_core_test_types, get_airflow_version, + get_provider_split_test_types, providers_test_type, ) from airflow_breeze.utils.console import console_print, get_console @@ -1979,6 +1982,12 @@ def _get_individual_providers_list(self): {f"Providers[{provider}]" for provider in get_available_distributions(include_not_ready=True)} ) self._filter_platform_excluded_test_types(current_test_types) + if not self._is_on_runs_on: + return current_test_types + for provider_id in PROVIDER_TEST_SPLITS: + if f"Providers[{provider_id}]" in current_test_types: + current_test_types.remove(f"Providers[{provider_id}]") + current_test_types.update(get_provider_split_test_types(provider_id)) return current_test_types @cached_property @@ -1990,7 +1999,9 @@ def individual_providers_test_types_list_as_strings_in_json(self) -> str | None: if not current_test_types: return None number_of_slices = ( - NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON if self._is_on_runs_on else NUMBER_OF_LOW_DEP_SLICES + math.ceil(len(current_test_types) / LOW_DEP_TEST_TYPES_PER_JOB_ON_RUNS_ON) + if self._is_on_runs_on + else NUMBER_OF_LOW_DEP_SLICES ) if len(current_test_types) < number_of_slices: return json.dumps(_get_test_list_as_json([current_test_types])) diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py b/dev/breeze/tests/test_pytest_args_for_test_types.py index 4332d8dceb7d6..233a04f558397 100644 --- a/dev/breeze/tests/test_pytest_args_for_test_types.py +++ b/dev/breeze/tests/test_pytest_args_for_test_types.py @@ -214,6 +214,45 @@ def test_pytest_args_for_missing_provider(): ) +@pytest.mark.parametrize( + ("test_type", "pytest_args"), + [ + pytest.param( + "Providers[amazon:hooks]", + ["providers/amazon/tests/unit/amazon/aws/hooks"], + id="named part runs its paths", + ), + pytest.param( + "Providers[standard:python]", + ["providers/standard/tests/unit/standard/operators/test_python.py"], + id="a part can be a single file", + ), + pytest.param( + "Providers[amazon:other]", + [ + "providers/amazon/tests", + "--ignore=providers/amazon/tests/unit/amazon/aws/hooks", + "--ignore=providers/amazon/tests/unit/amazon/aws/operators", + ], + id="other part ignores the named parts", + ), + ], +) +def test_pytest_args_for_split_provider_parts(test_type: str, pytest_args: list[str]): + assert ( + convert_test_type_to_pytest_args(test_group=GroupOfTests.PROVIDERS, test_type=test_type) + == pytest_args + ) + + +@pytest.mark.parametrize( + "test_type", ["Providers[amazon:missing]", "Providers[airbyte:other]"], ids=["unknown part", "unsplit"] +) +def test_pytest_args_for_unknown_provider_part(test_type: str): + with pytest.raises(SystemExit): + convert_test_type_to_pytest_args(test_group=GroupOfTests.PROVIDERS, test_type=test_type) + + @pytest.mark.parametrize( ("test_group", "parallel_test_types", "folders"), [ diff --git a/dev/breeze/tests/test_run_test_args.py b/dev/breeze/tests/test_run_test_args.py index b146e61d3059b..da3b19dd1473a 100644 --- a/dev/breeze/tests/test_run_test_args.py +++ b/dev/breeze/tests/test_run_test_args.py @@ -23,6 +23,7 @@ from airflow_breeze.commands.testing_commands import _run_test, pull_testcontainers_images from airflow_breeze.global_constants import TESTCONTAINERS_IMAGES_BY_PROVIDER, GroupOfTests from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.run_tests import is_provider_selected_in_test_types @pytest.fixture(autouse=True) @@ -127,6 +128,34 @@ def test_test_is_skipped_when_all_providers_excluded_for_python_version( mock_run_command.assert_not_called() +def test_split_provider_part_is_skipped_when_provider_excluded_for_python_version( + mock_run_command, mock_get_excluded_provider_folders +): + mock_get_excluded_provider_folders.return_value = ["amazon"] + return_code, message = _run_test( + shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="Providers[amazon:hooks]"), + extra_pytest_args=(), + python_version="3.14", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + assert return_code == 0 + assert "Skipped" in message + mock_run_command.assert_not_called() + + +@pytest.mark.parametrize( + ("test_types", "expected"), + [ + pytest.param(["Providers[amazon:hooks]"], True, id="part of the provider"), + pytest.param(["Providers[google:cloud-hooks]"], False, id="part of another provider"), + ], +) +def test_split_provider_part_selects_its_provider(test_types: list[str], expected: bool): + assert is_provider_selected_in_test_types("amazon", test_types) is expected + + def test_test_is_not_skipped_when_some_providers_remain(mock_run_command, mock_get_excluded_provider_folders): """When only some providers are excluded, the test should still run.""" mock_get_excluded_provider_folders.return_value = ["http"] diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 311f3658a62f6..4a6016d78dc63 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -39,10 +39,10 @@ NUMBER_OF_CORE_SLICES, NUMBER_OF_CORE_SLICES_ON_RUNS_ON, NUMBER_OF_LOW_DEP_SLICES, - NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, GithubEvents, get_airflow_version, + get_provider_split_test_types, ) from airflow_breeze.utils.functools_cache import clearable_cache from airflow_breeze.utils.packages import get_available_distributions @@ -113,6 +113,12 @@ f"Providers[{provider}]" for provider in get_available_distributions(include_not_ready=True) ] +LIST_OF_ALL_PROVIDER_TESTS_ON_RUNS_ON = [ + test_type + for provider in get_available_distributions(include_not_ready=True) + for test_type in get_provider_split_test_types(provider) +] + LIST_OF_ALL_PROVIDER_TESTS_AS_JSON = json.dumps( _get_test_list_as_json(_split_list(sorted(LIST_OF_ALL_PROVIDER_TESTS), 5)) ) @@ -2253,20 +2259,28 @@ def test_list_splitting(): @pytest.mark.parametrize( - ("github_event", "github_context_dict", "expected_core_slices", "expected_low_dep_slices"), + ( + "github_event", + "github_context_dict", + "expected_core_slices", + "expected_low_dep_slices", + "expected_low_dep_test_types", + ), [ pytest.param( GithubEvents.PULL_REQUEST, {}, NUMBER_OF_CORE_SLICES, NUMBER_OF_LOW_DEP_SLICES, + LIST_OF_ALL_PROVIDER_TESTS, id="public runners", ), pytest.param( GithubEvents.PUSH, {"run_id": 42}, NUMBER_OF_CORE_SLICES_ON_RUNS_ON, - NUMBER_OF_LOW_DEP_SLICES_ON_RUNS_ON, + 3, + LIST_OF_ALL_PROVIDER_TESTS_ON_RUNS_ON, id="RunsOn runners", ), ], @@ -2276,6 +2290,7 @@ def test_number_of_test_type_slices_follows_runners( github_context_dict: dict[str, Any], expected_core_slices: int, expected_low_dep_slices: int, + expected_low_dep_test_types: list[str], ): selective_checks = SelectiveChecks( pr_labels=("full tests needed",), @@ -2291,7 +2306,9 @@ def test_number_of_test_type_slices_follows_runners( assert sorted(" ".join(s["test_types"] for s in core_slices).split()) == sorted( ALL_CI_SELECTIVE_TEST_TYPES.split() ) - assert sum(len(s["test_types"].split()) for s in low_dep_slices) == len(LIST_OF_ALL_PROVIDER_TESTS) + assert sorted(" ".join(s["test_types"] for s in low_dep_slices).split()) == sorted( + expected_low_dep_test_types + ) @pytest.mark.parametrize( @@ -2312,7 +2329,7 @@ def test_number_of_test_type_slices_follows_runners( pytest.param( GithubEvents.PUSH, {"run_id": 42}, - sorted(LIST_OF_ALL_PROVIDER_TESTS), + sorted(LIST_OF_ALL_PROVIDER_TESTS_ON_RUNS_ON), id="RunsOn runners list each provider", ), ], @@ -2353,6 +2370,23 @@ def test_providers_xdist_test_types_are_not_split(platform: str, expected_test_t assert [s["test_types"] for s in slices] == [expected_test_types] +@patch.object(SelectiveChecks, "_is_on_runs_on", new=property(lambda self: True)) +def test_individual_providers_split_the_longest_running_providers_on_runs_on(): + selective_checks = SelectiveChecks( + files=("providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py",), + commit_ref=NEUTRAL_COMMIT, + default_branch="main", + github_event=GithubEvents.PULL_REQUEST, + ) + slices = json.loads(selective_checks.individual_providers_test_types_list_as_strings_in_json) + assert len(slices) == 1 + test_types = slices[0]["test_types"].split() + assert "Providers[amazon]" not in test_types + assert {"Providers[amazon:hooks]", "Providers[amazon:operators]", "Providers[amazon:other]"} <= set( + test_types + ) + + def test_excluded_providers(): stderr = SelectiveChecks( files=(), diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 1e61d0bd85608..c487ea60a757b 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -466,6 +466,8 @@ function check_force_lowest_dependencies() { local provider_id # shellcheck disable=SC2001 provider_id=$(echo "${TEST_TYPE}" | sed 's/Providers\[\(.*\)\]/\1/') + # A split provider's test type names one part of its tests: Providers[:] + provider_id=${provider_id%%:*} echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${provider_id}${COLOR_RESET}" echo From c53e217a3bb81c5b8a16c18937d19b8e58966003 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 02:55:07 +0100 Subject: [PATCH 05/13] Keep split provider test types out of compose project and artifact names Docker Compose project names and GitHub artifact names reject the colon that names a part of a split provider. Generated-by: Claude Opus 5 --- .../src/airflow_breeze/utils/run_tests.py | 3 ++- .../airflow_breeze/utils/selective_checks.py | 4 +++- .../tests/test_pytest_args_for_test_types.py | 18 +++++++++++++++++- dev/breeze/tests/test_selective_checks.py | 1 + 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index f2e2f39f20bc3..045d2ae9b71ce 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -203,7 +203,8 @@ def run_docker_compose_tests( def file_name_from_test_type(test_type: str): test_type_no_brackets = test_type.lower().replace("[", "_").replace("]", "") - return re.sub("[,.]", "_", test_type_no_brackets)[:30] + # Also names the docker compose project, which allows only lowercase letters, digits, "-" and "_". + return re.sub("[^a-z0-9_-]", "_", test_type_no_brackets)[:30] def test_paths(test_type: str, backend: str) -> tuple[str, str, str]: diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 622c5042ca6cf..2c5bcb7f63ff5 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -888,11 +888,13 @@ def _get_test_type_description(provider_test_types: list[str]) -> str: first_provider = first_provider.replace("Providers[", "").replace("]", "") if last_provider.startswith("Providers["): last_provider = last_provider.replace("Providers[", "").replace("]", "") - return ( + description = ( f"{first_provider[:13]}...{last_provider[:13]}" if first_provider != last_provider else (first_provider[:29]) ) + # The description ends up in artifact names (via JOB_ID), which must not contain ":". + return description.replace(":", "-") def _get_test_list_as_json(list_of_list_of_types: list[list[str]]) -> list[dict[str, str]] | None: diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py b/dev/breeze/tests/test_pytest_args_for_test_types.py index 233a04f558397..8db353ab634cf 100644 --- a/dev/breeze/tests/test_pytest_args_for_test_types.py +++ b/dev/breeze/tests/test_pytest_args_for_test_types.py @@ -20,7 +20,11 @@ from airflow_breeze.global_constants import GroupOfTests from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH -from airflow_breeze.utils.run_tests import convert_parallel_types_to_folders, convert_test_type_to_pytest_args +from airflow_breeze.utils.run_tests import ( + convert_parallel_types_to_folders, + convert_test_type_to_pytest_args, + file_name_from_test_type, +) def _all_providers() -> list[str]: @@ -433,3 +437,15 @@ def xtest_wrong_types_for_parallel_test_types(test_group: GroupOfTests, parallel test_group=test_group, parallel_test_types_list=parallel_test_types.split(" "), ) + + +@pytest.mark.parametrize( + ("test_type", "file_name"), + [ + pytest.param("Providers[amazon:hooks]", "providers_amazon_hooks", id="split provider part"), + pytest.param("Providers[google:cloud-hooks]", "providers_google_cloud-hooks", id="part with a dash"), + pytest.param("Providers[apache.hive,http]", "providers_apache_hive_http", id="provider list"), + ], +) +def test_file_name_from_test_type_is_a_valid_compose_project_name(test_type: str, file_name: str): + assert file_name_from_test_type(test_type) == file_name diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4a6016d78dc63..c21ad0f4c2417 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2380,6 +2380,7 @@ def test_individual_providers_split_the_longest_running_providers_on_runs_on(): ) slices = json.loads(selective_checks.individual_providers_test_types_list_as_strings_in_json) assert len(slices) == 1 + assert ":" not in slices[0]["description"] test_types = slices[0]["test_types"].split() assert "Providers[amazon]" not in test_types assert {"Providers[amazon:hooks]", "Providers[amazon:operators]", "Providers[amazon:other]"} <= set( From a6bc3b8bf088f15144ec838524a800eae1eaa3c3 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 03:10:56 +0100 Subject: [PATCH 06/13] Skip split provider test types whose provider is ignored A split provider part selects a folder inside the provider's tests, and pytest runs an explicitly selected path even when its provider folder is ignored, so a skipped or excluded provider's parts would still run. Generated-by: Claude Opus 5 --- .../src/airflow_breeze/commands/testing_commands.py | 9 ++++++++- dev/breeze/tests/test_run_test_args.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index cbfb6377736bf..2028a36d77ea9 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -275,7 +275,14 @@ def _run_test( # Skip "FOLDER" in case "--ignore=FOLDER" is passed as an argument # Which might be the case if we are ignoring some providers during compatibility checks pytest_args_before_skip = pytest_args - pytest_args = [arg for arg in pytest_args if f"--ignore={arg}" not in pytest_args] + # Paths under an ignored folder go too: an explicitly selected path overrides an ancestor's --ignore. + ignored_paths = [arg.removeprefix("--ignore=") for arg in pytest_args if arg.startswith("--ignore=")] + pytest_args = [ + arg + for arg in pytest_args + if arg.startswith("-") + or not any(arg == ignored or arg.startswith(f"{ignored}/") for ignored in ignored_paths) + ] # If no test directory is left (all positional args were excluded/ignored), skip if pytest_args_before_skip != pytest_args and pytest_args[0].startswith("--"): return 0, f"Skipped test, no tests needed: {shell_params.test_type}" diff --git a/dev/breeze/tests/test_run_test_args.py b/dev/breeze/tests/test_run_test_args.py index da3b19dd1473a..840862ecf2668 100644 --- a/dev/breeze/tests/test_run_test_args.py +++ b/dev/breeze/tests/test_run_test_args.py @@ -110,6 +110,19 @@ def test_test_is_skipped_if_all_are_ignored(mock_run_command): mock_run_command.assert_called_once() # called only to compose down +def test_split_provider_part_is_skipped_if_its_provider_is_ignored(mock_run_command): + _run_test( + shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="Providers[amazon:hooks]"), + extra_pytest_args=("--ignore=providers/amazon/tests",), + python_version="3.9", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + + mock_run_command.assert_called_once() # called only to compose down + + def test_test_is_skipped_when_all_providers_excluded_for_python_version( mock_run_command, mock_get_excluded_provider_folders ): From 237a19f06d80e5ce054b41f6ab3383420ce328ac Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 03:40:57 +0100 Subject: [PATCH 07/13] Fit RunsOn runner sizes to the work of each job and record their CPU and memory use Several jobs ran a single process, or a minute or two of work, on 32-CPU or 8-CPU runners, and fourteen Helm unit test jobs each spent more time preparing the image than running their tests. Charting each job's CPU and memory use lets the size classes be set from measurements rather than from how the jobs are expected to behave. Generated-by: Claude Opus 5 --- .../actions/runs-on-cost-summary/action.yml | 4 +++- .../workflows/additional-prod-image-tests.yml | 8 ++++---- .github/workflows/basic-tests.yml | 4 ++-- .github/workflows/ci-amd.yml | 10 +++++----- .github/workflows/ci-arm.yml | 10 +++++----- .github/workflows/finalize-tests.yml | 2 +- .github/workflows/helm-tests.yml | 2 +- dev/breeze/doc/ci/04_selective_checks.md | 11 ++++++++--- .../airflow_breeze/utils/selective_checks.py | 5 +++++ dev/breeze/tests/test_selective_checks.py | 19 +++++++++++++++++++ 10 files changed, 53 insertions(+), 22 deletions(-) diff --git a/.github/actions/runs-on-cost-summary/action.yml b/.github/actions/runs-on-cost-summary/action.yml index fd0d2007f103c..d31493ae7462f 100644 --- a/.github/actions/runs-on-cost-summary/action.yml +++ b/.github/actions/runs-on-cost-summary/action.yml @@ -19,7 +19,7 @@ # Puts the job's EC2 cost next to the GitHub-hosted price in the job summary. Used by every job # that can run on RunsOn: through the breeze action, or directly by jobs that do not install Breeze. name: 'RunsOn cost summary' -description: 'Add the EC2 cost of a RunsOn job to its summary; a no-op elsewhere.' +description: 'Add the EC2 cost and CPU and memory use of a RunsOn job to its summary; a no-op elsewhere.' runs: using: 'composite' steps: @@ -28,3 +28,5 @@ runs: uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 with: show_costs: summary + # Charts the job's CPU and memory use in its summary, to size its runner class from measurements. + metrics: cpu,memory diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index a0109fb65a642..47657cbe67834 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -201,7 +201,7 @@ jobs: timeout-minutes: 60 name: "Docker Compose quick start with PROD image verifying" if: inputs.run-generic-prod-image-tests == 'true' - runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -331,7 +331,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Event driven e2e test" - runners: ${{ format(inputs.runners, '32+64', '128+256') }} + runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -359,7 +359,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Go SDK e2e test" - runners: ${{ format(inputs.runners, '32+64', '128+256') }} + runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -373,7 +373,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "TypeScript SDK e2e test" - runners: ${{ format(inputs.runners, '32+64', '128+256') }} + runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index d1b84c5a1567d..86f59d5f1ac35 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -148,7 +148,7 @@ jobs: # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. UV_PYTHON: "${{ inputs.default-python-version }}" SHARED_DISTRIBUTIONS: ${{ join(fromJSON(inputs.shared-distributions-as-json), ' ') }} - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -255,7 +255,7 @@ jobs: check-translation-completness: timeout-minutes: 15 name: "Check translation completeness" - runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 27d8a07fd7c42..8fcbfbdcec342 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -1208,7 +1208,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} + runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1293,7 +1293,7 @@ jobs: tests-ts-sdk: name: "TypeScript SDK tests" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 15 permissions: contents: read @@ -1329,7 +1329,7 @@ jobs: tests-java-sdk: name: "Java SDK tests" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} timeout-minutes: 15 permissions: contents: read @@ -1361,7 +1361,7 @@ jobs: build-java-sdk-docs: name: "Java SDK docs" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 30 permissions: contents: read @@ -1774,7 +1774,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 6c9f36e4f43cc..1f6b192b2b320 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -1089,7 +1089,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} + runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1174,7 +1174,7 @@ jobs: tests-ts-sdk: name: "TypeScript SDK tests" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 15 permissions: contents: read @@ -1210,7 +1210,7 @@ jobs: tests-java-sdk: name: "Java SDK tests" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} timeout-minutes: 15 permissions: contents: read @@ -1242,7 +1242,7 @@ jobs: build-java-sdk-docs: name: "Java SDK docs" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 30 permissions: contents: read @@ -1653,7 +1653,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 9715a8cfc6124..d9d5c463b154f 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -80,7 +80,7 @@ permissions: contents: read jobs: update-constraints: - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }} timeout-minutes: 80 name: "Update constraints" permissions: diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index cf140b98c623e..f8000a7c4dd19 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -98,7 +98,7 @@ jobs: tests-helm-release: timeout-minutes: 80 name: "Release Helm" - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 6e74b2fff5f3d..6727910026fb3 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -671,7 +671,7 @@ GitHub Actions to pass the list of parameters to a command to execute | generated-dependencies-changed | Whether generated dependencies have changed ("true"/"false") | false | | | has-migrations | Whether the PR has migrations ("true"/"false") | false | | | hatch-build-changed | When hatch build.py changed in the PR. | false | | -| helm-test-packages-list-as-string | List of helm packages to test as JSON array | \["airflow_aux", "airflow_core"\] | * | +| helm-test-packages | List of helm packages to test as JSON array (`["All"]` on RunsOn: one job runs them all) | \["airflow_aux", "airflow_core"\] | * | | helm-version | Which Helm version to use for tests | v3.15.3 | | | in-memory-runner-type | Like runner-type, but the RunsOn template puts docker in RAM (`extras=tmpfs`) unless disabled | \["ubuntu-22.04"\] | | | include-success-outputs | Whether to include outputs of successful parallel tests ("true"/"false") | false | | @@ -771,7 +771,11 @@ the larger instance types too; xdist follows the host's CPUs, so a larger runner Jobs using `--run-in-parallel` (the DB, lowest-dependencies and special tests) run one serial pytest per test type instead, so they never use more CPUs than they have test types, and jobs running a single pytest (integration and system tests) use one. The size classes follow that: the xdist jobs and every job -running provider tests in parallel take `'32+64', '128+256'`, the others `'8+16', '32+64'` or smaller. +running provider tests in parallel take `'32+64', '128+256'`, the others `'8+16', '32+64'` or smaller: +a job running one process (an end-to-end test against one Airflow stack, a Gradle test run) takes +`'8+16', '32+64'`, and a job with a minute or two of work takes `'4+8', '16+32'`. On RunsOn the Helm unit +tests run all their packages in one job per Kubernetes version, since each package takes about a minute +under xdist against two to three minutes of preparing the image. On RunsOn `providers-test-types-list-as-strings-in-json` lists each provider as its own test type, in one job, instead of the long-running providers plus one group of all the others, so the job is not left waiting on that group. The Non-DB xdist jobs read `providers-xdist-test-types-list-as-strings-in-json` @@ -791,7 +795,8 @@ the on-demand launch and whole-job rerun a capacity miss or interruption costs; on a public label the same call returns the label unchanged. Steps that only make sense on RunsOn are gated on `runner.environment == 'self-hosted'`: -* The EC2 cost summary of each job. +* The EC2 cost summary of each job, with charts of the job's CPU and memory use (`metrics: cpu,memory` of + `runs-on/action`), the measurements to size a job's runner class by. * The docker address pools (`scripts/ci/configure_docker_address_pools.sh`). Every test type of a parallel run is a docker compose project with a network of its own, and Docker's default pools fit about 30 networks, fewer than a RunsOn runner runs test types at once; the script splits the same range into diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 2c5bcb7f63ff5..adc18a5f7bd8f 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -35,6 +35,7 @@ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( ALL_PYTHON_MAJOR_MINOR_VERSIONS, + ALL_TEST_TYPE, APACHE_AIRFLOW_GITHUB_REPOSITORY, CI_AMD_PLATFORM, CI_ARM_PLATFORM, @@ -2285,6 +2286,10 @@ def disable_airflow_repo_cache(self) -> bool: @cached_property def helm_test_packages(self) -> str: + # A Helm test package takes about a minute under xdist on a RunsOn runner, against two to three + # minutes of preparing the image, so RunsOn runs all of them in one job. + if self._is_on_runs_on: + return json.dumps([ALL_TEST_TYPE]) return json.dumps(all_helm_test_packages()) @cached_property diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index c21ad0f4c2417..ab266c8afc6c1 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -41,6 +41,7 @@ NUMBER_OF_LOW_DEP_SLICES, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, GithubEvents, + all_helm_test_packages, get_airflow_version, get_provider_split_test_types, ) @@ -2388,6 +2389,24 @@ def test_individual_providers_split_the_longest_running_providers_on_runs_on(): ) +@pytest.mark.parametrize( + ("github_event", "github_context_dict", "expected_packages"), + [ + pytest.param(GithubEvents.PULL_REQUEST, {}, all_helm_test_packages(), id="public runners"), + pytest.param(GithubEvents.PUSH, {"run_id": 42}, ["All"], id="RunsOn runners"), + ], +) +def test_helm_test_packages_follow_runners( + github_event: GithubEvents, github_context_dict: dict[str, Any], expected_packages: list[str] +): + selective_checks = SelectiveChecks( + github_event=github_event, + github_repository="apache/airflow", + github_context_dict=github_context_dict, + ) + assert json.loads(selective_checks.helm_test_packages) == expected_packages + + def test_excluded_providers(): stderr = SelectiveChecks( files=(), From e70d8d7261bb5bb8762f56868ad961722351d1ca Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 03:49:27 +0100 Subject: [PATCH 08/13] Skip ignored test selections by the paths the test type selects The check for an empty run treated the first remaining argument, or any path-like argument, as a test selection, so option values such as the "8" of "-n 8" or a path passed to --rootdir decided whether tests ran. It now compares the ignored folders with the paths the test type itself selects, and passes paths the user names explicitly to pytest as they are. Generated-by: Claude Opus 5 --- .../commands/testing_commands.py | 29 ++++++----- .../src/airflow_breeze/utils/run_tests.py | 52 +++++++++++++------ dev/breeze/tests/test_run_test_args.py | 38 ++++++++++++++ 3 files changed, 91 insertions(+), 28 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 2028a36d77ea9..8934b40cbf6a9 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -129,6 +129,8 @@ are_all_test_paths_excluded, file_name_from_test_type, generate_args_for_pytest, + get_selected_test_paths, + is_path_under_any, is_provider_selected_in_test_types, run_docker_compose_tests, ) @@ -272,20 +274,21 @@ def _run_test( integration=shell_params.integration, ) pytest_args.extend(extra_pytest_args) - # Skip "FOLDER" in case "--ignore=FOLDER" is passed as an argument - # Which might be the case if we are ignoring some providers during compatibility checks - pytest_args_before_skip = pytest_args - # Paths under an ignored folder go too: an explicitly selected path overrides an ancestor's --ignore. - ignored_paths = [arg.removeprefix("--ignore=") for arg in pytest_args if arg.startswith("--ignore=")] - pytest_args = [ - arg - for arg in pytest_args - if arg.startswith("-") - or not any(arg == ignored or arg.startswith(f"{ignored}/") for ignored in ignored_paths) - ] - # If no test directory is left (all positional args were excluded/ignored), skip - if pytest_args_before_skip != pytest_args and pytest_args[0].startswith("--"): + # Drop the test paths the test type selects under an "--ignore=FOLDER" argument, which is the case when + # some providers are skipped during compatibility checks: pytest runs an explicitly selected path even + # when it or an ancestor is ignored. Paths the user passes explicitly go to pytest as they are. + ignored_folders = [arg.removeprefix("--ignore=") for arg in pytest_args if arg.startswith("--ignore=")] + selected_paths = get_selected_test_paths( + test_group=shell_params.test_group, + test_type=shell_params.test_type, + skip_db_tests=shell_params.skip_db_tests, + parallel_test_types_list=shell_params.parallel_test_types_list, + integration=shell_params.integration, + ) + ignored_selections = {path for path in selected_paths if is_path_under_any(path, ignored_folders)} + if selected_paths and ignored_selections == set(selected_paths): return 0, f"Skipped test, no tests needed: {shell_params.test_type}" + pytest_args = [arg for arg in pytest_args if arg not in ignored_selections] run_cmd.extend(pytest_args) try: remove_docker_networks(networks=[f"{compose_project_name}_default"]) diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 045d2ae9b71ce..2a7f84b8cf7a8 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -244,23 +244,17 @@ def get_excluded_test_provider_folders(python_version: str) -> list[str]: return get_test_folders(excluded_folders) -def are_all_test_paths_excluded( +def get_selected_test_paths( *, test_group: GroupOfTests, test_type: str, - python_version: str, skip_db_tests: bool = False, parallel_test_types_list: list[str] | None = None, integration: tuple | None = None, -) -> bool: - """Check if all test paths for the given test type are excluded or suspended. - - Uses the same provider.yaml exclusion data as generate_args_for_pytest to determine - whether every test directory that would be generated for this test type is in the - excluded or suspended provider set. +) -> list[str]: + """The test paths generate_args_for_pytest selects for the test type, without the pytest options. - Returns False when test_type is "None" (user provides explicit paths via extra_pytest_args) - or when any test path remains after exclusions. + Empty when test_type is "None": the user then provides explicit paths via extra_pytest_args. """ if skip_db_tests and parallel_test_types_list: initial_paths = convert_parallel_types_to_folders( @@ -274,17 +268,45 @@ def are_all_test_paths_excluded( integration=integration, ) # Filter to only actual test directory paths (not pytest flags like -m, --include-quarantined) - test_dir_paths = [p for p in initial_paths if not p.startswith("-")] + return [p for p in initial_paths if not p.startswith("-")] + + +def is_path_under_any(path: str, folders: list[str]) -> bool: + return any(path == folder or path.startswith(f"{folder}/") for folder in folders) + + +def are_all_test_paths_excluded( + *, + test_group: GroupOfTests, + test_type: str, + python_version: str, + skip_db_tests: bool = False, + parallel_test_types_list: list[str] | None = None, + integration: tuple | None = None, +) -> bool: + """Check if all test paths for the given test type are excluded or suspended. + + Uses the same provider.yaml exclusion data as generate_args_for_pytest to determine + whether every test directory that would be generated for this test type is in the + excluded or suspended provider set. + + Returns False when test_type is "None" (user provides explicit paths via extra_pytest_args) + or when any test path remains after exclusions. + """ + test_dir_paths = get_selected_test_paths( + test_group=test_group, + test_type=test_type, + skip_db_tests=skip_db_tests, + parallel_test_types_list=parallel_test_types_list, + integration=integration, + ) if not test_dir_paths: return False excluded_or_suspended = [ *get_excluded_test_provider_folders(python_version), *get_suspended_test_provider_folders(), ] - return all( - any(p == folder or p.startswith(f"{folder}/") for folder in excluded_or_suspended) - for p in test_dir_paths - ) + return all(is_path_under_any(p, excluded_or_suspended) for p in test_dir_paths) TEST_TYPE_CORE_MAP_TO_PYTEST_ARGS: dict[str, list[str]] = { diff --git a/dev/breeze/tests/test_run_test_args.py b/dev/breeze/tests/test_run_test_args.py index 840862ecf2668..d48880496c015 100644 --- a/dev/breeze/tests/test_run_test_args.py +++ b/dev/breeze/tests/test_run_test_args.py @@ -123,6 +123,44 @@ def test_split_provider_part_is_skipped_if_its_provider_is_ignored(mock_run_comm mock_run_command.assert_called_once() # called only to compose down +def test_test_is_skipped_if_its_only_path_is_ignored_next_to_path_valued_options(mock_run_command): + _run_test( + shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="Providers[amazon:hooks]"), + extra_pytest_args=( + "--ignore=providers/amazon/tests", + "--basetemp", + "/tmp/breeze-pytest", + "--rootdir", + "airflow-core/tests", + ), + python_version="3.9", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + + mock_run_command.assert_called_once() # called only to compose down + + +def test_explicit_paths_go_to_pytest_as_they_are(mock_run_command): + _run_test( + shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="None"), + extra_pytest_args=( + "providers/amazon/tests/unit/amazon/aws/hooks", + "providers/http/tests", + "--ignore=providers/amazon/tests", + ), + python_version="3.9", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + + run_cmd = mock_run_command.call_args_list[1].args[0] + assert "providers/http/tests" in run_cmd + assert "providers/amazon/tests/unit/amazon/aws/hooks" in run_cmd + + def test_test_is_skipped_when_all_providers_excluded_for_python_version( mock_run_command, mock_get_excluded_provider_folders ): From 9117f791f1db145bf90f094b9a01f4a6a33499db Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 04:28:54 +0100 Subject: [PATCH 09/13] Reject split provider test types where all test types share one pytest Under xdist every test type runs in one pytest process, where the --ignore arguments of a provider's other part would silently be dropped or would remove its sibling parts, so the wrong tests would run. Generated-by: Claude Opus 5 --- dev/breeze/src/airflow_breeze/utils/run_tests.py | 6 ++++++ dev/breeze/tests/test_pytest_args_for_test_types.py | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 2a7f84b8cf7a8..986b55e15d8ce 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -669,6 +669,12 @@ def generate_args_for_pytest( def convert_parallel_types_to_folders(test_group: GroupOfTests, parallel_test_types_list: list[str]): + # All test types share one pytest here, where the --ignore of a ":other" part would also drop + # the provider's other parts, so a split provider runs as a whole: Providers[]. + split_parts = [test_type for test_type in parallel_test_types_list if ":" in test_type] + if split_parts: + console_print(f"[error]Split provider test types cannot run in one pytest process: {split_parts}[/]") + sys.exit(1) args = [] for _test_type in parallel_test_types_list: args.extend( diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py b/dev/breeze/tests/test_pytest_args_for_test_types.py index 8db353ab634cf..88444958690ba 100644 --- a/dev/breeze/tests/test_pytest_args_for_test_types.py +++ b/dev/breeze/tests/test_pytest_args_for_test_types.py @@ -257,6 +257,14 @@ def test_pytest_args_for_unknown_provider_part(test_type: str): convert_test_type_to_pytest_args(test_group=GroupOfTests.PROVIDERS, test_type=test_type) +def test_folders_for_parallel_test_types_reject_split_provider_parts(): + with pytest.raises(SystemExit): + convert_parallel_types_to_folders( + test_group=GroupOfTests.PROVIDERS, + parallel_test_types_list=["Providers[amazon:hooks]", "Providers[amazon:other]"], + ) + + @pytest.mark.parametrize( ("test_group", "parallel_test_types", "folders"), [ From 44cc2be9388d6b7522ae09bcb0d88eeb1153b880 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 04:43:59 +0100 Subject: [PATCH 10/13] Count only test folders as the selections of a test type Test suites such as All-Postgres add pytest options with values, which were taken for selected test paths, so a run whose every test folder was ignored still started pytest with nothing selected. Generated-by: Claude Opus 5 --- dev/breeze/src/airflow_breeze/utils/run_tests.py | 16 +++++++++------- dev/breeze/tests/test_run_test_args.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index 986b55e15d8ce..5391df135be7b 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -267,12 +267,17 @@ def get_selected_test_paths( test_type=test_type, integration=integration, ) - # Filter to only actual test directory paths (not pytest flags like -m, --include-quarantined) - return [p for p in initial_paths if not p.startswith("-")] + # Only the test folders: test suites such as "All-Postgres" add pytest options with values too + return [p for p in initial_paths if is_path_under_any(p, get_all_test_group_folders())] + + +def get_all_test_group_folders() -> list[str]: + return [folder for group_folders in TEST_GROUP_TO_TEST_FOLDERS.values() for folder in group_folders] def is_path_under_any(path: str, folders: list[str]) -> bool: - return any(path == folder or path.startswith(f"{folder}/") for folder in folders) + path = path.rstrip("/") + return any(path == folder.rstrip("/") or path.startswith(f"{folder.rstrip('/')}/") for folder in folders) def are_all_test_paths_excluded( @@ -683,9 +688,6 @@ def convert_parallel_types_to_folders(test_group: GroupOfTests, parallel_test_ty test_type=_test_type, ) ) - all_test_prefixes: list[str] = [] + all_test_prefixes = get_all_test_group_folders() # leave only folders, strip --pytest-args that exclude some folders with `-` prefix - for group_folders in TEST_GROUP_TO_TEST_FOLDERS.values(): - for group_folder in group_folders: - all_test_prefixes.append(group_folder) return [arg for arg in args if any(arg.startswith(prefix) for prefix in all_test_prefixes)] diff --git a/dev/breeze/tests/test_run_test_args.py b/dev/breeze/tests/test_run_test_args.py index d48880496c015..bbcbb598e21ca 100644 --- a/dev/breeze/tests/test_run_test_args.py +++ b/dev/breeze/tests/test_run_test_args.py @@ -142,6 +142,19 @@ def test_test_is_skipped_if_its_only_path_is_ignored_next_to_path_valued_options mock_run_command.assert_called_once() # called only to compose down +def test_test_suite_is_skipped_if_all_its_paths_are_ignored(mock_run_command): + _run_test( + shell_params=ShellParams(test_group=GroupOfTests.CORE, test_type="All-Postgres"), + extra_pytest_args=("--ignore=airflow-core/tests/unit",), + python_version="3.9", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + + mock_run_command.assert_called_once() # called only to compose down + + def test_explicit_paths_go_to_pytest_as_they_are(mock_run_command): _run_test( shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="None"), From aea92c5dc36f724f4dd407e1844f4005423cc435 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 04:53:20 +0100 Subject: [PATCH 11/13] Run only its own test type in each worker of a parallel test run Each worker kept the whole list of test types, so with --skip-db-tests every worker selected the folders of all of them. Generated-by: Claude Opus 5 --- .../src/airflow_breeze/params/shell_params.py | 2 ++ dev/breeze/tests/test_shell_params.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 6d1170848259a..6bfbb062bfdde 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -264,6 +264,8 @@ class ShellParams: def clone_with_test(self, test_type: str) -> ShellParams: new_params = deepcopy(self) new_params.test_type = test_type + # The clone runs this one test type: kept, the list would make it select every test type's folders. + new_params.parallel_test_types_list = [] return new_params @cached_property diff --git a/dev/breeze/tests/test_shell_params.py b/dev/breeze/tests/test_shell_params.py index 78e3d56d69143..630abecc953fe 100644 --- a/dev/breeze/tests/test_shell_params.py +++ b/dev/breeze/tests/test_shell_params.py @@ -29,7 +29,7 @@ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.commands.common_options import option_project_name -from airflow_breeze.global_constants import MOUNT_SELECTED, PYCACHE_PREFIX_IN_CONTAINER +from airflow_breeze.global_constants import MOUNT_SELECTED, PYCACHE_PREFIX_IN_CONTAINER, GroupOfTests from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.path_utils import ( SCRIPTS_CI_DOCKER_COMPOSE_BASE_PATH, @@ -425,3 +425,16 @@ def test_sphinx_airflow_theme_is_mounted_only_when_a_path_is_given(theme_path: s ) # Compose interpolates the bind source from this variable, so it must be exported together with the file. assert shell_params.env_variables_for_docker_commands["SPHINX_AIRFLOW_THEME_PATH"] == theme_path + + +def test_clone_with_test_runs_only_its_own_test_type(): + shell_params = ShellParams( + test_group=GroupOfTests.PROVIDERS, + parallel_test_types_list=["Providers[amazon:hooks]", "Providers[amazon:other]"], + ) + + clone = shell_params.clone_with_test(test_type="Providers[amazon:hooks]") + + assert clone.test_type == "Providers[amazon:hooks]" + assert clone.parallel_test_types_list == [] + assert shell_params.parallel_test_types_list == ["Providers[amazon:hooks]", "Providers[amazon:other]"] From e28e2fe54f7e1ef0d4b39a1997ea6c286f187495 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 05:18:14 +0100 Subject: [PATCH 12/13] Raise the AIO limit on RunsOn runners for parallel MySQL tests A few dozen MySQL servers running at once exhaust the default fs.aio-max-nr, and InnoDB then fails with "io_setup() failed with EAGAIN". Generated-by: Claude Opus 5 --- .github/actions/prepare_breeze_and_image/action.yml | 4 ++-- ...ress_pools.sh => configure_runner_for_parallel_tests.sh} | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) rename scripts/ci/{configure_docker_address_pools.sh => configure_runner_for_parallel_tests.sh} (85%) diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 939d057e0f61b..7cad3d170337a 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -71,9 +71,9 @@ runs: else ./scripts/tools/free_up_disk_space.sh fi - - name: "Configure docker address pools" + - name: "Configure runner for parallel tests" shell: bash - run: ./scripts/ci/configure_docker_address_pools.sh + run: ./scripts/ci/configure_runner_for_parallel_tests.sh # Only self-hosted runners are large enough to run more test types at once than the default pools fit. if: runner.environment == 'self-hosted' - name: "Install Breeze" diff --git a/scripts/ci/configure_docker_address_pools.sh b/scripts/ci/configure_runner_for_parallel_tests.sh similarity index 85% rename from scripts/ci/configure_docker_address_pools.sh rename to scripts/ci/configure_runner_for_parallel_tests.sh index 15cac414070d1..4b97150972ebb 100755 --- a/scripts/ci/configure_docker_address_pools.sh +++ b/scripts/ci/configure_runner_for_parallel_tests.sh @@ -16,6 +16,8 @@ # specific language governing permissions and limitations # under the License. +# Large runners run a test type per CPU, each a docker compose project with its own database container. +# # Every test type of a parallel test run is a docker compose project with a network of its own. Docker's # default address pools hand out /16 and /20 subnets, which is about 30 networks per host, so large runners # running a test type per CPU fail with "all predefined address pools have been fully subnetted". @@ -37,4 +39,8 @@ daemon_json.write_text(json.dumps(config, indent=2) + "\n") print(daemon_json.read_text()) EOF sudo systemctl restart docker + +# Each MySQL server reserves Linux AIO contexts for InnoDB, and the default limit of 65536 runs out with a few +# dozen of them: "io_setup() failed with EAGAIN". +sudo sysctl -w fs.aio-max-nr=1048576 docker info --format '{{.ServerVersion}}' > /dev/null From 8ce4c9902f7adfdccca183c6c42fac3df0a3a3c2 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 28 Oct 2026 05:18:19 +0100 Subject: [PATCH 13/13] Keep the provider compatibility tests on the grouped provider test types Each test type of a compatibility test installs an Airflow version and the providers before running, so listing every provider as its own test type multiplied those installs and the jobs ran out of time. Generated-by: Claude Opus 5 --- .github/workflows/ci-amd.yml | 10 +++++--- .github/workflows/ci-arm.yml | 10 +++++--- .github/workflows/special-tests.yml | 12 ++++++---- .github/workflows/test-providers.yml | 2 +- dev/breeze/doc/ci/04_selective_checks.md | 23 +++++++++++-------- .../airflow_breeze/utils/selective_checks.py | 19 +++++++++++---- dev/breeze/tests/test_selective_checks.py | 15 ++++++++++-- 7 files changed, 65 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 8fcbfbdcec342..d873a0782f05e 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -319,6 +319,8 @@ jobs: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + providers-split-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-split-test-types-list-as-strings-in-json }} providers-xdist-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.providers-xdist-test-types-list-as-strings-in-json }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} @@ -810,7 +812,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.postgres-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-migration-tests: "true" run-coverage: ${{ needs.build-info.outputs.run-coverage }} @@ -869,7 +871,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.mysql-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -931,7 +933,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -1022,6 +1024,8 @@ jobs: ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} providers-test-types-list-as-strings-in-json: > ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + providers-split-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 1f6b192b2b320..f45c9bdbd69a1 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -200,6 +200,8 @@ jobs: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + providers-split-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-split-test-types-list-as-strings-in-json }} providers-xdist-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.providers-xdist-test-types-list-as-strings-in-json }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} @@ -691,7 +693,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.postgres-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-migration-tests: "true" run-coverage: ${{ needs.build-info.outputs.run-coverage }} @@ -750,7 +752,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.mysql-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -812,7 +814,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -903,6 +905,8 @@ jobs: ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} providers-test-types-list-as-strings-in-json: > ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + providers-split-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index f508ca2224fee..1b47ac5259a47 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -44,6 +44,10 @@ on: # yamllint disable-line rule:truthy description: "The list of providers test types to run separated by spaces" required: true type: string + providers-split-test-types-list-as-strings-in-json: + description: "The providers test types for the jobs running them in parallel in the CI image" + required: true + type: string run-coverage: description: "Whether to run coverage or not (true/false)" required: true @@ -134,7 +138,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.providers-test-types-list-as-strings-in-json }} + test-types-as-strings-in-json: ${{ inputs.providers-split-test-types-list-as-strings-in-json }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} skip-providers-tests: ${{ inputs.skip-providers-tests }} @@ -184,7 +188,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.providers-test-types-list-as-strings-in-json }} + test-types-as-strings-in-json: ${{ inputs.providers-split-test-types-list-as-strings-in-json }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} skip-providers-tests: ${{ inputs.skip-providers-tests }} @@ -235,7 +239,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.providers-test-types-list-as-strings-in-json }} + test-types-as-strings-in-json: ${{ inputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -287,7 +291,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.providers-test-types-list-as-strings-in-json }} + test-types-as-strings-in-json: ${{ inputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index b1fefd3385297..ccf9dc1230498 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -178,7 +178,7 @@ jobs: timeout-minutes: 80 # yamllint disable rule:line-length name: Compat ${{ matrix.compat.airflow-version }}:P${{ matrix.compat.python-version }}:${{ matrix.test-types.description }} - runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} strategy: fail-fast: false matrix: diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 6727910026fb3..9103f36f793ea 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -691,7 +691,8 @@ GitHub Actions to pass the list of parameters to a command to execute | postgres-versions | Which versions of Postgres to use for tests as JSON array | \['12'\] | | | prod-image-build | Whether PROD image build is needed | true | | | providers-compatibility-tests-matrix | Matrix of providers compatibility tests: (python_version, airflow_version, removed_providers). Excludes unbuilt python versions and the entry whose airflow_version equals the current branch version (unreleased on a vX-Y-test branch until GA) | \[{}\] | | -| providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (one per provider on RunsOn) | Providers Providers\[-google\] | * | +| providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers | Providers Providers\[-google\] | * | +| providers-split-test-types-list-as-strings-in-json | Provider test types for the jobs running them in parallel in the CI image (one per provider on RunsOn) | Providers Providers\[-google\] | * | | providers-xdist-test-types-list-as-strings-in-json | Which test types the Non-DB (xdist) provider tests run, with no provider split out | Providers | * | | pyproject-toml-changed | When pyproject.toml changed in the PR. | false | | | python-versions | List of python versions to use for that build | \['3.11'\] | | @@ -776,10 +777,13 @@ a job running one process (an end-to-end test against one Airflow stack, a Gradl `'8+16', '32+64'`, and a job with a minute or two of work takes `'4+8', '16+32'`. On RunsOn the Helm unit tests run all their packages in one job per Kubernetes version, since each package takes about a minute under xdist against two to three minutes of preparing the image. -On RunsOn `providers-test-types-list-as-strings-in-json` lists each provider as its own test type, in -one job, instead of the long-running providers plus one group of all the others, so the job is not left -waiting on that group. The Non-DB xdist jobs read `providers-xdist-test-types-list-as-strings-in-json` -instead, which never splits the providers: xdist runs them all in one pytest anyway. +On RunsOn `providers-split-test-types-list-as-strings-in-json`, which the provider jobs running in the CI +image read, lists each provider as its own test type, in one job, instead of the long-running providers +plus one group of all the others, so the job is not left waiting on that group. The provider +compatibility tests keep the grouped `providers-test-types-list-as-strings-in-json`, because each of their +test types installs an Airflow version first. The Non-DB xdist jobs read +`providers-xdist-test-types-list-as-strings-in-json` instead, which never splits the providers: xdist runs +them all in one pytest anyway. The matrix slices follow it too. On public runners the core test types are split into `NUMBER_OF_CORE_SLICES` (2) jobs, to keep the Non-DB xdist collection inside the runner's memory, and the individually listed providers into `NUMBER_OF_LOW_DEP_SLICES` (5). On RunsOn the core test types @@ -797,10 +801,11 @@ gated on `runner.environment == 'self-hosted'`: * The EC2 cost summary of each job, with charts of the job's CPU and memory use (`metrics: cpu,memory` of `runs-on/action`), the measurements to size a job's runner class by. -* The docker address pools (`scripts/ci/configure_docker_address_pools.sh`). Every test type of a parallel - run is a docker compose project with a network of its own, and Docker's default pools fit about 30 - networks, fewer than a RunsOn runner runs test types at once; the script splits the same range into - `/24` subnets. +* The runner settings for parallel tests (`scripts/ci/configure_runner_for_parallel_tests.sh`). Every test + type of a parallel run is a docker compose project with a network and a database of its own. Docker's + default address pools fit about 30 networks, fewer than a RunsOn runner runs test types at once, so the + script splits the same range into `/24` subnets; and it raises `fs.aio-max-nr`, which a few dozen MySQL + servers exhaust. * The image stashes (`.github/actions/stash-restore` / `stash-save`). On RunsOn the image tarballs go to the stack's S3 cache bucket, named by their sha256, and only the digest is stashed as a GitHub artifact. Any RunsOn job can write to the bucket, so a tarball is used only if it matches the digest stashed by diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index adc18a5f7bd8f..84439b69204e8 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1957,14 +1957,25 @@ def providers_test_types_list_as_strings_in_json(self) -> str: if not self.run_unit_tests: return "[]" current_test_types = self._get_providers_test_types_for_branch() - if current_test_types and self._is_on_runs_on: - # One serial pytest per provider keeps the 32 CPUs of a RunsOn runner busy; grouped - # providers would leave the job waiting on the large "all other providers" group. - return json.dumps(_get_test_list_as_json([sorted(self._get_individual_providers_list())])) self._extract_long_provider_tests(current_test_types) self._filter_platform_excluded_test_types(current_test_types) return json.dumps(_get_test_list_as_json([sorted(current_test_types)])) + @cached_property + def providers_split_test_types_list_as_strings_in_json(self) -> str: + """Provider test types for the jobs running them in parallel in the CI image. + + On RunsOn every provider (and every part of a split provider) is a test type of its own, in one + job: one serial pytest per provider keeps the 32 CPUs of the runner busy, where grouped providers + would leave the job waiting on the large "all other providers" group. The compatibility tests keep + the grouped list, as each of their test types installs an Airflow version first. + """ + if not self.run_unit_tests: + return "[]" + if self._is_on_runs_on and self._get_providers_test_types_for_branch(): + return json.dumps(_get_test_list_as_json([sorted(self._get_individual_providers_list())])) + return self.providers_test_types_list_as_strings_in_json + @cached_property def providers_xdist_test_types_list_as_strings_in_json(self) -> str: """Provider test types for the xdist jobs, which run them all in one pytest, so none is split out.""" diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index ab266c8afc6c1..2ca9c3930a779 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2345,8 +2345,19 @@ def test_providers_test_types_follow_runners( github_repository="apache/airflow", github_context_dict=github_context_dict, ) - slices = json.loads(selective_checks.providers_test_types_list_as_strings_in_json) - assert [s["test_types"].split() for s in slices] == [expected_test_types] + split_slices = json.loads(selective_checks.providers_split_test_types_list_as_strings_in_json) + assert [s["test_types"].split() for s in split_slices] == [expected_test_types] + # The compatibility tests install an Airflow version per test type, so they always get the groups + grouped_slices = json.loads(selective_checks.providers_test_types_list_as_strings_in_json) + assert [s["test_types"].split() for s in grouped_slices] == [ + [ + "Providers[-amazon,celery,google,standard]", + "Providers[amazon]", + "Providers[celery]", + "Providers[google]", + "Providers[standard]", + ] + ] @pytest.mark.parametrize(