diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 263db0346d444..7cad3d170337a 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -71,6 +71,11 @@ runs: else ./scripts/tools/free_up_disk_space.sh fi + - name: "Configure runner for parallel tests" + shell: bash + run: ./scripts/ci/configure_runner_for_parallel_tests.sh + # Only self-hosted runners are large enough to run more test types at once than the default pools fit. + if: runner.environment == 'self-hosted' - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze diff --git a/.github/actions/runs-on-cost-summary/action.yml b/.github/actions/runs-on-cost-summary/action.yml index fd0d2007f103c..d31493ae7462f 100644 --- a/.github/actions/runs-on-cost-summary/action.yml +++ b/.github/actions/runs-on-cost-summary/action.yml @@ -19,7 +19,7 @@ # Puts the job's EC2 cost next to the GitHub-hosted price in the job summary. Used by every job # that can run on RunsOn: through the breeze action, or directly by jobs that do not install Breeze. name: 'RunsOn cost summary' -description: 'Add the EC2 cost of a RunsOn job to its summary; a no-op elsewhere.' +description: 'Add the EC2 cost and CPU and memory use of a RunsOn job to its summary; a no-op elsewhere.' runs: using: 'composite' steps: @@ -28,3 +28,5 @@ runs: uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 with: show_costs: summary + # Charts the job's CPU and memory use in its summary, to size its runner class from measurements. + metrics: cpu,memory diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index a0109fb65a642..47657cbe67834 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -201,7 +201,7 @@ jobs: timeout-minutes: 60 name: "Docker Compose quick start with PROD image verifying" if: inputs.run-generic-prod-image-tests == 'true' - runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" GITHUB_REPOSITORY: ${{ github.repository }} @@ -331,7 +331,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Event driven e2e test" - runners: ${{ format(inputs.runners, '32+64', '128+256') }} + runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -359,7 +359,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "Go SDK e2e test" - runners: ${{ format(inputs.runners, '32+64', '128+256') }} + runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} @@ -373,7 +373,7 @@ jobs: uses: ./.github/workflows/airflow-e2e-tests.yml with: workflow-name: "TypeScript SDK e2e test" - runners: ${{ format(inputs.runners, '32+64', '128+256') }} + runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index d1b84c5a1567d..86f59d5f1ac35 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -148,7 +148,7 @@ jobs: # Without a pin uv picks the runner's system Python, which can be older than Airflow supports. UV_PYTHON: "${{ inputs.default-python-version }}" SHARED_DISTRIBUTIONS: ${{ join(fromJSON(inputs.shared-distributions-as-json), ' ') }} - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -255,7 +255,7 @@ jobs: check-translation-completness: timeout-minutes: 15 name: "Check translation completeness" - runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }} steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 75c3ff8edc4f1..d873a0782f05e 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -319,6 +319,10 @@ jobs: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + providers-split-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-split-test-types-list-as-strings-in-json }} + providers-xdist-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-xdist-test-types-list-as-strings-in-json }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} @@ -797,7 +801,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -808,7 +812,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.postgres-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-migration-tests: "true" run-coverage: ${{ needs.build-info.outputs.run-coverage }} @@ -856,7 +860,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -867,7 +871,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.mysql-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -916,7 +920,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -929,7 +933,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -990,7 +994,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-xdist-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} @@ -1020,6 +1024,8 @@ jobs: ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} providers-test-types-list-as-strings-in-json: > ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + providers-split-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1206,7 +1212,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} + runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1291,7 +1297,7 @@ jobs: tests-ts-sdk: name: "TypeScript SDK tests" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 15 permissions: contents: read @@ -1327,7 +1333,7 @@ jobs: tests-java-sdk: name: "Java SDK tests" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} timeout-minutes: 15 permissions: contents: read @@ -1359,7 +1365,7 @@ jobs: build-java-sdk-docs: name: "Java SDK docs" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 30 permissions: contents: read @@ -1772,7 +1778,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 341e1e021603e..f45c9bdbd69a1 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -200,6 +200,10 @@ jobs: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} providers-test-types-list-as-strings-in-json: >- ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + providers-split-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-split-test-types-list-as-strings-in-json }} + providers-xdist-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-xdist-test-types-list-as-strings-in-json }} pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} @@ -678,7 +682,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "postgres" test-name: "Postgres" @@ -689,7 +693,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.postgres-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-migration-tests: "true" run-coverage: ${{ needs.build-info.outputs.run-coverage }} @@ -737,7 +741,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "mysql" test-name: "MySQL" @@ -748,7 +752,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.mysql-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -797,7 +801,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '8+16', '32+128') }} + runners: ${{ format(needs.build-info.outputs.in-memory-runner-type, '32+64', '128+256') }} platform: ${{ needs.build-info.outputs.platform }} backend: "sqlite" test-name: "Sqlite" @@ -810,7 +814,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} run-migration-tests: "true" @@ -871,7 +875,7 @@ jobs: excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} excludes: ${{ needs.build-info.outputs.sqlite-exclude }} test-types-as-strings-in-json: > - ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + ${{ needs.build-info.outputs.providers-xdist-test-types-list-as-strings-in-json }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} @@ -901,6 +905,8 @@ jobs: ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} providers-test-types-list-as-strings-in-json: > ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + providers-split-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-split-test-types-list-as-strings-in-json }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1087,7 +1093,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(needs.build-info.outputs.runner-type, '32+64', '128+256') }} + runners: ${{ format(needs.build-info.outputs.runner-type, '8+16', '32+64') }} platform: ${{ needs.build-info.outputs.platform }} default-python-version: "${{ needs.build-info.outputs.default-python-version }}" python-versions: ${{ needs.build-info.outputs.python-versions }} @@ -1172,7 +1178,7 @@ jobs: tests-ts-sdk: name: "TypeScript SDK tests" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 15 permissions: contents: read @@ -1208,7 +1214,7 @@ jobs: tests-java-sdk: name: "Java SDK tests" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} timeout-minutes: 15 permissions: contents: read @@ -1240,7 +1246,7 @@ jobs: build-java-sdk-docs: name: "Java SDK docs" needs: [build-info] - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} timeout-minutes: 30 permissions: contents: read @@ -1651,7 +1657,7 @@ jobs: - tests-special - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers - runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0], '4+8', '16+32') }} if: needs.build-info.outputs.run-unit-tests == 'true' steps: - name: "Cleanup repo" diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 9715a8cfc6124..d9d5c463b154f 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -80,7 +80,7 @@ permissions: contents: read jobs: update-constraints: - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }} timeout-minutes: 80 name: "Update constraints" permissions: diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index cf140b98c623e..f8000a7c4dd19 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -98,7 +98,7 @@ jobs: tests-helm-release: timeout-minutes: 80 name: "Release Helm" - runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '4+8', '16+32') }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integration-system-tests.yml b/.github/workflows/integration-system-tests.yml index 8b24223a55f93..e61e4070e0c1b 100644 --- a/.github/workflows/integration-system-tests.yml +++ b/.github/workflows/integration-system-tests.yml @@ -75,7 +75,7 @@ jobs: timeout-minutes: 30 if: inputs.testable-core-integrations != '[]' name: "Integration core ${{ matrix.integration }}" - runs-on: ${{ format(fromJSON(inputs.runners)[0], '32+64', '128+256') }} + runs-on: ${{ format(fromJSON(inputs.runners)[0], '8+16', '32+64') }} strategy: fail-fast: false max-parallel: 20 diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index 09658246df9a6..1b47ac5259a47 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -44,6 +44,10 @@ on: # yamllint disable-line rule:truthy description: "The list of providers test types to run separated by spaces" required: true type: string + providers-split-test-types-list-as-strings-in-json: + description: "The providers test types for the jobs running them in parallel in the CI image" + required: true + type: string run-coverage: description: "Whether to run coverage or not (true/false)" required: true @@ -123,7 +127,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} + runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" @@ -134,7 +138,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.providers-test-types-list-as-strings-in-json }} + test-types-as-strings-in-json: ${{ inputs.providers-split-test-types-list-as-strings-in-json }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} skip-providers-tests: ${{ inputs.skip-providers-tests }} @@ -173,7 +177,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} + runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} upgrade-sqlalchemy: "true" test-name: "LatestSQLAlchemy-Postgres" @@ -184,7 +188,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.providers-test-types-list-as-strings-in-json }} + test-types-as-strings-in-json: ${{ inputs.providers-split-test-types-list-as-strings-in-json }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} skip-providers-tests: ${{ inputs.skip-providers-tests }} @@ -224,7 +228,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} + runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" @@ -235,7 +239,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.providers-test-types-list-as-strings-in-json }} + test-types-as-strings-in-json: ${{ inputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -276,7 +280,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.in-memory-runners, '8+16', '32+128') }} + runners: ${{ format(inputs.in-memory-runners, '32+64', '128+256') }} platform: ${{ inputs.platform }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" @@ -287,7 +291,7 @@ jobs: backend-versions: "['${{ inputs.default-postgres-version }}']" excluded-providers-as-string: ${{ inputs.excluded-providers-as-string }} excludes: "[]" - test-types-as-strings-in-json: ${{ inputs.providers-test-types-list-as-strings-in-json }} + test-types-as-strings-in-json: ${{ inputs.providers-split-test-types-list-as-strings-in-json }} include-success-outputs: ${{ inputs.include-success-outputs }} run-coverage: ${{ inputs.run-coverage }} debug-resources: ${{ inputs.debug-resources }} @@ -352,7 +356,7 @@ jobs: contents: read packages: read with: - runners: ${{ format(inputs.runners, '32+64', '128+256') }} + runners: ${{ format(inputs.runners, '8+16', '32+64') }} platform: ${{ inputs.platform }} test-name: "SystemTest" test-scope: "System" diff --git a/Dockerfile.ci b/Dockerfile.ci index 46435c08f484c..fd9c2012ca946 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1779,6 +1779,8 @@ function check_force_lowest_dependencies() { local provider_id # shellcheck disable=SC2001 provider_id=$(echo "${TEST_TYPE}" | sed 's/Providers\[\(.*\)\]/\1/') + # A split provider's test type names one part of its tests: Providers[:] + provider_id=${provider_id%%:*} echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${provider_id}${COLOR_RESET}" echo diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 4bf5fffa28457..9103f36f793ea 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -651,7 +651,7 @@ GitHub Actions to pass the list of parameters to a command to execute | all-versions | If set to true, then all python, k8s, DB versions are used for tests. | false | | | basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | | ci-image-build | Whether CI image build is needed | true | | -| core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core | API Always Providers | * | +| core-test-types-list-as-strings-in-json | Which test types should be run for unit tests for core (2 slices; 1 on RunsOn) | API Always Providers | * | | debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | | | default-branch | Which branch is default for the build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main | | | default-constraints-branch | Which branch is default for the build ("constraints-main" for main branch, "constraints-2-4" etc.) | constraints-main | | @@ -671,11 +671,11 @@ GitHub Actions to pass the list of parameters to a command to execute | generated-dependencies-changed | Whether generated dependencies have changed ("true"/"false") | false | | | has-migrations | Whether the PR has migrations ("true"/"false") | false | | | hatch-build-changed | When hatch build.py changed in the PR. | false | | -| helm-test-packages-list-as-string | List of helm packages to test as JSON array | \["airflow_aux", "airflow_core"\] | * | +| helm-test-packages | List of helm packages to test as JSON array (`["All"]` on RunsOn: one job runs them all) | \["airflow_aux", "airflow_core"\] | * | | helm-version | Which Helm version to use for tests | v3.15.3 | | | in-memory-runner-type | Like runner-type, but the RunsOn template puts docker in RAM (`extras=tmpfs`) unless disabled | \["ubuntu-22.04"\] | | | include-success-outputs | Whether to include outputs of successful parallel tests ("true"/"false") | false | | -| individual-providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (individually listed) | Providers[\amazon\] Providers\[google\] | * | +| individual-providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers (individually listed; 5 slices, 3 on RunsOn)| Providers[\amazon\] Providers\[google\] | * | | is-committer-build | Whether the build is triggered by a committer | false | | | is-legacy-ui-api-labeled | Whether the PR is labeled as legacy UI/API | false | | | java-sdk-version | JDK version used to build the lang-SDK Java artifacts natively in CI | 17 | | @@ -692,6 +692,8 @@ GitHub Actions to pass the list of parameters to a command to execute | prod-image-build | Whether PROD image build is needed | true | | | providers-compatibility-tests-matrix | Matrix of providers compatibility tests: (python_version, airflow_version, removed_providers). Excludes unbuilt python versions and the entry whose airflow_version equals the current branch version (unreleased on a vX-Y-test branch until GA) | \[{}\] | | | providers-test-types-list-as-strings-in-json | Which test types should be run for unit tests for providers | Providers Providers\[-google\] | * | +| providers-split-test-types-list-as-strings-in-json | Provider test types for the jobs running them in parallel in the CI image (one per provider on RunsOn) | Providers Providers\[-google\] | * | +| providers-xdist-test-types-list-as-strings-in-json | Which test types the Non-DB (xdist) provider tests run, with no provider split out | Providers | * | | pyproject-toml-changed | When pyproject.toml changed in the PR. | false | | | python-versions | List of python versions to use for that build | \['3.11'\] | | | python-versions-list-as-string | Which versions of MySQL to use for tests as space-separated string | 3.11 | * | @@ -767,12 +769,43 @@ picks its size class with `format(fromJSON(inputs.runners)[0], , )` (` `'8+16', '32+64'` or `'32+64', '128+256'`), or, for a single-job reusable workflow, its caller passes `format(, , )`. Each class is a range up to twice its size, so Spot can come from the larger instance types too; xdist follows the host's CPUs, so a larger runner runs more test workers. +Jobs using `--run-in-parallel` (the DB, lowest-dependencies and special tests) run one serial pytest per +test type instead, so they never use more CPUs than they have test types, and jobs running a single pytest +(integration and system tests) use one. The size classes follow that: the xdist jobs and every job +running provider tests in parallel take `'32+64', '128+256'`, the others `'8+16', '32+64'` or smaller: +a job running one process (an end-to-end test against one Airflow stack, a Gradle test run) takes +`'8+16', '32+64'`, and a job with a minute or two of work takes `'4+8', '16+32'`. On RunsOn the Helm unit +tests run all their packages in one job per Kubernetes version, since each package takes about a minute +under xdist against two to three minutes of preparing the image. +On RunsOn `providers-split-test-types-list-as-strings-in-json`, which the provider jobs running in the CI +image read, lists each provider as its own test type, in one job, instead of the long-running providers +plus one group of all the others, so the job is not left waiting on that group. The provider +compatibility tests keep the grouped `providers-test-types-list-as-strings-in-json`, because each of their +test types installs an Airflow version first. The Non-DB xdist jobs read +`providers-xdist-test-types-list-as-strings-in-json` instead, which never splits the providers: xdist runs +them all in one pytest anyway. +The matrix slices follow it too. On public runners the core test types are split into +`NUMBER_OF_CORE_SLICES` (2) jobs, to keep the Non-DB xdist collection inside the runner's memory, and +the individually listed providers into `NUMBER_OF_LOW_DEP_SLICES` (5). On RunsOn the core test types +stay in one job (`NUMBER_OF_CORE_SLICES_ON_RUNS_ON`), because a core DB job lasts as long as its +slowest test type whichever slice it is in, and the providers are split into jobs of about 40 test types +(`LOW_DEP_TEST_TYPES_PER_JOB_ON_RUNS_ON`), which the heaviest-first ordering keeps the 32 CPUs busy with: +3 jobs for all providers, one for a change touching a few. On RunsOn the longest-running providers +(`PROVIDER_TEST_SPLITS`: amazon, google and standard) are also split into several test types such as +`Providers[amazon:hooks]`, with `Providers[amazon:other]` running the rest of the provider's tests, so +none of them leaves the job waiting on one long serial pytest. `spot=co` takes Spot from the pools with the most capacity regardless of price, which is cheaper than the on-demand launch and whole-job rerun a capacity miss or interruption costs; on a public label the same call returns the label unchanged. Steps that only make sense on RunsOn are gated on `runner.environment == 'self-hosted'`: -* The EC2 cost summary of each job. +* The EC2 cost summary of each job, with charts of the job's CPU and memory use (`metrics: cpu,memory` of + `runs-on/action`), the measurements to size a job's runner class by. +* The runner settings for parallel tests (`scripts/ci/configure_runner_for_parallel_tests.sh`). Every test + type of a parallel run is a docker compose project with a network and a database of its own. Docker's + default address pools fit about 30 networks, fewer than a RunsOn runner runs test types at once, so the + script splits the same range into `/24` subnets; and it raises `fs.aio-max-nr`, which a few dozen MySQL + servers exhaust. * The image stashes (`.github/actions/stash-restore` / `stash-save`). On RunsOn the image tarballs go to the stack's S3 cache bucket, named by their sha256, and only the digest is stashed as a GitHub artifact. Any RunsOn job can write to the bucket, so a tarball is used only if it matches the digest stashed by diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 279f05d2a86d0..8934b40cbf6a9 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -90,9 +90,11 @@ ALL_TEST_TYPE, ALLOWED_KUBERNETES_VERSIONS, ALLOWED_TEST_TYPE_CHOICES, + PROVIDER_TEST_SPLITS, TESTCONTAINERS_IMAGES_BY_PROVIDER, GroupOfTests, all_selective_core_test_types, + get_provider_split_test_types, providers_test_type, ) from airflow_breeze.params.build_prod_params import BuildProdParams @@ -127,6 +129,8 @@ are_all_test_paths_excluded, file_name_from_test_type, generate_args_for_pytest, + get_selected_test_paths, + is_path_under_any, is_provider_selected_in_test_types, run_docker_compose_tests, ) @@ -270,13 +274,21 @@ def _run_test( integration=shell_params.integration, ) pytest_args.extend(extra_pytest_args) - # Skip "FOLDER" in case "--ignore=FOLDER" is passed as an argument - # Which might be the case if we are ignoring some providers during compatibility checks - pytest_args_before_skip = pytest_args - pytest_args = [arg for arg in pytest_args if f"--ignore={arg}" not in pytest_args] - # If no test directory is left (all positional args were excluded/ignored), skip - if pytest_args_before_skip != pytest_args and pytest_args[0].startswith("--"): + # Drop the test paths the test type selects under an "--ignore=FOLDER" argument, which is the case when + # some providers are skipped during compatibility checks: pytest runs an explicitly selected path even + # when it or an ancestor is ignored. Paths the user passes explicitly go to pytest as they are. + ignored_folders = [arg.removeprefix("--ignore=") for arg in pytest_args if arg.startswith("--ignore=")] + selected_paths = get_selected_test_paths( + test_group=shell_params.test_group, + test_type=shell_params.test_type, + skip_db_tests=shell_params.skip_db_tests, + parallel_test_types_list=shell_params.parallel_test_types_list, + integration=shell_params.integration, + ) + ignored_selections = {path for path in selected_paths if is_path_under_any(path, ignored_folders)} + if selected_paths and ignored_selections == set(selected_paths): return 0, f"Skipped test, no tests needed: {shell_params.test_type}" + pytest_args = [arg for arg in pytest_args if arg not in ignored_selections] run_cmd.extend(pytest_args) try: remove_docker_networks(networks=[f"{compose_project_name}_default"]) @@ -391,6 +403,11 @@ def _run_tests_in_pool( # tests are still running. We are only adding here test types that take more than 2 minutes to run # on a fast machine in parallel sorting_order = [ + *( + test_type + for provider_id in PROVIDER_TEST_SPLITS + for test_type in get_provider_split_test_types(provider_id) + ), "Providers[standard]", "Providers[amazon]", "Providers[google]", diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index efa1c3425b0d0..70146552743bc 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -968,6 +968,42 @@ def get_legacy_base_image(debian_version: str = ALLOWED_DEBIAN_VERSIONS[0]) -> s # Number of slices for core test types (splitting reduces memory per xdist collection) NUMBER_OF_CORE_SLICES = 2 +# Slice counts on the self-hosted RunsOn runners. Jobs using `--run-in-parallel` run one serial pytest per +# test type, so a job never uses more CPUs than it has test types. The core jobs are bound by their slowest +# test type and the Non-DB xdist collection gets enough memory on RunsOn, so one slice only removes jobs. +# Low dep providers run on 32-CPU runners and the heaviest test types start first, so about 40 test types per +# job keep those CPUs busy while the short ones fill in around the long ones. +NUMBER_OF_CORE_SLICES_ON_RUNS_ON = 1 +LOW_DEP_TEST_TYPES_PER_JOB_ON_RUNS_ON = 40 + +# On RunsOn the longest-running providers are split further in the individually listed provider test types, +# so every part runs in its own pytest process: `Providers[:]` runs the listed paths (relative to the +# provider's `tests` folder), and `Providers[:other]` runs the rest of the provider's tests. +PROVIDER_TEST_SPLITS: dict[str, dict[str, list[str]]] = { + "amazon": { + "hooks": ["unit/amazon/aws/hooks"], + "operators": ["unit/amazon/aws/operators"], + }, + "google": { + "cloud-hooks": ["unit/google/cloud/hooks"], + "cloud-operators": ["unit/google/cloud/operators"], + }, + "standard": { + "decorators": ["unit/standard/decorators"], + "python": ["unit/standard/operators/test_python.py"], + }, +} +PROVIDER_TEST_SPLIT_OTHER_PART = "other" + + +def get_provider_split_test_types(provider_id: str) -> list[str]: + """Test types running the tests of a split provider, or the provider's single test type.""" + parts = PROVIDER_TEST_SPLITS.get(provider_id) + if not parts: + return [f"Providers[{provider_id}]"] + return [f"Providers[{provider_id}:{part}]" for part in [*parts, PROVIDER_TEST_SPLIT_OTHER_PART]] + + # Milestone Tag Assistant configuration # Labels indicating a bug fix PR that should have a milestone MILESTONE_BUG_LABELS: frozenset[str] = frozenset({"kind:bug", "type:bug-fix"}) diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 6d1170848259a..6bfbb062bfdde 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -264,6 +264,8 @@ class ShellParams: def clone_with_test(self, test_type: str) -> ShellParams: new_params = deepcopy(self) new_params.test_type = test_type + # The clone runs this one test type: kept, the list would make it select every test type's folders. + new_params.parallel_test_types_list = [] return new_params @cached_property diff --git a/dev/breeze/src/airflow_breeze/utils/ci_minutes_estimate.py b/dev/breeze/src/airflow_breeze/utils/ci_minutes_estimate.py index fd51a2140b597..5a3470091f83a 100644 --- a/dev/breeze/src/airflow_breeze/utils/ci_minutes_estimate.py +++ b/dev/breeze/src/airflow_breeze/utils/ci_minutes_estimate.py @@ -125,11 +125,13 @@ def count_provider_test_files(provider_id: str) -> int: def find_tested_provider_ids(provider_test_types: list[str]) -> list[str] | None: """Return the ids of individually tested providers, or None when all providers are tested.""" - provider_ids = [] + provider_ids: list[str] = [] for test_type in provider_test_types: if not test_type.startswith("Providers[") or test_type.startswith("Providers[-"): return None - provider_ids.extend(test_type[len("Providers[") : -1].split(",")) + provider_ids.extend( + provider_id.partition(":")[0] for provider_id in test_type[len("Providers[") : -1].split(",") + ) return provider_ids diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index ef87dd1f6e3a7..5391df135be7b 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -26,6 +26,8 @@ ALL_TEST_SUITES, ALL_TEST_TYPE, NONE_TEST_TYPE, + PROVIDER_TEST_SPLIT_OTHER_PART, + PROVIDER_TEST_SPLITS, GroupOfTests, SelectiveCoreTestType, all_helm_test_packages, @@ -201,7 +203,8 @@ def run_docker_compose_tests( def file_name_from_test_type(test_type: str): test_type_no_brackets = test_type.lower().replace("[", "_").replace("]", "") - return re.sub("[,.]", "_", test_type_no_brackets)[:30] + # Also names the docker compose project, which allows only lowercase letters, digits, "-" and "_". + return re.sub("[^a-z0-9_-]", "_", test_type_no_brackets)[:30] def test_paths(test_type: str, backend: str) -> tuple[str, str, str]: @@ -241,23 +244,17 @@ def get_excluded_test_provider_folders(python_version: str) -> list[str]: return get_test_folders(excluded_folders) -def are_all_test_paths_excluded( +def get_selected_test_paths( *, test_group: GroupOfTests, test_type: str, - python_version: str, skip_db_tests: bool = False, parallel_test_types_list: list[str] | None = None, integration: tuple | None = None, -) -> bool: - """Check if all test paths for the given test type are excluded or suspended. - - Uses the same provider.yaml exclusion data as generate_args_for_pytest to determine - whether every test directory that would be generated for this test type is in the - excluded or suspended provider set. +) -> list[str]: + """The test paths generate_args_for_pytest selects for the test type, without the pytest options. - Returns False when test_type is "None" (user provides explicit paths via extra_pytest_args) - or when any test path remains after exclusions. + Empty when test_type is "None": the user then provides explicit paths via extra_pytest_args. """ if skip_db_tests and parallel_test_types_list: initial_paths = convert_parallel_types_to_folders( @@ -270,13 +267,51 @@ def are_all_test_paths_excluded( test_type=test_type, integration=integration, ) - # Filter to only actual test directory paths (not pytest flags like -m, --include-quarantined) - test_dir_paths = [p for p in initial_paths if not p.startswith("-")] + # Only the test folders: test suites such as "All-Postgres" add pytest options with values too + return [p for p in initial_paths if is_path_under_any(p, get_all_test_group_folders())] + + +def get_all_test_group_folders() -> list[str]: + return [folder for group_folders in TEST_GROUP_TO_TEST_FOLDERS.values() for folder in group_folders] + + +def is_path_under_any(path: str, folders: list[str]) -> bool: + path = path.rstrip("/") + return any(path == folder.rstrip("/") or path.startswith(f"{folder.rstrip('/')}/") for folder in folders) + + +def are_all_test_paths_excluded( + *, + test_group: GroupOfTests, + test_type: str, + python_version: str, + skip_db_tests: bool = False, + parallel_test_types_list: list[str] | None = None, + integration: tuple | None = None, +) -> bool: + """Check if all test paths for the given test type are excluded or suspended. + + Uses the same provider.yaml exclusion data as generate_args_for_pytest to determine + whether every test directory that would be generated for this test type is in the + excluded or suspended provider set. + + Returns False when test_type is "None" (user provides explicit paths via extra_pytest_args) + or when any test path remains after exclusions. + """ + test_dir_paths = get_selected_test_paths( + test_group=test_group, + test_type=test_type, + skip_db_tests=skip_db_tests, + parallel_test_types_list=parallel_test_types_list, + integration=integration, + ) if not test_dir_paths: return False - excluded = set(get_excluded_test_provider_folders(python_version)) - suspended = set(get_suspended_test_provider_folders()) - return all(p in excluded | suspended for p in test_dir_paths) + excluded_or_suspended = [ + *get_excluded_test_provider_folders(python_version), + *get_suspended_test_provider_folders(), + ] + return all(is_path_under_any(p, excluded_or_suspended) for p in test_dir_paths) TEST_TYPE_CORE_MAP_TO_PYTEST_ARGS: dict[str, list[str]] = { @@ -382,7 +417,10 @@ def is_provider_selected_in_test_types(provider_id: str, test_types: list[str]) if provider_id not in excluded: return True elif test_type.startswith(PROVIDERS_LIST_PREFIX): - included = test_type[len(PROVIDERS_LIST_PREFIX) : -1].split(",") + included = [ + included_id.partition(":")[0] + for included_id in test_type[len(PROVIDERS_LIST_PREFIX) : -1].split(",") + ] if provider_id in included: return True else: @@ -447,6 +485,8 @@ def convert_test_type_to_pytest_args( ) providers_with_exclusions.remove(provider_test_to_exclude) return providers_with_exclusions + if test_type.startswith(PROVIDERS_LIST_PREFIX) and ":" in test_type: + return convert_provider_part_to_pytest_args(test_type[len(PROVIDERS_LIST_PREFIX) : -1]) if test_type.startswith(PROVIDERS_LIST_PREFIX): provider_list = test_type[len(PROVIDERS_LIST_PREFIX) : -1].split(",") providers_to_test = [] @@ -482,6 +522,22 @@ def convert_test_type_to_pytest_args( sys.exit(1) +def convert_provider_part_to_pytest_args(provider_part: str) -> list[str]: + """Pytest args for one part of a split provider, given as ``:``.""" + provider_id, _, part = provider_part.partition(":") + parts = PROVIDER_TEST_SPLITS.get(provider_id, {}) + tests_folder = f"providers/{provider_id.replace('.', '/')}/tests" + if part == PROVIDER_TEST_SPLIT_OTHER_PART and parts: + return [ + tests_folder, + *(f"--ignore={tests_folder}/{path}" for paths in parts.values() for path in paths), + ] + if part not in parts: + console_print(f"[error]Unknown part {part!r} of the {provider_id!r} provider tests[/]") + sys.exit(1) + return [f"{tests_folder}/{path}" for path in parts[part]] + + def generate_args_for_pytest( *, test_group: GroupOfTests, @@ -618,6 +674,12 @@ def generate_args_for_pytest( def convert_parallel_types_to_folders(test_group: GroupOfTests, parallel_test_types_list: list[str]): + # All test types share one pytest here, where the --ignore of a ":other" part would also drop + # the provider's other parts, so a split provider runs as a whole: Providers[]. + split_parts = [test_type for test_type in parallel_test_types_list if ":" in test_type] + if split_parts: + console_print(f"[error]Split provider test types cannot run in one pytest process: {split_parts}[/]") + sys.exit(1) args = [] for _test_type in parallel_test_types_list: args.extend( @@ -626,9 +688,6 @@ def convert_parallel_types_to_folders(test_group: GroupOfTests, parallel_test_ty test_type=_test_type, ) ) - all_test_prefixes: list[str] = [] + all_test_prefixes = get_all_test_group_folders() # leave only folders, strip --pytest-args that exclude some folders with `-` prefix - for group_folders in TEST_GROUP_TO_TEST_FOLDERS.values(): - for group_folder in group_folders: - all_test_prefixes.append(group_folder) return [arg for arg in args if any(arg.startswith(prefix) for prefix in all_test_prefixes)] diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index b5415d504d32b..84439b69204e8 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -20,6 +20,7 @@ import difflib import itertools import json +import math import os import re import sys @@ -34,6 +35,7 @@ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( ALL_PYTHON_MAJOR_MINOR_VERSIONS, + ALL_TEST_TYPE, APACHE_AIRFLOW_GITHUB_REPOSITORY, CI_AMD_PLATFORM, CI_ARM_PLATFORM, @@ -51,8 +53,11 @@ HELM_VERSION, JAVA_SDK_VERSION, KIND_VERSION, + LOW_DEP_TEST_TYPES_PER_JOB_ON_RUNS_ON, NUMBER_OF_CORE_SLICES, + NUMBER_OF_CORE_SLICES_ON_RUNS_ON, NUMBER_OF_LOW_DEP_SLICES, + PROVIDER_TEST_SPLITS, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, PUBLIC_AMD_RUNNERS, PUBLIC_ARM_RUNNERS, @@ -72,6 +77,7 @@ all_helm_test_packages, all_selective_core_test_types, get_airflow_version, + get_provider_split_test_types, providers_test_type, ) from airflow_breeze.utils.console import console_print, get_console @@ -883,11 +889,13 @@ def _get_test_type_description(provider_test_types: list[str]) -> str: first_provider = first_provider.replace("Providers[", "").replace("]", "") if last_provider.startswith("Providers["): last_provider = last_provider.replace("Providers[", "").replace("]", "") - return ( + description = ( f"{first_provider[:13]}...{last_provider[:13]}" if first_provider != last_provider else (first_provider[:29]) ) + # The description ends up in artifact names (via JOB_ID), which must not contain ":". + return description.replace(":", "-") def _get_test_list_as_json(list_of_list_of_types: list[list[str]]) -> list[dict[str, str]] | None: @@ -1877,9 +1885,10 @@ def core_test_types_list_as_strings_in_json(self) -> str | None: if not self.run_unit_tests: return None current_test_types = sorted(set(self._get_core_test_types_to_run())) - if len(current_test_types) <= NUMBER_OF_CORE_SLICES: + number_of_slices = NUMBER_OF_CORE_SLICES_ON_RUNS_ON if self._is_on_runs_on else NUMBER_OF_CORE_SLICES + if len(current_test_types) <= number_of_slices: return json.dumps(_get_test_list_as_json([current_test_types])) - list_of_list_of_types = _split_list(current_test_types, NUMBER_OF_CORE_SLICES) + list_of_list_of_types = _split_list(current_test_types, number_of_slices) return json.dumps(_get_test_list_as_json(list_of_list_of_types)) @cached_property @@ -1929,10 +1938,7 @@ def _filter_platform_excluded_test_types(self, current_test_types: set[str]) -> if kept: current_test_types.add(f"Providers[{','.join(kept)}]") - @cached_property - def providers_test_types_list_as_strings_in_json(self) -> str: - if not self.run_unit_tests: - return "[]" + def _get_providers_test_types_for_branch(self) -> set[str]: current_test_types = set(self._get_providers_test_types_to_run()) if self._default_branch != "main": test_types_to_remove: set[str] = set() @@ -1944,10 +1950,44 @@ def providers_test_types_list_as_strings_in_json(self) -> str: ) test_types_to_remove.add(test_type) current_test_types = current_test_types - test_types_to_remove + return current_test_types + + @cached_property + def providers_test_types_list_as_strings_in_json(self) -> str: + if not self.run_unit_tests: + return "[]" + current_test_types = self._get_providers_test_types_for_branch() self._extract_long_provider_tests(current_test_types) self._filter_platform_excluded_test_types(current_test_types) return json.dumps(_get_test_list_as_json([sorted(current_test_types)])) + @cached_property + def providers_split_test_types_list_as_strings_in_json(self) -> str: + """Provider test types for the jobs running them in parallel in the CI image. + + On RunsOn every provider (and every part of a split provider) is a test type of its own, in one + job: one serial pytest per provider keeps the 32 CPUs of the runner busy, where grouped providers + would leave the job waiting on the large "all other providers" group. The compatibility tests keep + the grouped list, as each of their test types installs an Airflow version first. + """ + if not self.run_unit_tests: + return "[]" + if self._is_on_runs_on and self._get_providers_test_types_for_branch(): + return json.dumps(_get_test_list_as_json([sorted(self._get_individual_providers_list())])) + return self.providers_test_types_list_as_strings_in_json + + @cached_property + def providers_xdist_test_types_list_as_strings_in_json(self) -> str: + """Provider test types for the xdist jobs, which run them all in one pytest, so none is split out.""" + if not self.run_unit_tests: + return "[]" + current_test_types = self._get_providers_test_types_for_branch() + if "Providers" in current_test_types and self._platform_excluded_providers: + current_test_types.remove("Providers") + current_test_types.add(f"Providers[-{','.join(sorted(self._platform_excluded_providers))}]") + self._filter_platform_excluded_test_types(current_test_types) + return json.dumps(_get_test_list_as_json([sorted(current_test_types)])) + def _get_individual_providers_list(self): current_test_types = set(self._get_providers_test_types_to_run(split_to_individual_providers=True)) if "Providers" in current_test_types: @@ -1956,6 +1996,12 @@ def _get_individual_providers_list(self): {f"Providers[{provider}]" for provider in get_available_distributions(include_not_ready=True)} ) self._filter_platform_excluded_test_types(current_test_types) + if not self._is_on_runs_on: + return current_test_types + for provider_id in PROVIDER_TEST_SPLITS: + if f"Providers[{provider_id}]" in current_test_types: + current_test_types.remove(f"Providers[{provider_id}]") + current_test_types.update(get_provider_split_test_types(provider_id)) return current_test_types @cached_property @@ -1966,11 +2012,14 @@ def individual_providers_test_types_list_as_strings_in_json(self) -> str | None: current_test_types = sorted(self._get_individual_providers_list()) if not current_test_types: return None - # We are hard-coding the number of lists as reasonable starting point to split the - # list of test types - and we can modify it in the future - if len(current_test_types) < NUMBER_OF_LOW_DEP_SLICES: + number_of_slices = ( + math.ceil(len(current_test_types) / LOW_DEP_TEST_TYPES_PER_JOB_ON_RUNS_ON) + if self._is_on_runs_on + else NUMBER_OF_LOW_DEP_SLICES + ) + if len(current_test_types) < number_of_slices: return json.dumps(_get_test_list_as_json([current_test_types])) - list_of_list_of_types = _split_list(current_test_types, NUMBER_OF_LOW_DEP_SLICES) + list_of_list_of_types = _split_list(current_test_types, number_of_slices) return json.dumps(_get_test_list_as_json(list_of_list_of_types)) @cached_property @@ -2248,6 +2297,10 @@ def disable_airflow_repo_cache(self) -> bool: @cached_property def helm_test_packages(self) -> str: + # A Helm test package takes about a minute under xdist on a RunsOn runner, against two to three + # minutes of preparing the image, so RunsOn runs all of them in one job. + if self._is_on_runs_on: + return json.dumps([ALL_TEST_TYPE]) return json.dumps(all_helm_test_packages()) @cached_property @@ -2322,17 +2375,18 @@ def runner_type(self) -> str: ) return json.dumps([template.format(run_id=run_id)]) + @cached_property + def _is_on_runs_on(self) -> bool: + # Reuses the routing decision of runner_type, which may have asked the GitHub API. + return self.runner_type not in (PUBLIC_AMD_RUNNERS, PUBLIC_ARM_RUNNERS) + @cached_property def in_memory_runner_type(self) -> str: """Runner labels like ``runner_type``, with docker on a RAM disk when the run is on RunsOn. The repository variable ``USE_IN_MEMORY_DOCKER=false`` keeps docker on disk. """ - # Reuses the routing decision of runner_type, which may have asked the GitHub API. - if ( - self.runner_type in (PUBLIC_AMD_RUNNERS, PUBLIC_ARM_RUNNERS) - or os.environ.get("USE_IN_MEMORY_DOCKER", "").casefold() == "false" - ): + if not self._is_on_runs_on or os.environ.get("USE_IN_MEMORY_DOCKER", "").casefold() == "false": return self.runner_type template = ( RUNS_ON_ARM_IN_MEMORY_RUNNER_TEMPLATE diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py b/dev/breeze/tests/test_pytest_args_for_test_types.py index 4332d8dceb7d6..88444958690ba 100644 --- a/dev/breeze/tests/test_pytest_args_for_test_types.py +++ b/dev/breeze/tests/test_pytest_args_for_test_types.py @@ -20,7 +20,11 @@ from airflow_breeze.global_constants import GroupOfTests from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH -from airflow_breeze.utils.run_tests import convert_parallel_types_to_folders, convert_test_type_to_pytest_args +from airflow_breeze.utils.run_tests import ( + convert_parallel_types_to_folders, + convert_test_type_to_pytest_args, + file_name_from_test_type, +) def _all_providers() -> list[str]: @@ -214,6 +218,53 @@ def test_pytest_args_for_missing_provider(): ) +@pytest.mark.parametrize( + ("test_type", "pytest_args"), + [ + pytest.param( + "Providers[amazon:hooks]", + ["providers/amazon/tests/unit/amazon/aws/hooks"], + id="named part runs its paths", + ), + pytest.param( + "Providers[standard:python]", + ["providers/standard/tests/unit/standard/operators/test_python.py"], + id="a part can be a single file", + ), + pytest.param( + "Providers[amazon:other]", + [ + "providers/amazon/tests", + "--ignore=providers/amazon/tests/unit/amazon/aws/hooks", + "--ignore=providers/amazon/tests/unit/amazon/aws/operators", + ], + id="other part ignores the named parts", + ), + ], +) +def test_pytest_args_for_split_provider_parts(test_type: str, pytest_args: list[str]): + assert ( + convert_test_type_to_pytest_args(test_group=GroupOfTests.PROVIDERS, test_type=test_type) + == pytest_args + ) + + +@pytest.mark.parametrize( + "test_type", ["Providers[amazon:missing]", "Providers[airbyte:other]"], ids=["unknown part", "unsplit"] +) +def test_pytest_args_for_unknown_provider_part(test_type: str): + with pytest.raises(SystemExit): + convert_test_type_to_pytest_args(test_group=GroupOfTests.PROVIDERS, test_type=test_type) + + +def test_folders_for_parallel_test_types_reject_split_provider_parts(): + with pytest.raises(SystemExit): + convert_parallel_types_to_folders( + test_group=GroupOfTests.PROVIDERS, + parallel_test_types_list=["Providers[amazon:hooks]", "Providers[amazon:other]"], + ) + + @pytest.mark.parametrize( ("test_group", "parallel_test_types", "folders"), [ @@ -394,3 +445,15 @@ def xtest_wrong_types_for_parallel_test_types(test_group: GroupOfTests, parallel test_group=test_group, parallel_test_types_list=parallel_test_types.split(" "), ) + + +@pytest.mark.parametrize( + ("test_type", "file_name"), + [ + pytest.param("Providers[amazon:hooks]", "providers_amazon_hooks", id="split provider part"), + pytest.param("Providers[google:cloud-hooks]", "providers_google_cloud-hooks", id="part with a dash"), + pytest.param("Providers[apache.hive,http]", "providers_apache_hive_http", id="provider list"), + ], +) +def test_file_name_from_test_type_is_a_valid_compose_project_name(test_type: str, file_name: str): + assert file_name_from_test_type(test_type) == file_name diff --git a/dev/breeze/tests/test_run_test_args.py b/dev/breeze/tests/test_run_test_args.py index b146e61d3059b..bbcbb598e21ca 100644 --- a/dev/breeze/tests/test_run_test_args.py +++ b/dev/breeze/tests/test_run_test_args.py @@ -23,6 +23,7 @@ from airflow_breeze.commands.testing_commands import _run_test, pull_testcontainers_images from airflow_breeze.global_constants import TESTCONTAINERS_IMAGES_BY_PROVIDER, GroupOfTests from airflow_breeze.params.shell_params import ShellParams +from airflow_breeze.utils.run_tests import is_provider_selected_in_test_types @pytest.fixture(autouse=True) @@ -109,6 +110,70 @@ def test_test_is_skipped_if_all_are_ignored(mock_run_command): mock_run_command.assert_called_once() # called only to compose down +def test_split_provider_part_is_skipped_if_its_provider_is_ignored(mock_run_command): + _run_test( + shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="Providers[amazon:hooks]"), + extra_pytest_args=("--ignore=providers/amazon/tests",), + python_version="3.9", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + + mock_run_command.assert_called_once() # called only to compose down + + +def test_test_is_skipped_if_its_only_path_is_ignored_next_to_path_valued_options(mock_run_command): + _run_test( + shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="Providers[amazon:hooks]"), + extra_pytest_args=( + "--ignore=providers/amazon/tests", + "--basetemp", + "/tmp/breeze-pytest", + "--rootdir", + "airflow-core/tests", + ), + python_version="3.9", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + + mock_run_command.assert_called_once() # called only to compose down + + +def test_test_suite_is_skipped_if_all_its_paths_are_ignored(mock_run_command): + _run_test( + shell_params=ShellParams(test_group=GroupOfTests.CORE, test_type="All-Postgres"), + extra_pytest_args=("--ignore=airflow-core/tests/unit",), + python_version="3.9", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + + mock_run_command.assert_called_once() # called only to compose down + + +def test_explicit_paths_go_to_pytest_as_they_are(mock_run_command): + _run_test( + shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="None"), + extra_pytest_args=( + "providers/amazon/tests/unit/amazon/aws/hooks", + "providers/http/tests", + "--ignore=providers/amazon/tests", + ), + python_version="3.9", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + + run_cmd = mock_run_command.call_args_list[1].args[0] + assert "providers/http/tests" in run_cmd + assert "providers/amazon/tests/unit/amazon/aws/hooks" in run_cmd + + def test_test_is_skipped_when_all_providers_excluded_for_python_version( mock_run_command, mock_get_excluded_provider_folders ): @@ -127,6 +192,34 @@ def test_test_is_skipped_when_all_providers_excluded_for_python_version( mock_run_command.assert_not_called() +def test_split_provider_part_is_skipped_when_provider_excluded_for_python_version( + mock_run_command, mock_get_excluded_provider_folders +): + mock_get_excluded_provider_folders.return_value = ["amazon"] + return_code, message = _run_test( + shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS, test_type="Providers[amazon:hooks]"), + extra_pytest_args=(), + python_version="3.14", + output=None, + test_timeout=60, + skip_docker_compose_down=True, + ) + assert return_code == 0 + assert "Skipped" in message + mock_run_command.assert_not_called() + + +@pytest.mark.parametrize( + ("test_types", "expected"), + [ + pytest.param(["Providers[amazon:hooks]"], True, id="part of the provider"), + pytest.param(["Providers[google:cloud-hooks]"], False, id="part of another provider"), + ], +) +def test_split_provider_part_selects_its_provider(test_types: list[str], expected: bool): + assert is_provider_selected_in_test_types("amazon", test_types) is expected + + def test_test_is_not_skipped_when_some_providers_remain(mock_run_command, mock_get_excluded_provider_folders): """When only some providers are excluded, the test should still run.""" mock_get_excluded_provider_folders.return_value = ["http"] diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index ddce03204adf8..2ca9c3930a779 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -37,10 +37,13 @@ DEFAULT_PYTHON_MAJOR_MINOR_VERSION, JAVA_SDK_VERSION, NUMBER_OF_CORE_SLICES, + NUMBER_OF_CORE_SLICES_ON_RUNS_ON, NUMBER_OF_LOW_DEP_SLICES, PROVIDERS_COMPATIBILITY_TESTS_MATRIX, GithubEvents, + all_helm_test_packages, get_airflow_version, + get_provider_split_test_types, ) from airflow_breeze.utils.functools_cache import clearable_cache from airflow_breeze.utils.packages import get_available_distributions @@ -111,6 +114,12 @@ f"Providers[{provider}]" for provider in get_available_distributions(include_not_ready=True) ] +LIST_OF_ALL_PROVIDER_TESTS_ON_RUNS_ON = [ + test_type + for provider in get_available_distributions(include_not_ready=True) + for test_type in get_provider_split_test_types(provider) +] + LIST_OF_ALL_PROVIDER_TESTS_AS_JSON = json.dumps( _get_test_list_as_json(_split_list(sorted(LIST_OF_ALL_PROVIDER_TESTS), 5)) ) @@ -2250,6 +2259,165 @@ def test_list_splitting(): ) +@pytest.mark.parametrize( + ( + "github_event", + "github_context_dict", + "expected_core_slices", + "expected_low_dep_slices", + "expected_low_dep_test_types", + ), + [ + pytest.param( + GithubEvents.PULL_REQUEST, + {}, + NUMBER_OF_CORE_SLICES, + NUMBER_OF_LOW_DEP_SLICES, + LIST_OF_ALL_PROVIDER_TESTS, + id="public runners", + ), + pytest.param( + GithubEvents.PUSH, + {"run_id": 42}, + NUMBER_OF_CORE_SLICES_ON_RUNS_ON, + 3, + LIST_OF_ALL_PROVIDER_TESTS_ON_RUNS_ON, + id="RunsOn runners", + ), + ], +) +def test_number_of_test_type_slices_follows_runners( + github_event: GithubEvents, + github_context_dict: dict[str, Any], + expected_core_slices: int, + expected_low_dep_slices: int, + expected_low_dep_test_types: list[str], +): + selective_checks = SelectiveChecks( + pr_labels=("full tests needed",), + default_branch="main", + github_event=github_event, + github_repository="apache/airflow", + github_context_dict=github_context_dict, + ) + core_slices = json.loads(selective_checks.core_test_types_list_as_strings_in_json) + low_dep_slices = json.loads(selective_checks.individual_providers_test_types_list_as_strings_in_json) + assert len(core_slices) == expected_core_slices + assert len(low_dep_slices) == expected_low_dep_slices + assert sorted(" ".join(s["test_types"] for s in core_slices).split()) == sorted( + ALL_CI_SELECTIVE_TEST_TYPES.split() + ) + assert sorted(" ".join(s["test_types"] for s in low_dep_slices).split()) == sorted( + expected_low_dep_test_types + ) + + +@pytest.mark.parametrize( + ("github_event", "github_context_dict", "expected_test_types"), + [ + pytest.param( + GithubEvents.PULL_REQUEST, + {}, + [ + "Providers[-amazon,celery,google,standard]", + "Providers[amazon]", + "Providers[celery]", + "Providers[google]", + "Providers[standard]", + ], + id="public runners group the providers", + ), + pytest.param( + GithubEvents.PUSH, + {"run_id": 42}, + sorted(LIST_OF_ALL_PROVIDER_TESTS_ON_RUNS_ON), + id="RunsOn runners list each provider", + ), + ], +) +def test_providers_test_types_follow_runners( + github_event: GithubEvents, github_context_dict: dict[str, Any], expected_test_types: list[str] +): + selective_checks = SelectiveChecks( + pr_labels=("full tests needed",), + default_branch="main", + github_event=github_event, + github_repository="apache/airflow", + github_context_dict=github_context_dict, + ) + split_slices = json.loads(selective_checks.providers_split_test_types_list_as_strings_in_json) + assert [s["test_types"].split() for s in split_slices] == [expected_test_types] + # The compatibility tests install an Airflow version per test type, so they always get the groups + grouped_slices = json.loads(selective_checks.providers_test_types_list_as_strings_in_json) + assert [s["test_types"].split() for s in grouped_slices] == [ + [ + "Providers[-amazon,celery,google,standard]", + "Providers[amazon]", + "Providers[celery]", + "Providers[google]", + "Providers[standard]", + ] + ] + + +@pytest.mark.parametrize( + ("platform", "expected_test_types"), + [ + pytest.param(CI_AMD_PLATFORM, "Providers", id="AMD"), + pytest.param(CI_ARM_PLATFORM, "Providers[-ibm.db2,ibm.mq]", id="ARM skips platform-excluded"), + ], +) +def test_providers_xdist_test_types_are_not_split(platform: str, expected_test_types: str): + selective_checks = SelectiveChecks( + files=("airflow-core/tests/test_example.py",), + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.PUSH, + github_repository="apache/airflow", + github_context_dict={"run_id": 42}, + default_branch="main", + pr_labels=("full tests needed",), + platform=platform, + ) + slices = json.loads(selective_checks.providers_xdist_test_types_list_as_strings_in_json) + assert [s["test_types"] for s in slices] == [expected_test_types] + + +@patch.object(SelectiveChecks, "_is_on_runs_on", new=property(lambda self: True)) +def test_individual_providers_split_the_longest_running_providers_on_runs_on(): + selective_checks = SelectiveChecks( + files=("providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py",), + commit_ref=NEUTRAL_COMMIT, + default_branch="main", + github_event=GithubEvents.PULL_REQUEST, + ) + slices = json.loads(selective_checks.individual_providers_test_types_list_as_strings_in_json) + assert len(slices) == 1 + assert ":" not in slices[0]["description"] + test_types = slices[0]["test_types"].split() + assert "Providers[amazon]" not in test_types + assert {"Providers[amazon:hooks]", "Providers[amazon:operators]", "Providers[amazon:other]"} <= set( + test_types + ) + + +@pytest.mark.parametrize( + ("github_event", "github_context_dict", "expected_packages"), + [ + pytest.param(GithubEvents.PULL_REQUEST, {}, all_helm_test_packages(), id="public runners"), + pytest.param(GithubEvents.PUSH, {"run_id": 42}, ["All"], id="RunsOn runners"), + ], +) +def test_helm_test_packages_follow_runners( + github_event: GithubEvents, github_context_dict: dict[str, Any], expected_packages: list[str] +): + selective_checks = SelectiveChecks( + github_event=github_event, + github_repository="apache/airflow", + github_context_dict=github_context_dict, + ) + assert json.loads(selective_checks.helm_test_packages) == expected_packages + + def test_excluded_providers(): stderr = SelectiveChecks( files=(), diff --git a/dev/breeze/tests/test_shell_params.py b/dev/breeze/tests/test_shell_params.py index 78e3d56d69143..630abecc953fe 100644 --- a/dev/breeze/tests/test_shell_params.py +++ b/dev/breeze/tests/test_shell_params.py @@ -29,7 +29,7 @@ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH from airflow_breeze.commands.common_options import option_project_name -from airflow_breeze.global_constants import MOUNT_SELECTED, PYCACHE_PREFIX_IN_CONTAINER +from airflow_breeze.global_constants import MOUNT_SELECTED, PYCACHE_PREFIX_IN_CONTAINER, GroupOfTests from airflow_breeze.params.shell_params import ShellParams from airflow_breeze.utils.path_utils import ( SCRIPTS_CI_DOCKER_COMPOSE_BASE_PATH, @@ -425,3 +425,16 @@ def test_sphinx_airflow_theme_is_mounted_only_when_a_path_is_given(theme_path: s ) # Compose interpolates the bind source from this variable, so it must be exported together with the file. assert shell_params.env_variables_for_docker_commands["SPHINX_AIRFLOW_THEME_PATH"] == theme_path + + +def test_clone_with_test_runs_only_its_own_test_type(): + shell_params = ShellParams( + test_group=GroupOfTests.PROVIDERS, + parallel_test_types_list=["Providers[amazon:hooks]", "Providers[amazon:other]"], + ) + + clone = shell_params.clone_with_test(test_type="Providers[amazon:hooks]") + + assert clone.test_type == "Providers[amazon:hooks]" + assert clone.parallel_test_types_list == [] + assert shell_params.parallel_test_types_list == ["Providers[amazon:hooks]", "Providers[amazon:other]"] diff --git a/scripts/ci/configure_runner_for_parallel_tests.sh b/scripts/ci/configure_runner_for_parallel_tests.sh new file mode 100755 index 0000000000000..4b97150972ebb --- /dev/null +++ b/scripts/ci/configure_runner_for_parallel_tests.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Large runners run a test type per CPU, each a docker compose project with its own database container. +# +# Every test type of a parallel test run is a docker compose project with a network of its own. Docker's +# default address pools hand out /16 and /20 subnets, which is about 30 networks per host, so large runners +# running a test type per CPU fail with "all predefined address pools have been fully subnetted". +# Splitting the same 172.17.0.0/12 range into /24 subnets gives 4096 networks without claiming new addresses. +set -euo pipefail + +DAEMON_JSON=/etc/docker/daemon.json + +sudo mkdir -p "$(dirname "${DAEMON_JSON}")" +sudo python3 - "${DAEMON_JSON}" <<'EOF' +import json +import pathlib +import sys + +daemon_json = pathlib.Path(sys.argv[1]) +config = json.loads(daemon_json.read_text()) if daemon_json.exists() and daemon_json.read_text().strip() else {} +config["default-address-pools"] = [{"base": "172.17.0.0/12", "size": 24}] +daemon_json.write_text(json.dumps(config, indent=2) + "\n") +print(daemon_json.read_text()) +EOF +sudo systemctl restart docker + +# Each MySQL server reserves Linux AIO contexts for InnoDB, and the default limit of 65536 runs out with a few +# dozen of them: "io_setup() failed with EAGAIN". +sudo sysctl -w fs.aio-max-nr=1048576 +docker info --format '{{.ServerVersion}}' > /dev/null diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 1e61d0bd85608..c487ea60a757b 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -466,6 +466,8 @@ function check_force_lowest_dependencies() { local provider_id # shellcheck disable=SC2001 provider_id=$(echo "${TEST_TYPE}" | sed 's/Providers\[\(.*\)\]/\1/') + # A split provider's test type names one part of its tests: Providers[:] + provider_id=${provider_id%%:*} echo echo "${COLOR_BLUE}Forcing dependencies to lowest versions for provider: ${provider_id}${COLOR_RESET}" echo