Serialise the workflows that publish shared state

.github/workflows/publish-docs-to-s3.yml

@@ -113,6 +113,7 @@ jobs:
       image-stash-ref: "publish-docs-${{ inputs.ref }}"
       registry-providers: ${{ steps.derive_registry_inputs.outputs.registry-providers }}
       registry-full-build: ${{ steps.derive_registry_inputs.outputs.registry-full-build }}
+      docs-kind: ${{ steps.docs_kind.outputs.docs-kind }}
     if: contains(fromJSON('[
       "ashb",
       "bugraoz93",
@@ -178,6 +179,26 @@ jobs:
             exit 0
           fi
           python3 dev/registry/derive_wave_providers.py
+      - name: "Classify the published docs"
+        id: docs_kind
+        env:
+          INCLUDE_DOCS: ${{ inputs.include-docs }}
+        shell: bash
+        # Provider and Airflow docs land in different paths of the bucket, so the publish job
+        # serialises each kind separately. A full build or a mixed list counts as `airflow`.
+        # Same non-provider list as the registry step above.
+        run: |
+          NON_PROVIDER_TOKENS=("all" "apache-airflow" "apache-airflow-ctl" "apache-airflow-mypy" "task-sdk" "java-sdk" "ts-sdk" "helm-chart" "docker-stack")
+          kind=providers
+          for token in $INCLUDE_DOCS; do
+            for np in "${NON_PROVIDER_TOKENS[@]}"; do
+              if [[ "$token" == "$np" ]]; then
+                kind=airflow
+              fi
+            done
+          done
+          echo "Publishing ${kind} docs ('${INCLUDE_DOCS}')."
+          echo "docs-kind=${kind}" >> "${GITHUB_OUTPUT}"
       - name: "Derive the default Python version"
         id: default_python
         shell: bash
@@ -585,6 +606,14 @@ jobs:
       needs.build-ts-sdk-docs.result == 'success' &&
       needs.update-registry.result != 'failure'
     name: "Publish documentation to S3"
+    outputs:
+      docs-synced: ${{ steps.sync_docs.outcome == 'success' }}
+    # Keyed on the resolved destination rather than the `destination` input, whose `auto` stands
+    # for either bucket, and on the kind of docs, so provider and Airflow publishes do not wait
+    # for each other. The bucket-wide packages-metadata.json is left to update-packages-metadata.
+    concurrency:
+      group: publish-docs-${{ needs.build-info.outputs.destination }}-${{ needs.build-info.outputs.docs-kind }}
+      cancel-in-progress: false
     permissions:
       id-token: write
       contents: read
@@ -697,6 +726,7 @@ jobs:
           aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-2
       - name: "Syncing docs to S3"
+        id: sync_docs
         env:
           DESTINATION_LOCATION: "${{ needs.build-info.outputs.destination-location }}"
           SOURCE_DIR_PATH: "/mnt/airflow-site/docs-archive/"
@@ -705,7 +735,8 @@ jobs:
         run: |
           breeze release-management publish-docs-to-s3 --source-dir-path ${SOURCE_DIR_PATH} \
           --destination-location ${DESTINATION_LOCATION} --stable-versions \
-          --exclude-docs "${EXCLUDE_DOCS}" --overwrite ${SKIP_WRITE_TO_STABLE_FOLDER}
+          --exclude-docs "${EXCLUDE_DOCS}" --overwrite ${SKIP_WRITE_TO_STABLE_FOLDER} \
+          --skip-packages-metadata
       - name: "Download schema artifacts"
         if: needs.build-info.outputs.publish-execution-api-schema == 'true' || needs.build-info.outputs.publish-supervisor-schema == 'true' || needs.build-info.outputs.publish-dag-schema == 'true'
         uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
@@ -735,6 +766,45 @@ jobs:
           breeze release-management publish-schemas-to-s3 \
             --destination-location "${SCHEMAS_DESTINATION}" "${args[@]}"
 
+  update-packages-metadata:
+    needs: [build-info, publish-docs-to-s3]
+    # Keyed on the sync step rather than the job, so a schema publish failing after a complete sync
+    # still gets its docs into the metadata, while a partial sync is not advertised.
+    if: ${{ !cancelled() && needs.publish-docs-to-s3.outputs.docs-synced == 'true' }}
+    name: "Update packages metadata in S3"
+    # packages-metadata.json is rebuilt from a scan of every package in the bucket, so provider and
+    # Airflow publishes share it. One group per bucket keeps a scan from missing a version another
+    # run is still writing; a pending run cancelled by a later one loses nothing, as that later run
+    # rescans the whole bucket.
+    concurrency:
+      group: publish-docs-metadata-${{ needs.build-info.outputs.destination }}
+      cancel-in-progress: false
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd  # v6.3.0
+        with:
+          aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: "Regenerate packages-metadata.json"
+        env:
+          DESTINATION_LOCATION: "${{ needs.build-info.outputs.destination-location }}"
+        run: >
+          breeze release-management publish-docs-to-s3
+          --destination-location ${DESTINATION_LOCATION} --packages-metadata-only
+
   update-registry:
     # Runs alongside the docs build rather than after it: the registry reads nothing the docs
     # build produces, so ordering the two only added its whole duration to the wait. The publish

.github/workflows/registry-backfill.yml

@@ -283,6 +283,12 @@ jobs:
     needs: [prepare, backfill]
     runs-on: ubuntu-latest
     name: "Publish versions.json"
+    # Shares registry-build.yml's group, so it never runs alongside a registry publish to the same
+    # bucket. Only this job: the backfill matrix writes per-version paths of its own, and putting
+    # a matrix in one group would cancel all but two of its entries.
+    concurrency:
+      group: registry-publish-${{ inputs.destination }}
+      cancel-in-progress: false
     steps:
       - name: "Checkout repository"
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1

.github/workflows/registry-build.yml

@@ -93,6 +93,15 @@ jobs:
   build-and-publish-registry:
     timeout-minutes: 45
     name: "Build & Publish Registry"
+    # An incremental build downloads the bucket's providers.json and modules.json, merges its own
+    # providers in and syncs them back, so two publishes to one bucket at once lose whichever
+    # finishes first. Shared with registry-backfill.yml's publish job; set on the job because this
+    # workflow is also called from publish-docs-to-s3.yml. GitHub keeps one pending run per group,
+    # so a third publish cancels the waiting one: that run shows as cancelled and is re-run, which
+    # beats the silent loss this replaces.
+    concurrency:
+      group: registry-publish-${{ inputs.destination }}
+      cancel-in-progress: false
     # On a dispatch this enforces the committer allowlist below; on the `workflow_call` path the
     # caller has already gated on its own allowlist, so this must not re-check it against a
     # second, separately-maintained copy.

.github/workflows/release-constraints.yml

@@ -202,6 +202,11 @@ jobs:
     timeout-minutes: 20
     name: "Publish and tag constraints"
     needs: [build-info, generate-constraints]
+    # Shared with the publish jobs of update-constraints-on-push.yml and
+    # update-constraints-on-push-stable.yml, so they never push to the same constraints branch at once.
+    concurrency:
+      group: constraints-publish-${{ needs.build-info.outputs.target-branch }}
+      cancel-in-progress: false
     permissions:
       contents: write
       packages: read

.github/workflows/release_dockerhub_image.yml

@@ -35,7 +35,7 @@ permissions:
   contents: read
   packages: read
 concurrency:
-  group: ${{ github.event.inputs.airflowVersion }}
+  group: release-dockerhub-image-${{ github.event.inputs.airflowVersion }}
   cancel-in-progress: true
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/update-constraints-on-push-stable.yml

@@ -119,6 +119,10 @@ jobs:
     timeout-minutes: 20
     name: "Commit and push constraints"
     needs: [build-info, generate-constraints]
+    # Shared with the other constraints publishers, so they never push to one branch at once.
+    concurrency:
+      group: constraints-publish-${{ needs.build-info.outputs.default-constraints-branch }}
+      cancel-in-progress: false
     permissions:
       contents: write
       packages: read

.github/workflows/update-constraints-on-push.yml

@@ -210,6 +210,11 @@ jobs:
     timeout-minutes: 20
     name: "Commit and push constraints"
     needs: [build-info, generate-constraints]
+    # Shared with the other constraints publishers. The baseline check below still decides
+    # whether this run may publish at all; this only stops the two pushes overlapping.
+    concurrency:
+      group: constraints-publish-${{ needs.build-info.outputs.default-constraints-branch }}
+      cancel-in-progress: false
     permissions:
       contents: write
       packages: read

dev/breeze/doc/09_release_management_tasks.rst

@@ -1055,6 +1055,17 @@ To check what documents will be published to S3, you can use the ``--dry-run`` f
      breeze release-management publish-docs --source-dir-path /User/pavan/airflow-site/docs-archive
      --destination-location s3://airflow-docs/docs --dry-run
 
+After the sync the command rebuilds ``manifest/packages-metadata.json`` from a scan of every package in
+the bucket. When several publishes to one bucket can run at once, sync with ``--skip-packages-metadata``
+and rebuild the file in a separate, serialised step with ``--packages-metadata-only``, which needs no
+``--source-dir-path``. The ``Publish Docs to S3`` workflow does this so that provider and Airflow docs
+publishes can sync side by side.
+
+.. code-block:: bash
+
+     breeze release-management publish-docs-to-s3 --destination-location s3://airflow-docs/docs \
+     --packages-metadata-only
+
 These are all available flags of ``release-management publish-docs-to-s3`` command:
 
 .. image:: ./images/output_release-management_publish-docs.svg