From c6c3391b47db8daa2728feb7b4ad54af80c1d0a7 Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 20 Nov 2026 04:25:03 -0800 Subject: [PATCH 1/6] feat(telemetry-cost-contract): new reusable warn-only cost-contract gate (comfy-infra#2684 task 1) Cost-governance lane C3 (comfy-observability D315) needs a PR-time check that a new billable telemetry call site carries a `## Cost impact` explanation. Pattern-copies linear-ticket.yml's fork-safe signal/validate split; detection reads only GitHub's changed-files patch fragments (no checkout, no secret needed). Repo-specific `patterns` and thin callers for cloud/frontend are the issue's remaining tasks 2-5. --- .github/workflows/telemetry-cost-contract.yml | 179 ++++++++ .../test-telemetry-cost-contract.yml | 48 ++ AGENTS.md | 4 +- README.md | 1 + docs/callers/README.md | 1 + docs/callers/telemetry-cost-contract.md | 151 +++++++ scripts/telemetry-cost-contract/README.md | 24 + scripts/telemetry-cost-contract/lib.py | 209 +++++++++ .../telemetry-cost-contract/tests/test_lib.py | 170 ++++++++ scripts/telemetry-cost-contract/validate.py | 409 ++++++++++++++++++ 10 files changed, 1194 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/telemetry-cost-contract.yml create mode 100644 .github/workflows/test-telemetry-cost-contract.yml create mode 100644 docs/callers/telemetry-cost-contract.md create mode 100644 scripts/telemetry-cost-contract/README.md create mode 100644 scripts/telemetry-cost-contract/lib.py create mode 100644 scripts/telemetry-cost-contract/tests/test_lib.py create mode 100644 scripts/telemetry-cost-contract/validate.py diff --git a/.github/workflows/telemetry-cost-contract.yml b/.github/workflows/telemetry-cost-contract.yml new file mode 100644 index 00000000..9a31fa28 --- /dev/null +++ b/.github/workflows/telemetry-cost-contract.yml @@ -0,0 +1,179 @@ +name: Telemetry Cost Contract (reusable) + +# Reusable gate requiring a PR that adds a new billable telemetry call site (per the +# caller's `patterns`) to explain the cost impact in its own body under a `## Cost impact` +# heading, or carry the `cost-exempt` label. Backs cost-governance lane C3 +# (comfy-observability program, D315): C2 found ten zero-consumer custom Datadog metrics +# that no PR-time check caught, only a manual inventory months after merge. +# +# TWO-STAGE, FORK-SAFE EXECUTION — same shape as linear-ticket.yml, same reason: this +# privileged validator is invoked by a caller that runs on `workflow_run: completed` of an +# unprivileged, no-op signal workflow in the consumer repo, so GitHub grants THIS run a +# write token even for a fork or Dependabot PR. Constraints (identical to linear-ticket.yml): +# * the signal workflow checks out nothing, runs no PR content, emits no artifact/cache; +# * this validator never downloads that run's artifacts/caches and never checks out the PR +# branch or merge commit — detection reads ONLY the `patch` fragments +# `GET .../pulls/{n}/files` returns (GitHub-owned metadata, not a checkout); +# * every PR-derived value (branch, title, body, labels, diff patch text) is untrusted +# DATA, passed through env, never shell-interpolated. +# The job runs on the DEFAULT branch, so its job-level check must NOT be the required merge +# check. validate.py publishes the `Telemetry cost contract` commit status on the refetched +# PR head SHA instead; branch protection requires THAT context. +# +# REPORTING MODES (identical semantics to linear-ticket.yml): +# enforce: true red status, job exits nonzero. Gating (once required). +# enforce: false, soft-fail: true red status, job stays green on a failing VERDICT. Loud +# (the default) pilot rung: shows on the PR like enforce does but blocks +# nothing while the context is not required. +# enforce: false, soft-fail: false green status; only the job summary and marker comment +# carry the verdict. The silent variant. +# +# PERMISSIONS the caller must grant (validated by GitHub at STARTUP, before any `if:`): +# actions: read # read the triggering workflow_run +# contents: read # checkout THIS repo's scripts (no PR code is ever checked out) +# pull-requests: write # read changed-file patches; the one marker diagnostic comment +# statuses: write # publish the `Telemetry cost contract` commit status +# +# No secret is required — detection and the Cost-impact-section check both read only +# GitHub-owned PR data. +# +# Caller pattern: docs/callers/telemetry-cost-contract.md. + +on: + workflow_call: + inputs: + workflows_ref: + description: >- + Full 40-hex commit SHA of THIS repo to load the validator scripts from. Pin to the + SAME SHA as the caller's `uses:` line. Required; validated to be a full lowercase + SHA and an ancestor of this repo's main before checkout. + type: string + required: true + patterns: + description: >- + Required. Newline-separated regexes (Python `re`); a match against an ADDED diff + line in any changed file marks the PR cost-sensitive. Newline-separated, not comma: + a regex alternation commonly contains a comma. Blank lines and `#`-comment lines + are skipped. At least one pattern must survive, and every surviving line must + compile, or the run fails as a caller misconfiguration (an empty pattern set would + silently disable the gate on every PR). + type: string + required: true + exempt-label: + description: >- + Single label that waives the requirement (recommended `cost-exempt`). Empty + disables exemption. + type: string + required: false + default: 'cost-exempt' + enforce: + description: >- + true (default) fails closed on a missing Cost-impact section or infrastructure + error, and the job itself exits nonzero. false is warn-only: a failing VERDICT + never exits the job nonzero, `soft-fail` decides how loudly it is reported. Warn-only + is not a promise the job always exits 0 — a broken run still does (a failed terminal + status write, a missing repo, malformed `patterns`, a non-`pull_request` trigger). + type: boolean + required: false + default: true + soft-fail: + description: >- + Warn-only only (ignored when `enforce: true`). true (default) publishes a red + `failure` commit status so the PR's check list shows the check failing — loud, but + non-blocking for as long as `Telemetry cost contract` is not a required status. + false restores the silent variant: warn-only publishes `success` and only the job + summary and comment carry the verdict. An ABSENT value (pin skew between `uses:` and + `workflows_ref`) is treated as false, so skew alone can never flip a caller to red. + type: boolean + required: false + default: true + secrets: {} + +permissions: {} + +concurrency: + group: telemetry-cost-contract-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: true + +jobs: + validate: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + pull-requests: write + statuses: write + steps: + - name: Require a pinned workflows_ref + # Identical trust-boundary check to linear-ticket.yml's step of the same name: SHAPE + # (full 40-hex lowercase SHA) proves the value is immutable; ANCESTRY (reachable from + # this repo's upstream main) proves it names a MERGED commit, not a fork-authored SHA + # that happens to be well-shaped. See linear-ticket.yml's step for the full rationale. + env: + WORKFLOWS_REF: ${{ inputs.workflows_ref }} + run: | + set -euo pipefail + REF="$(printf '%s' "$WORKFLOWS_REF" | tr -d '[:space:]')" + if [ -z "$REF" ]; then + echo "::error::workflows_ref is required; pin it to the same commit SHA as the uses: line (see the caller doc)" + exit 1 + fi + safe_ref=$(printf '%s' "$REF" | tr -c 'A-Za-z0-9._/-' '?'); safe_ref=${safe_ref:0:64} + if [ "${#REF}" -ne 40 ] || [ -n "${REF//[0-9a-f]/}" ]; then + echo "::error::workflows_ref must be a full 40-hex lowercase commit SHA of Comfy-Org/github-workflows (got '${safe_ref}'). Branches, tags and PR-head refs are mutable and are rejected before checkout." + exit 1 + fi + scratch=$(mktemp -d) || { echo "::error::ancestry check for workflows_ref (${safe_ref}) could not allocate scratch space; failing closed."; exit 1; } + trap 'rm -rf "$scratch"' EXIT + git init -q "$scratch" || { echo "::error::ancestry check for workflows_ref (${safe_ref}) could not init a scratch repo; failing closed."; exit 1; } + fetch_upstream() { + for attempt in 1 2 3; do + if timeout 40 git -C "$scratch" -c http.lowSpeedLimit=1000 -c http.lowSpeedTime=30 \ + fetch --quiet --no-tags --filter=blob:none "$@"; then + return 0 + fi + [ "$attempt" -lt 3 ] && sleep $((attempt * 5)) + done + return 1 + } + if ! fetch_upstream --depth=1 https://github.com/Comfy-Org/github-workflows "$REF"; then + echo "::error::workflows_ref (${safe_ref}) could not be fetched from Comfy-Org/github-workflows after 3 attempts; failing closed." + exit 1 + fi + if ! fetch_upstream https://github.com/Comfy-Org/github-workflows +refs/heads/main:refs/heads/upstream-main; then + echo "::error::main of Comfy-Org/github-workflows could not be read after 3 attempts, so workflows_ref (${safe_ref}) cannot be shown merged upstream; failing closed." + exit 1 + fi + ancestry_rc=0 + GIT_NO_LAZY_FETCH=1 git -C "$scratch" merge-base --is-ancestor "$REF" upstream-main || ancestry_rc=$? + if [ "$ancestry_rc" -gt 1 ]; then + echo "::error::the ancestry test for workflows_ref (${safe_ref}) could not COMPLETE (git error, not a verdict); failing closed." + exit 1 + fi + if [ "$ancestry_rc" -ne 0 ]; then + echo "::error::workflows_ref (${safe_ref}) is not an ancestor of Comfy-Org/github-workflows main. A well-shaped SHA can still be fork-authored; only merged upstream commits may be pinned. If this is a genuine just-merged pin, re-run." + exit 1 + fi + - name: Checkout validator scripts + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Comfy-Org/github-workflows + ref: ${{ inputs.workflows_ref }} + path: .telemetry-cost-contract-tools + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Validate the telemetry cost contract + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PATTERNS: ${{ inputs.patterns }} + EXEMPT_LABEL: ${{ inputs.exempt-label }} + ENFORCE: ${{ inputs.enforce }} + SOFT_FAIL: ${{ inputs.soft-fail }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: python3 .telemetry-cost-contract-tools/scripts/telemetry-cost-contract/validate.py diff --git a/.github/workflows/test-telemetry-cost-contract.yml b/.github/workflows/test-telemetry-cost-contract.yml new file mode 100644 index 00000000..57ae1db5 --- /dev/null +++ b/.github/workflows/test-telemetry-cost-contract.yml @@ -0,0 +1,48 @@ +# CONTRACT: this is one of THIS repo's own path-filtered script tests, NOT a reusable +# workflow. It is not callable and has no caller pattern: +# TRIGGERS — `pull_request` and `push` to main, both filtered to +# scripts/telemetry-cost-contract/** and this file. +# INPUTS — none. SECRETS — none. Read-only `contents: read`; the suite is hermetic and +# makes no network call. +# The reusable workflow these scripts back is .github/workflows/telemetry-cost-contract.yml. + +name: Test telemetry-cost-contract scripts + +# Runs the hermetic lib.py unittest suite for the telemetry-cost-contract gate. The gate +# decides whether every consuming repo's PRs can merge once enforced, so a regression in +# pattern parsing, diff-added-line extraction, or the Cost-impact-section check silently +# passes or blocks PRs — cheap to guard with a unit run on change. + +on: + pull_request: + paths: + - 'scripts/telemetry-cost-contract/**' + - '.github/workflows/test-telemetry-cost-contract.yml' + - '.github/workflows/telemetry-cost-contract.yml' + push: + branches: [main] + paths: + - 'scripts/telemetry-cost-contract/**' + - '.github/workflows/test-telemetry-cost-contract.yml' + - '.github/workflows/telemetry-cost-contract.yml' + +permissions: + contents: read + +jobs: + test: + name: unittest + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Compile the validator (syntax gate for validate.py, which the suite does not import) + run: python3 -m py_compile scripts/telemetry-cost-contract/lib.py scripts/telemetry-cost-contract/validate.py + - name: lib suite + run: python3 -m unittest discover -s scripts/telemetry-cost-contract/tests -p 'test_*.py' -v diff --git a/AGENTS.md b/AGENTS.md index d574331d..a6252ce9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ whatever you touched. ```bash # Python suites — agents-md-integrity, coderabbit-config, cursor-approve, # cursor-review, groom, linear-ticket, public-repo-hygiene, refresh-reviewers, -# workflow-pins: +# telemetry-cost-contract, workflow-pins: python3 -m unittest discover -s /tests -p 'test_*.py' -v # coderabbit-config first: pip install --require-hashes --only-binary=:all: -r @@ -79,7 +79,7 @@ anything there; `.github/workflows/` and `scripts/check-pr-size/` are the except v0 grades deterministically — **no LLM in the grading path; keep it that way**. v1 plans a split on `/derisk`, the ONLY place a model runs, and every floor it shows comes from v0's grader. A pr-derisk caller executes BOTH trees. -- `scripts/{area-label,linear-ticket,check-pr-size}/` — logic behind those reusables. +- `scripts/{area-label,linear-ticket,telemetry-cost-contract,check-pr-size}/` — logic behind those reusables. - `README.md` (public catalog) + `docs/callers/` (per-reusable setup guides) — keep both in sync when you add a workflow. diff --git a/README.md b/README.md index 5fca45bb..23b2489c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ complete, copy-pasteable caller. |---|---|---| | [`detect-unreviewed-merge.yml`](.github/workflows/detect-unreviewed-merge.yml) | SOC 2 compliance — detects PRs merged without prior approval and opens a tracking issue in [`Comfy-Org/unreviewed-merges`](https://github.com/Comfy-Org/unreviewed-merges). | [detect-unreviewed-merge.md](docs/callers/detect-unreviewed-merge.md) | | [`linear-ticket.yml`](.github/workflows/linear-ticket.yml) | **Requires a Linear issue that Linear has LINKED to the exact PR when the PR targets a protected base branch** — not a `TEAM-123`-shaped string in author text. The validator reads GitHub's branch `protected` property, so any number of branches protected by branch protection or rulesets are covered without a caller-maintained list; unprotected targets publish no `Linear ticket` status and do not query Linear (a success status would be unsafe because GitHub scopes statuses to a SHA, not a PR). For protected targets, the only thing that passes is an attachment Linear returns for the PR's canonical `html_url` (`attachmentsForURL`, PR URL passed as a GraphQL **variable**) whose issue satisfies the configured team (`team-keys`, matched on the resolved API `team.key`, never a prefix) and state (`require-open-issue` rejects `completed`/`canceled` via `state.type`) policy; a PR linked to several tickets passes when at least one satisfies policy. Identifiers mined from branch/title/body are **diagnostics only** (capped at 20, one batched lookup) — they explain a red check, never turn it green. Fork- and Dependabot-safe by a two-stage split: an unprivileged, checks-out-nothing **signal** workflow on `pull_request`, then a privileged validator on `workflow_run` that GitHub grants the secret + write token. It publishes the required **`Linear ticket`** commit status on the refetched PR head SHA (its own default-branch job is *not* the merge check), refetches the head SHA and base branch before every terminal write so a superseded or retargeted run can't overwrite a newer result, maintains one marker diagnostic comment, and **fails closed** on unreadable branch protection or Linear auth/schema/timeout/rate-limit errors (an infrastructure failure, never a green pass) with bounded retry for Linear's async-link race. `enforce: false` is warn-only — a failing verdict never exits the job nonzero (a broken run still does), and by default (`soft-fail: true`) it still publishes the **red** status so the pilot is visible on the PR while blocking nothing, since blocking is branch protection requiring the context, not this workflow's call (`soft-fail: false` restores the silent always-green variant); one `exempt-label` (e.g. `linear-exempt`) waives it per-PR, and an opt-in `exempt-actors` list (empty by default) skips bot PRs (e.g. `dependabot[bot]`) with no built-in bypass. An opt-in `exempt-paths` list exempts a PR whose changed files **all** match (config-only chores arriving from many different authors); it is decided IN the validator — reading the GitHub-owned changed-file list, failing closed past that endpoint's silent 3000-file cap — precisely so an exempt PR still publishes the green required status, which a `paths-ignore:` on the signal workflow could not (an unpublished required context leaves the PR pending forever). Logic + hermetic tests live in [`scripts/linear-ticket/`](scripts/linear-ticket); `workflows_ref` is **required**, shape- and ancestry-checked before checkout. Requires `LINEAR_API_TOKEN` + the two-workflow caller. | [linear-ticket.md](docs/callers/linear-ticket.md) | +| [`telemetry-cost-contract.yml`](.github/workflows/telemetry-cost-contract.yml) | **Requires a `## Cost impact` PR-body section when a PR adds a new billable telemetry call site.** Scans every changed file's diff for an added line matching the caller's `patterns` (new Datadog metrics-client calls, new Sentry captures, new PostHog/RUM capture calls — repo-specific) via GitHub's own changed-files `patch` fragments, never a checkout. A match requires either a `## Cost impact` section with real content (priced unit, baseline/scale forecast, protected signal — the observability program's TDD §7.1 contract) in the PR body, or the `cost-exempt` label; a PR with no match passes trivially, and an unprotected base branch skips entirely. No secret is required. Fork- and Dependabot-safe by the same two-stage `pull_request` → `workflow_run` split as `linear-ticket.yml`, publishing the required **`Telemetry cost contract`** commit status on the refetched PR head SHA, maintaining one marker comment, and failing closed on an unreadable/truncated changed-files read. `enforce: false` (default) is warn-only with the same loud/silent `soft-fail` modes as `linear-ticket.yml`. Logic + hermetic tests live in [`scripts/telemetry-cost-contract/`](scripts/telemetry-cost-contract); `workflows_ref` is **required**, shape- and ancestry-checked before checkout. | [telemetry-cost-contract.md](docs/callers/telemetry-cost-contract.md) | | [`cursor-review.yml`](.github/workflows/cursor-review.yml) | Label-triggered multi-model code review. A 3-lab × 2-review-type cursor-agent panel runs adversarial + edge-case passes, a judge model consolidates them into one PR review with per-finding severity badges, and the triggerer gets Slack start/complete DMs. Advisory by default: it posts a review, it does not gate. Opt-in `blocking: true` adds a fail-closed **Blocking gate** check — red while any finding thread is unresolved and non-outdated, and red whenever the round that should have produced those threads did not land (the trigger job failed, the post exited zero without reaching the PR, every finding was demoted to the review body, the diff was over `diff_size_cap`, or the run was cancelled) — which blocks merge once the caller's ruleset marks it required (two switches; see [the setup guide](docs/callers/cursor-review.md)). Separately and **always on**, a **Panel integrity** check reports whether the panel that reviewed the PR was a *whole* panel — red when fewer reviewer cells submitted than ran, when findings could not be anchored to the diff and were demoted to the review body, when no review carrying resolvable threads reached the PR, when the judge never adjudicated, or when the jobs that decide whether to review at all did not succeed (an undecided run is not a clean run; a run cancelled after a newer push passes it without judging, while any other cancel stays red). It takes no input and gates nothing by itself; ` / Panel integrity` is the stable context to mark required if you want completeness gated — **not** an individual panel leg, whose name carries the model id and changes whenever the pins move. Two consequences to read before bumping onto it: a cell that does not submit now fails its own leg instead of exiting green, so **your caller job goes red** on a measured ~40% of runs (that red is the signal — the consolidated review still posts, and the Blocking gate is unaffected); and the deliberate no-panel branches leave it *skipped*, which a required check counts as passing, so a skip answers "no panel was warranted", never "the panel was fine". Scope and limits — including forgeability and same-SHA supersession — are in [the setup guide](docs/callers/cursor-review.md). Prompts and scripts live in [`.github/cursor-review/`](.github/cursor-review) — the single source of truth, so consumer repos carry only a thin caller. Generated code is excluded from BOTH the diff-size gate and the diff sent to the panel via the shared [`check-pr-size`](scripts/check-pr-size) classifier (base-ref `linguist-generated`, base-blob Go generated markers — a PR cannot self-exempt a file from review, lockfiles, plus `extra_generated_globs` / `extra_lockfiles`) — the same detection as `pr-size.yml`, so codegen never blows the review cap or wastes review signal. Blank and comment-only lines are also discounted from the size count by default (`ignore_comments`) so AI comment volume doesn't trip the cap and skip the review — comments still reach the panel, only the count ignores them. The panel model list is fixed by the reusable — there is no `models` input (`models` is a job *output*); an optional `runs_on` routes only the credential-free heavy jobs (`diff-size`, `preflight`, panel, judge) to a caller's runner pool, while every write-credential job stays on GitHub-hosted `ubuntu-latest`. Least privilege by construction: no job both checks out PR code and holds a write-scoped credential — every job that checks out PR code and runs `cursor-agent` over it (every panel cell and the judge) holds `contents: read` only, and hands its result to a separate, checkout-free `Post review` job via an artifact; that job and the over-the-cap comment job are the only holders of `pull-requests: write`, and neither checks out PR code. Requires `CURSOR_API_KEY` (+ optional `SLACK_BOT_TOKEN`). | [cursor-review.md](docs/callers/cursor-review.md) | | [`cursor-approve.yml`](.github/workflows/cursor-approve.yml) + [`axis-correctness.yml`](.github/workflows/axis-correctness.yml) / [`axis-conformance.yml`](.github/workflows/axis-conformance.yml) / [`axis-business.yml`](.github/workflows/axis-business.yml) / [`axis-design.yml`](.github/workflows/axis-design.yml) / [`axis-completeness.yml`](.github/workflows/axis-completeness.yml) | Per-axis approval after a passing cursor-review round (`approve_gate == 'pass'`). Each axis is its own reusable declaring only the secrets it needs (correctness/conformance: `CURSOR_API_KEY` only; business/design/completeness add optional read-only Linear/Notion tokens, held by a local read-only MCP proxy the agent never sees the tokens of, private repos only), runs one read-only `cursor-agent` judgement over the PR head via the shared [`cursor-axis-base.yml`](.github/workflows/cursor-axis-base.yml), and uploads one verdict artifact. `cursor-approve.yml`, called with `phase: start` and `phase: decide`, keeps one live status card on the PR and approves through `auto-approve.py approve-external` (head re-read, reviewed-SHA marker, own stale approvals withdrawn, never over `needs-human-review` or `skip-cursor-review`, both re-read live). Prompts and the aggregator live in [`.github/cursor-approve/`](.github/cursor-approve). Requires `CURSOR_API_KEY` + `APPROVER_TOKEN`. | [cursor-approve.md](docs/callers/cursor-approve.md) | | [`cursor-review-auto-label.yml`](.github/workflows/cursor-review-auto-label.yml) | Companion to `cursor-review.yml`. On PR assignment, applies the review label for an opted-in reviewer (via the CLOUD_CODE_BOT app token, so the label actually triggers the review). The opt-in roster lives in the caller's `vars.CURSOR_REVIEW_OPTED_IN_LOGINS` — no roster is baked into the workflow. Requires `vars.APP_ID` + `CLOUD_CODE_BOT_PRIVATE_KEY`, or the optional `bot_app_id` + `BOT_APP_PRIVATE_KEY` pair to mint from a narrower App instead. | [cursor-review-auto-label.md](docs/callers/cursor-review-auto-label.md) | diff --git a/docs/callers/README.md b/docs/callers/README.md index 315d5cbc..48ff4a16 100644 --- a/docs/callers/README.md +++ b/docs/callers/README.md @@ -22,6 +22,7 @@ per-workflow pages give you a complete, copy-pasteable caller: | `stale.yml` | [stale.md](stale.md) | `SLACK_BOT_TOKEN` (optional) | | `detect-unreviewed-merge.yml` | [detect-unreviewed-merge.md](detect-unreviewed-merge.md) | `UNREVIEWED_MERGES_TOKEN` | | `linear-ticket.yml` | [linear-ticket.md](linear-ticket.md) | `LINEAR_API_TOKEN` + a two-workflow caller | +| `telemetry-cost-contract.yml` | [telemetry-cost-contract.md](telemetry-cost-contract.md) | `patterns` input + a two-workflow caller | Everything below applies to all of them. Read it once. diff --git a/docs/callers/telemetry-cost-contract.md b/docs/callers/telemetry-cost-contract.md new file mode 100644 index 00000000..5a90b5a4 --- /dev/null +++ b/docs/callers/telemetry-cost-contract.md @@ -0,0 +1,151 @@ +# `telemetry-cost-contract.yml` — require a cost-impact explanation for new billable telemetry + +Read [the shared caller contract](README.md) first. + +## What it does + +For a PR targeting a **protected base branch**, scans every changed file's diff for an +**added** line matching one of this caller's `patterns` regexes (a new Datadog metrics-client +call, a new Sentry capture, a new PostHog `.capture(`, a new RUM custom action — whatever the +repo's `patterns` input names). When a match is found, the PR body must carry a `## Cost +impact` section with real content (priced unit, baseline/scale forecast, protected signal — +the observability program's TDD section 7.1 contract), or the PR must carry the `cost-exempt` +label. A PR with no matching added line passes trivially; a PR targeting an unprotected branch +skips without scanning at all. + +Detection reads only the `patch` fragments GitHub's changed-files API returns — never a +checkout of the PR branch — so it is safe to run on fork and Dependabot PRs. + +### Why two workflows + +Same reason as `linear-ticket.yml`: the validator needs a write token (to publish the commit +status and the marker comment) that GitHub withholds from fork and Dependabot PR runs. The fix +is the standard **`pull_request` → `workflow_run`** split: + +- an unprivileged **signal** workflow runs on the PR (`contents: read`, no secret, checks out + nothing), and +- a privileged **validate** workflow runs on `workflow_run: completed` of the signal workflow + and calls this reusable, which GitHub grants a write token even for a fork PR. + +The validate job runs on your **default branch**, so its job-level check is *not* the thing +you require in branch protection. This reusable publishes a commit status named +**`Telemetry cost contract`** on the PR head SHA; you require **that context** once warn-only +observation is done. + +## Prerequisites + +None — no secret is required. Detection and the Cost-impact-section check both read only +GitHub-owned PR data (changed-file patches, PR body, PR labels). + +## Caller — two files + +### 1. `.github/workflows/telemetry-cost-contract-signal.yml` (unprivileged, no-op) + +```yaml +name: Telemetry Cost Contract Signal # this exact name is referenced by the validate workflow + +on: + pull_request: + types: [opened, edited, reopened, ready_for_review, synchronize, labeled, unlabeled] + +permissions: + contents: read + +jobs: + signal: + runs-on: ubuntu-latest + steps: + - name: Signal + run: echo "Signalling the telemetry-cost-contract validator (no PR content is checked out)." +``` + +The `labeled`/`unlabeled` types are required so adding or removing `cost-exempt` reruns the +check. This workflow checks out nothing and emits no artifact — that is what makes the +`workflow_run` privilege boundary safe. + +### 2. `.github/workflows/telemetry-cost-contract-validate.yml` (privileged, calls this reusable) + +```yaml +name: Telemetry Cost Contract + +on: + workflow_run: + workflows: ["Telemetry Cost Contract Signal"] # must match the signal workflow's name exactly + types: [completed] + +jobs: + validate: + permissions: + actions: read + contents: read + pull-requests: write + statuses: write + uses: Comfy-Org/github-workflows/.github/workflows/telemetry-cost-contract.yml@ # v1 + with: + workflows_ref: + patterns: | + statsd\.(Gauge|Count|Histogram|Distribution)\( + sentrygo\.CaptureException\( + # exempt-label: cost-exempt # default; empty disables exemption + enforce: false # WARN-ONLY during rollout; flip to true when ready + # soft-fail: true # default; warn-only still shows a RED (non-blocking) check +``` + +`patterns` is **newline-separated**, not comma-separated — a regex alternation like the Go +example above routinely contains no comma, but a comma-separated grammar would corrupt a +pattern that did. Blank lines and `#`-comment lines are skipped. + +Example `patterns` for a TypeScript/frontend repo: + +```yaml + with: + patterns: | + posthog\.capture\( + Sentry\.(captureException|addBreadcrumb)\( + useRumAction\( +``` + +## Required permissions + +```yaml +actions: read +contents: read +pull-requests: write +statuses: write +``` + +**Grant the whole union** — see [the shared caller contract](README.md#permissions) for why a +short grant fails the whole run with no job-level detail. + +## Inputs + +| Input | Default | Notes | +|---|---|---| +| `workflows_ref` | — (**required**) | Pin to the SAME full commit SHA as `uses:`. Validated to be a full 40-hex lowercase SHA **and an ancestor of this repo's main** before checkout. | +| `patterns` | — (**required**) | Newline-separated regexes (Python `re`). A match against an added diff line marks the PR cost-sensitive. Blank/`#`-comment lines are skipped. At least one must survive and every one must compile, or the run fails as a caller misconfiguration. | +| `exempt-label` | `cost-exempt` | Single label that waives the requirement. Empty disables exemption. | +| `enforce` | `true` | `false` is warn-only: a failing **verdict** never exits the job nonzero. `soft-fail` decides how loudly it is reported. | +| `soft-fail` | `true` | Warn-only only. `true` publishes a red `failure` status (loud, non-blocking while the context is not required). `false` restores the silent variant (always-green warn-only). | + +## Rollout (warn-only first) + +1. Land both caller files with `enforce: false`, and leave `Telemetry cost contract` out of + your ruleset's required checks. Observe real PRs for about a week: does `patterns` catch + the call sites you expect, does it false-positive on moved/renamed lines, how often is + `cost-exempt` actually used. +2. Tune `patterns` from the observed false positives/negatives. +3. Flip `enforce: true`. The check still is not required, so nothing blocks yet — this step + only makes a systemic breakage visible in the Actions tab. +4. Add the **`Telemetry cost contract`** status context as a required check in the rulesets + that should enforce it. Only now does anything block. + +Nothing here changes branch protection automatically — the last step is a manual repo-admin +action, matching every other reusable in this catalog. + +## Gotchas + +Same two gotchas as `linear-ticket.yml` (read that page's Gotchas section for the full +rationale): the required check is the commit *status* `Telemetry cost contract`, not the job +`Telemetry Cost Contract / validate` which runs on your default branch; and a red check while +warn-only is a visible, non-blocking signal, not a stuck merge — what blocks is your ruleset +requiring the context. diff --git a/scripts/telemetry-cost-contract/README.md b/scripts/telemetry-cost-contract/README.md new file mode 100644 index 00000000..861c293f --- /dev/null +++ b/scripts/telemetry-cost-contract/README.md @@ -0,0 +1,24 @@ +# `scripts/telemetry-cost-contract/` — the cost-impact-explanation gate + +The scripts behind +[`telemetry-cost-contract.yml`](../../.github/workflows/telemetry-cost-contract.yml). A +consumer repo runs the reusable workflow via a thin two-workflow caller (signal + validate) +and keeps no logic of its own beyond its `patterns` input; this directory holds the decision +path, loaded from the `workflows_ref` SHA the caller pins. Standard-library Python only, +matching this repo's convention. + +| File | Role | +|---|---| +| `lib.py` | Pure, network-free core — `patterns` parsing, diff-added-line extraction, the detection match (`first_match`), the `## Cost impact` section check, failure-category selection and copy. Imported by `validate.py` and the test suite. | +| `validate.py` | The side-effecting orchestration: resolve the PR from the `workflow_run` event, skip unprotected base branches, read changed files with diff patches via `GET .../pulls/{n}/files` (GitHub-owned metadata, no checkout), apply the gate, publish the `Telemetry cost contract` commit status, and maintain the single marker PR comment. GitHub via the `gh` CLI. Imports `lib.py`. | +| `tests/` | Hermetic `unittest` suite over the pure core — no network. | + +## The invariant + +A PR only turns red when an **added** diff line matches a caller-configured pattern AND the +PR body has no non-empty `## Cost impact` section AND the PR carries no exemption label. +Detection reads only the `patch` fragments GitHub's changed-files API already returns — +nothing here checks out the PR branch or executes PR content, which is what makes it safe to +run in the privileged `workflow_run` validator job on a fork or Dependabot PR. See +[`docs/callers/telemetry-cost-contract.md`](../../docs/callers/telemetry-cost-contract.md) for +the caller setup and rollout. diff --git a/scripts/telemetry-cost-contract/lib.py b/scripts/telemetry-cost-contract/lib.py new file mode 100644 index 00000000..9445d51b --- /dev/null +++ b/scripts/telemetry-cost-contract/lib.py @@ -0,0 +1,209 @@ +"""Pure, network-free core of the telemetry-cost-contract gate. + +A PR that adds a NEW billable telemetry call site (a Datadog metrics client call, a new +Sentry capture, a PostHog `.capture(`, a new RUM custom action — whatever regex the calling +repo configures via `patterns`) must either explain the cost impact in its own body, under a +`## Cost impact` heading (the TDD section 7.1 contract: priced unit, baseline/scale forecast, +protected signal), or carry the `cost-exempt` label. This is the lightweight repository check +that cost-governance lane C3 (comfy-observability program, D315) asked for, so the ten +zero-consumer custom metrics C2 found months after merge get caught at PR time instead. + +Detection reads ONLY GitHub-owned PR metadata — the `patch` fragments `GET +.../pulls/{n}/files` returns for each changed file — never a checkout of the PR branch. That +is what makes this safe to run on a fork PR's privileged `workflow_run` validator: the +untrusted diff text is pattern-MATCHED, never executed or interpolated. + +Side-effecting orchestration (resolve the PR, read its files, publish the commit status, +upsert the comment) lives in validate.py, which imports this. Standard library only, matching +this repo's Python convention. +""" + +from __future__ import annotations + +import re +from typing import NamedTuple + +# The hidden marker on the single PR comment this gate maintains. +MARKER = "" + +# The commit-status context. Human-facing AND a contract (see linear-ticket's CONTEXT for why +# a rename silently retires a required check) — check every caller's ruleset before renaming. +CONTEXT = "Telemetry cost contract" + +# A unified diff's hunk lines: '+' or '-' prefixed content lines, vs. the '+++'/'---' file +# headers and '@@' hunk headers, which must never be mistaken for added content. +_ADDED_LINE_RE = re.compile(r"^\+(?!\+\+)(.*)$", re.MULTILINE) + +_COST_IMPACT_HEADING_RE = re.compile(r"^#{1,6}\s*cost impact\b", re.IGNORECASE | re.MULTILINE) +_ANY_HEADING_RE = re.compile(r"^#{1,6}\s", re.MULTILINE) + + +def parse_patterns(raw: str) -> list[re.Pattern]: + """Parse the caller's newline-separated ``patterns`` input into compiled regexes. + + Newline-separated, not comma-separated: a regex alternation (`a|b`) is common here and a + comma inside a character class or quantifier is legal regex syntax, so comma-splitting + would silently corrupt a pattern. Blank lines and lines starting with ``#`` (comments) are + skipped. Raises ValueError — rejecting the whole run as a caller misconfiguration — when + no pattern survives, or any surviving line fails to compile: an empty pattern set would + make every PR trivially "not cost-sensitive", silently disabling the gate, and a pattern + that cannot compile is a caller typo that must not degrade to "never matches" in silence. + """ + patterns: list[re.Pattern] = [] + for line in (raw or "").splitlines(): + text = line.strip() + if not text or text.startswith("#"): + continue + try: + patterns.append(re.compile(text)) + except re.error as exc: + raise ValueError(f"invalid pattern {text!r}: {exc}") from exc + if not patterns: + raise ValueError("no patterns configured (every line was blank or a comment)") + return patterns + + +def added_lines(patch: str) -> list[str]: + """Content of every added line in a unified-diff ``patch`` fragment, '+' stripped. + + Deliberately line-moves-safe in the cheap direction GitHub already gives us: a line that + only moved position within the same hunk still appears here as added (the diff algorithm, + not this function, decides that) — this function only strips the `+`/`-`/`@@` diff syntax + from genuinely added content. A rename-only diff (file moved, content untouched) carries no + added lines and matches nothing, which is correct: nothing new was written. + """ + return [m.group(1) for m in _ADDED_LINE_RE.finditer(patch or "")] + + +def first_match(files: list[dict], patterns: list[re.Pattern]) -> tuple[str, str] | None: + """The first (filename, pattern-text) whose pattern matches an added line in some file's + patch, in file order then pattern order; None if nothing matches. + + A file with no ``patch`` key (binary, or a diff too large for GitHub to render — GitHub + omits `patch` past its ~1MB-per-file render ceiling and does not flag the omission) + contributes no lines and is silently skipped, matching the diff-added-lines-only scope: + this check is a code-review nudge, not a security boundary, so a binary/huge-diff file + simply cannot trip it either way. + """ + for entry in files or []: + patch = entry.get("patch") if isinstance(entry, dict) else None + if not patch: + continue + lines = added_lines(patch) + if not lines: + continue + blob = "\n".join(lines) + for pattern in patterns: + if pattern.search(blob): + return entry.get("filename") or "", pattern.pattern + return None + + +def has_cost_impact_section(body: str) -> bool: + """True iff ``body`` has a ``## Cost impact`` (any heading depth, case-insensitive) + heading followed by at least one non-blank line before the next heading or EOF. + + A bare heading with nothing under it (copy-pasted template, never filled in) does not + count — TDD section 7.1 requires actual content (priced unit, baseline, protected signal), + not just the section's existence. + """ + match = _COST_IMPACT_HEADING_RE.search(body or "") + if not match: + return False + rest = (body or "")[match.end():] + next_heading = _ANY_HEADING_RE.search(rest) + section_body = rest[:next_heading.start()] if next_heading else rest + return any(line.strip() for line in section_body.splitlines()) + + +def soft_fail_enabled(raw: str | None) -> bool: + """Parse the ``SOFT_FAIL`` env var. An ABSENT value means the SILENT warn-only variant — + see linear-ticket's identical helper for why pin-skew must never upgrade green to red.""" + return (raw or "").strip().lower() == "true" + + +_GUIDANCE = { + "missing_cost_impact": ( + "This PR adds a new billable telemetry call site matching this repository's " + "cost-sensitive detection patterns, but its body has no `## Cost impact` section. " + "Per the observability program's TDD section 7.1, add one naming: the priced unit " + "(events, sessions, custom-metric series, …), a baseline/scale forecast (current " + "cloud MAU, opted-in Desktop cohort, the local-MAU ceiling), and the protected signal " + "this change must not regress. If this change adds no new billable volume, apply the " + "`cost-exempt` label instead." + ), + "changed_files_unavailable": ( + "This check could not be completed because the PR's changed-file list (with diff " + "patches) could not be read completely. A partial list could miss the very file that " + "added a billable call site, so this fails closed instead of declaring a partial scan " + "clean. This is an infrastructure error, not a verdict on cost impact." + ), +} + + +def failure_guidance(category: str) -> str: + """The fixed, category-specific paragraph for the PR comment and job summary.""" + return _GUIDANCE[category] + + +_STATUS_HEADLINE = { + "missing_cost_impact": "New telemetry call site with no Cost impact section", + "changed_files_unavailable": "Could not verify — the PR's changed-file list is unreadable", +} + +_ADVISORY = ( + "> ⚠️ **Advisory — this check is warn-only.** It is reported as failing so it shows up in " + "the PR's check list, but nothing this workflow does blocks a merge: what blocks is this " + "repository's ruleset requiring the `Telemetry cost contract` status context, and a " + "warn-only pilot is meant to run *before* that context is required. If your merge button " + "is actually blocked on `Telemetry cost contract`, the context is already required while " + "the pilot is still warn-only — tell the repository owners, because that combination is a " + "misconfiguration." +) + + +def advisory_note(category: str) -> str: + if category not in _GUIDANCE: + raise KeyError(category) + return _ADVISORY + + +class FailureOutcome(NamedTuple): + verdict: str + state: str + description: str + exit_code: int + advisory: bool + + +def failure_outcome(category: str, enforce: bool, soft_fail: bool) -> FailureOutcome: + """Map (category, mode) to the reported outcome — see linear-ticket's identical table for + the full rationale: enforce gates for real; warn-only+soft-fail is loud but non-blocking; + warn-only without soft-fail is silent (status green, verdict only in the summary/comment). + """ + if category not in _GUIDANCE: + raise KeyError(category) + headline = _STATUS_HEADLINE[category] + if enforce: + return FailureOutcome( + verdict=f"❌ fail ({category})", + state="failure", + description=headline, + exit_code=1, + advisory=False, + ) + if soft_fail: + return FailureOutcome( + verdict=f"❌ fail ({category}) — warn-only", + state="failure", + description=f"Warn-only: {headline[0].lower()}{headline[1:]}", + exit_code=0, + advisory=True, + ) + return FailureOutcome( + verdict=f"⚠️ warn-only (would fail: {category})", + state="success", + description=f"Warn-only (would fail): {headline[0].lower()}{headline[1:]}", + exit_code=0, + advisory=False, + ) diff --git a/scripts/telemetry-cost-contract/tests/test_lib.py b/scripts/telemetry-cost-contract/tests/test_lib.py new file mode 100644 index 00000000..c410e75f --- /dev/null +++ b/scripts/telemetry-cost-contract/tests/test_lib.py @@ -0,0 +1,170 @@ +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +import lib + + +class ParsePatternsTests(unittest.TestCase): + def test_single_pattern(self): + patterns = lib.parse_patterns(r"statsd\.Gauge\(") + self.assertEqual(len(patterns), 1) + self.assertTrue(patterns[0].search("statsd.Gauge(foo)")) + + def test_blank_lines_and_comments_skipped(self): + patterns = lib.parse_patterns("# a comment\n\nposthog\\.capture\\(\n\n") + self.assertEqual(len(patterns), 1) + + def test_multiple_lines(self): + patterns = lib.parse_patterns("foo\nbar") + self.assertEqual(len(patterns), 2) + + def test_empty_raises(self): + with self.assertRaises(ValueError): + lib.parse_patterns("") + + def test_only_comments_raises(self): + with self.assertRaises(ValueError): + lib.parse_patterns("# nothing here\n\n") + + def test_invalid_regex_raises(self): + with self.assertRaises(ValueError): + lib.parse_patterns("foo(") + + +class AddedLinesTests(unittest.TestCase): + PATCH = ( + "@@ -1,3 +1,4 @@\n" + " unchanged line\n" + "-removed line\n" + "+added line one\n" + "+added line two\n" + ) + + def test_extracts_only_added_content(self): + lines = lib.added_lines(self.PATCH) + self.assertEqual(lines, ["added line one", "added line two"]) + + def test_file_header_not_treated_as_added(self): + patch = "--- a/x.go\n+++ b/x.go\n@@ -1 +1 @@\n+real_added_line\n" + self.assertEqual(lib.added_lines(patch), ["real_added_line"]) + + def test_empty_patch(self): + self.assertEqual(lib.added_lines(""), []) + self.assertEqual(lib.added_lines(None), []) + + +class FirstMatchTests(unittest.TestCase): + def setUp(self): + self.patterns = lib.parse_patterns(r"statsd\.Gauge\(" + "\n" + r"posthog\.capture\(") + + def test_matches_added_line(self): + files = [{"filename": "a.go", "patch": "+statsd.Gauge(\"x\", 1)\n"}] + match = lib.first_match(files, self.patterns) + self.assertEqual(match, ("a.go", r"statsd\.Gauge\(")) + + def test_no_match_on_removed_line_only(self): + files = [{"filename": "a.go", "patch": "-statsd.Gauge(\"x\", 1)\n"}] + self.assertIsNone(lib.first_match(files, self.patterns)) + + def test_no_match_when_pattern_absent(self): + files = [{"filename": "a.go", "patch": "+fmt.Println(\"hi\")\n"}] + self.assertIsNone(lib.first_match(files, self.patterns)) + + def test_file_without_patch_skipped(self): + files = [{"filename": "binary.png"}] + self.assertIsNone(lib.first_match(files, self.patterns)) + + def test_second_pattern_matches_second_file(self): + files = [ + {"filename": "a.go", "patch": "+fmt.Println(1)\n"}, + {"filename": "b.ts", "patch": "+posthog.capture('x')\n"}, + ] + match = lib.first_match(files, self.patterns) + self.assertEqual(match, ("b.ts", r"posthog\.capture\(")) + + def test_empty_files_list(self): + self.assertIsNone(lib.first_match([], self.patterns)) + + +class CostImpactSectionTests(unittest.TestCase): + def test_present_with_content(self): + body = "Summary\n\n## Cost impact\n\nBills per event; forecast 1k/day.\n\n## Testing\n" + self.assertTrue(lib.has_cost_impact_section(body)) + + def test_case_insensitive_and_any_heading_depth(self): + self.assertTrue(lib.has_cost_impact_section("### cost IMPACT\nsome text")) + + def test_absent(self): + self.assertFalse(lib.has_cost_impact_section("## Summary\nnothing about cost")) + + def test_empty_section_does_not_count(self): + body = "## Cost impact\n\n## Testing\nstuff" + self.assertFalse(lib.has_cost_impact_section(body)) + + def test_heading_with_only_whitespace_does_not_count(self): + body = "## Cost impact\n \n\n## Testing\n" + self.assertFalse(lib.has_cost_impact_section(body)) + + def test_section_at_end_of_body(self): + body = "## Cost impact\nBills $X/month." + self.assertTrue(lib.has_cost_impact_section(body)) + + def test_empty_body(self): + self.assertFalse(lib.has_cost_impact_section("")) + self.assertFalse(lib.has_cost_impact_section(None)) + + +class SoftFailEnabledTests(unittest.TestCase): + def test_true_string(self): + self.assertTrue(lib.soft_fail_enabled("true")) + self.assertTrue(lib.soft_fail_enabled("TRUE")) + + def test_absent_is_false(self): + self.assertFalse(lib.soft_fail_enabled(None)) + self.assertFalse(lib.soft_fail_enabled("")) + + def test_false_string(self): + self.assertFalse(lib.soft_fail_enabled("false")) + + +class FailureOutcomeTests(unittest.TestCase): + def test_enforce_fails_closed(self): + outcome = lib.failure_outcome("missing_cost_impact", enforce=True, soft_fail=True) + self.assertEqual(outcome.state, "failure") + self.assertEqual(outcome.exit_code, 1) + self.assertFalse(outcome.advisory) + + def test_warn_only_soft_fail_is_red_but_exits_zero(self): + outcome = lib.failure_outcome("missing_cost_impact", enforce=False, soft_fail=True) + self.assertEqual(outcome.state, "failure") + self.assertEqual(outcome.exit_code, 0) + self.assertTrue(outcome.advisory) + + def test_warn_only_silent_is_green(self): + outcome = lib.failure_outcome("missing_cost_impact", enforce=False, soft_fail=False) + self.assertEqual(outcome.state, "success") + self.assertEqual(outcome.exit_code, 0) + self.assertFalse(outcome.advisory) + + def test_unknown_category_raises(self): + with self.assertRaises(KeyError): + lib.failure_outcome("not_a_category", enforce=True, soft_fail=True) + + def test_every_category_has_guidance_and_headline(self): + for category in ("missing_cost_impact", "changed_files_unavailable"): + self.assertTrue(lib.failure_guidance(category)) + lib.failure_outcome(category, enforce=True, soft_fail=True) + + def test_descriptions_pairwise_distinct(self): + descriptions = { + lib.failure_outcome(c, enforce=True, soft_fail=True).description + for c in ("missing_cost_impact", "changed_files_unavailable") + } + self.assertEqual(len(descriptions), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/telemetry-cost-contract/validate.py b/scripts/telemetry-cost-contract/validate.py new file mode 100644 index 00000000..8b66e4d5 --- /dev/null +++ b/scripts/telemetry-cost-contract/validate.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +"""Side-effecting orchestration behind telemetry-cost-contract.yml. + +Imports lib.py for every decision and owns only I/O: resolve the PR from the workflow_run +event, refetch it, skip unprotected base branches, read its changed files WITH diff patches +(GitHub-owned metadata — no checkout of the PR branch, no secrets), apply the detection +patterns, require a `## Cost impact` PR-body section (or the exempt label) when a match is +found, publish the `Telemetry cost contract` commit status, and maintain one marker comment. + +Every value derived from the PR (branch, title, body, labels, diff patch text) is untrusted +DATA: it is only ever pattern-MATCHED, never shell-interpolated or executed. This mirrors +linear-ticket.yml's fork-safety contract — see that workflow's header for the full rationale +on why this script runs in the PRIVILEGED `workflow_run` job rather than on the PR itself. + +Contract (all via env, set by telemetry-cost-contract.yml): + GH_TOKEN caller GITHUB_TOKEN, for ``gh api`` (statuses:write, pull-requests:write) + GH_REPO owner/repo (github.repository) + GITHUB_EVENT_PATH workflow_run event payload + PATTERNS newline-separated regexes; a match against an added diff line marks + the PR cost-sensitive. Required; see lib.parse_patterns. + EXEMPT_LABEL exemption label name; empty disables exemption (default: cost-exempt, + set by the workflow, not this script) + ENFORCE "true" (fail closed) / "false" (warn-only) + SOFT_FAIL warn-only only: "true" publishes a red (non-blocking) status; "false" + or ABSENT keeps the silent always-green warn-only behaviour + RUN_URL html_url of this workflow run, for the status target and comment + GITHUB_STEP_SUMMARY written with the human-readable outcome + +Standard library only; GitHub via the ``gh`` CLI, matching linear-ticket/validate.py. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import urllib.parse + +import lib + +# GitHub's documented ceiling on GET /repos/{owner}/{repo}/pulls/{number}/files: at most 3000 +# entries, truncated SILENTLY. A short read can hide the one file that added a billable call +# site, so a PR at or past the cap fails closed rather than scanning a partial diff. +CHANGED_FILES_CAP = 3000 + + +class ChangedFilesUnavailable(Exception): + """The PR's changed-file list (with patches) could not be read, or could not be trusted + to be complete. Always fails the check closed (``changed_files_unavailable``).""" + + +def log(msg: str) -> None: + print(msg, file=sys.stderr, flush=True) + + +def error(msg: str) -> None: + print(f"::error::{msg}", file=sys.stderr, flush=True) + + +def warning(msg: str) -> None: + print(f"::warning::{msg}", file=sys.stderr, flush=True) + + +def summary(msg: str) -> None: + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + with open(path, "a", encoding="utf-8") as handle: + handle.write(msg + "\n") + + +# ── GitHub via gh ─────────────────────────────────────────────────────────────────────── +class GitHub: + def __init__(self, repo: str): + self.repo = repo + + def _run(self, args: list[str], stdin: str | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + ["gh", "api", *args], input=stdin, text=True, capture_output=True, + ) + + def get(self, path: str, *, paginate: bool = False): + args = [path] + if paginate: + args.insert(0, "--paginate") + result = self._run(args) + if result.returncode != 0: + log(f"gh api {path} failed: {result.stderr.strip()}") + return None + try: + return json.loads(result.stdout or "null") + except json.JSONDecodeError: + return None + + def publish_status(self, sha: str, state: str, description: str, target_url: str, + *, critical: bool = False) -> bool: + """Publish the commit status. A TERMINAL write is critical=True: if it fails, the old + status stays as the required context, matching linear-ticket's identical write.""" + result = self._run([ + "--method", "POST", f"/repos/{self.repo}/statuses/{sha}", + "-f", f"state={state}", + "-f", f"context={lib.CONTEXT}", + "-f", f"description={description[:140]}", + "-f", f"target_url={target_url}", + ]) + if result.returncode != 0: + emit = error if critical else warning + emit(f"Failed to publish '{state}' status on {sha}: {result.stderr.strip()}") + return False + return True + + def find_marker_comment(self, pr: int) -> int | None: + comments = self.get(f"/repos/{self.repo}/issues/{pr}/comments?per_page=100", + paginate=True) or [] + for comment in comments: + if lib.MARKER not in (comment.get("body") or ""): + continue + user = comment.get("user") or {} + if user.get("type") == "Bot" and user.get("login") == "github-actions[bot]": + return comment.get("id") + return None + + def upsert_marker_comment(self, pr: int, body: str) -> None: + existing = self.find_marker_comment(pr) + if existing is not None: + result = self._run( + ["--method", "PATCH", f"/repos/{self.repo}/issues/comments/{existing}", + "-F", "body=@-"], stdin=body) + if result.returncode != 0: + warning(f"Failed to update marker comment {existing}: {result.stderr.strip()}") + else: + result = self._run( + ["--method", "POST", f"/repos/{self.repo}/issues/{pr}/comments", + "-F", "body=@-"], stdin=body) + if result.returncode != 0: + warning(f"Failed to create marker comment: {result.stderr.strip()}") + + def delete_marker_comment(self, pr: int) -> None: + existing = self.find_marker_comment(pr) + if existing is None: + return + result = self._run(["--method", "DELETE", f"/repos/{self.repo}/issues/comments/{existing}"]) + if result.returncode != 0: + warning(f"Failed to delete marker comment {existing}: {result.stderr.strip()}") + + def changed_files(self, pr: int, declared_count: int | None = None) -> list[dict]: + """Every changed-file entry (filename + patch, if GitHub rendered one), as untrusted + DATA. Paginated at 100/page. Raises ChangedFilesUnavailable — never returns a partial + list — on the same conditions linear-ticket's identical method does: request failure, + at/past CHANGED_FILES_CAP, or disagreement with the PR's own ``changed_files`` count. + """ + if isinstance(declared_count, int) and declared_count >= CHANGED_FILES_CAP: + raise ChangedFilesUnavailable( + f"the PR reports {declared_count} changed files, at or past GitHub's " + f"{CHANGED_FILES_CAP}-file ceiling for the changed-files API, which truncates " + "silently") + data = self.get(f"/repos/{self.repo}/pulls/{pr}/files?per_page=100", paginate=True) + if not isinstance(data, list): + raise ChangedFilesUnavailable( + "the changed-files API request failed or returned a non-list body") + if len(data) >= CHANGED_FILES_CAP: + raise ChangedFilesUnavailable( + f"the changed-files API returned {len(data)} entries, at or past its " + f"{CHANGED_FILES_CAP}-file ceiling, so the list may be truncated") + if isinstance(declared_count, int) and len(data) != declared_count: + raise ChangedFilesUnavailable( + f"the PR reports {declared_count} changed files but the changed-files API " + f"returned {len(data)} entries; the list does not match the PR") + entries: list[dict] = [] + for entry in data: + if not isinstance(entry, dict) or not entry.get("filename"): + raise ChangedFilesUnavailable( + "a changed-file entry was malformed (not an object, or no filename)") + entries.append(entry) + return entries + + def current_pr_target(self, pr: int) -> tuple[str | None, str | None]: + data = self.get(f"/repos/{self.repo}/pulls/{pr}") + return ( + (data or {}).get("head", {}).get("sha"), + (data or {}).get("base", {}).get("ref"), + ) + + def branch_is_protected(self, branch: str) -> bool | None: + encoded = urllib.parse.quote(branch, safe="") + data = self.get(f"/repos/{self.repo}/branches/{encoded}") + protected = (data or {}).get("protected") + return protected if isinstance(protected, bool) else None + + +# ── orchestration ─────────────────────────────────────────────────────────────────────── +class Validator: + def __init__(self, gh: GitHub, patterns, enforce: bool, soft_fail: bool, run_url: str, + exempt_label: str): + self.gh = gh + self.patterns = patterns + self.enforce = enforce + self.soft_fail = soft_fail + self.run_url = run_url + self.exempt_label = exempt_label + self.pr_number: int | None = None + self.validated_sha: str | None = None + self.validated_base_branch: str | None = None + + def _guard_supersession(self) -> bool: + now_sha, now_base = self.gh.current_pr_target(self.pr_number) + if now_sha and now_sha != self.validated_sha: + log(f"PR head moved {self.validated_sha} -> {now_sha}; a newer run supersedes " + "this one. Not writing a terminal status.") + return False + if now_base and now_base != self.validated_base_branch: + log(f"PR base moved {self.validated_base_branch} -> {now_base}; a newer run " + "supersedes this one. Not writing a terminal status.") + return False + return True + + def finish_pass(self, reason: str, status_desc: str) -> int: + summary("## telemetry-cost-contract: ✅ pass") + summary("") + summary(reason) + self.gh.delete_marker_comment(self.pr_number) + if self._guard_supersession(): + if not self.gh.publish_status(self.validated_sha, "success", status_desc, + self.run_url, critical=True): + return 1 + return 0 + + def finish_not_required(self, base_branch: str) -> int: + if not self._guard_supersession(): + return 0 + summary("## telemetry-cost-contract: not required") + summary("") + summary(f"PR targets the unprotected `{base_branch}` branch.") + self.gh.delete_marker_comment(self.pr_number) + return 0 + + def finish_fail(self, category: str, detail: str) -> int: + guidance = lib.failure_guidance(category) + outcome = lib.failure_outcome(category, self.enforce, self.soft_fail) + + body_lines = [ + lib.MARKER, "", + f"### Telemetry cost contract check — {outcome.verdict}", "", + guidance, + ] + if detail: + body_lines += ["", detail] + if outcome.advisory: + body_lines += ["", lib.advisory_note(category)] + body_lines += [ + "", "---", + (f"After updating the PR body, re-run this check from the " + f"[workflow run]({self.run_url}) or edit the PR to trigger a fresh run. A " + f"repository maintainer can waive the requirement by applying the " + f"`{self.exempt_label or 'cost-exempt'}` label."), + ] + + if not self._guard_supersession(): + return outcome.exit_code + + self.gh.upsert_marker_comment(self.pr_number, "\n".join(body_lines)) + + summary(f"## telemetry-cost-contract: {outcome.verdict}") + summary("") + summary(guidance) + if detail: + summary(detail) + + if not self.gh.publish_status(self.validated_sha, outcome.state, outcome.description, + self.run_url, critical=True): + return 1 + return outcome.exit_code + + def run(self, event: dict) -> int: + wr = event.get("workflow_run") or {} + head_sha = wr.get("head_sha") or "" + if not head_sha: + error("workflow_run.head_sha missing from event") + return 1 + + wr_event = wr.get("event") or "" + if wr_event != "pull_request": + error(f"Triggering workflow_run event was '{wr_event}', not 'pull_request'. The " + "signal workflow must run on pull_request; refusing to validate.") + return 1 + + self.pr_number = self._resolve_pr(event, head_sha) + if self.pr_number is None: + return 1 + + pr = self.gh.get(f"/repos/{self.gh.repo}/pulls/{self.pr_number}") + if not pr: + error(f"Could not fetch PR #{self.pr_number}") + return 1 + self.validated_sha = pr.get("head", {}).get("sha") + base_branch = pr.get("base", {}).get("ref") or "" + self.validated_base_branch = base_branch + body = pr.get("body") or "" + labels = [lbl.get("name") for lbl in (pr.get("labels") or [])] + + log(f"Validating PR #{self.pr_number} at head {self.validated_sha}") + if not base_branch: + error(f"PR #{self.pr_number} has no base branch; refusing to validate.") + return 1 + protected = self.gh.branch_is_protected(base_branch) + if protected is None: + error(f"Could not determine whether base branch '{base_branch}' is protected; " + "failing closed.") + return 1 + if not protected: + log(f"PR targets unprotected branch '{base_branch}' — contract not required.") + return self.finish_not_required(base_branch) + + self.gh.publish_status(self.validated_sha, "pending", + "Scanning diff for cost-sensitive call sites…", self.run_url) + + if self.exempt_label and self.exempt_label in labels: + log(f"PR carries the '{self.exempt_label}' label — exempt.") + return self.finish_pass( + f"PR carries the `{self.exempt_label}` label — the cost-contract requirement " + "is waived.", + f"Exempt via {self.exempt_label} label") + + try: + files = self.gh.changed_files(self.pr_number, pr.get("changed_files")) + except ChangedFilesUnavailable as exc: + error(f"Could not read the changed-file list for PR #{self.pr_number}: {exc}. " + "Failing closed.") + return self.finish_fail("changed_files_unavailable", f"Changed-file lookup: {exc}.") + + match = lib.first_match(files, self.patterns) + if match is None: + log(f"No cost-sensitive pattern matched across {len(files)} changed file(s) — pass.") + return self.finish_pass( + "No added line matched this repository's cost-sensitive detection patterns.", + "No new cost-sensitive telemetry call site detected") + + filename, pattern_text = match + log(f"Cost-sensitive pattern matched in {filename} (pattern: {pattern_text!r})") + if lib.has_cost_impact_section(body): + return self.finish_pass( + f"A new telemetry call site was detected (matched `{pattern_text}` in " + f"`{filename}`), and the PR body carries a `## Cost impact` section.", + "Cost impact section present") + + return self.finish_fail( + "missing_cost_impact", + f"Matched pattern `{pattern_text}` against an added line in `{filename}`.") + + def _resolve_pr(self, event: dict, head_sha: str) -> int | None: + """Exactly one open PR. Same-repo runs carry workflow_run.pull_requests; fork runs do + not, so fall back to the commit->PR association (GitHub-owned data either way).""" + wr = event.get("workflow_run") or {} + candidates = [pr.get("number") for pr in (wr.get("pull_requests") or []) if pr.get("number")] + if not candidates: + assoc = self.gh.get(f"/repos/{self.gh.repo}/commits/{head_sha}/pulls") or [] + candidates = [ + pr.get("number") for pr in assoc + if pr.get("state") == "open" + and (pr.get("base") or {}).get("repo", {}).get("full_name") == self.gh.repo + ] + + open_prs: list[int] = [] + for number in dict.fromkeys(candidates): + data = self.gh.get(f"/repos/{self.gh.repo}/pulls/{number}") + if data and data.get("state") == "open": + open_prs.append(number) + + if len(open_prs) != 1: + error(f"Expected exactly one open PR associated with {head_sha}, found " + f"{len(open_prs)} (event={wr.get('event')}). Refusing to publish an " + "ambiguous result.") + return None + return open_prs[0] + + +def main() -> int: + repo = os.environ.get("GH_REPO") + event_path = os.environ.get("GITHUB_EVENT_PATH") + + if not repo: + error("GH_REPO is required") + return 1 + if not event_path or not os.path.isfile(event_path): + error("GITHUB_EVENT_PATH is required and must point at the event payload") + return 1 + + try: + patterns = lib.parse_patterns(os.environ.get("PATTERNS", "")) + except ValueError as exc: + error(f"Invalid patterns input: {exc}. Provide one regex per line (blank lines and " + "'#' comments are skipped).") + return 1 + + with open(event_path, encoding="utf-8") as handle: + event = json.load(handle) + + enforce = os.environ.get("ENFORCE", "true") != "false" + soft_fail = lib.soft_fail_enabled(os.environ.get("SOFT_FAIL")) + run_url = os.environ.get("RUN_URL", "") + exempt_label = os.environ.get("EXEMPT_LABEL", "") + + validator = Validator(GitHub(repo), patterns, enforce, soft_fail, run_url, exempt_label) + return validator.run(event) + + +if __name__ == "__main__": + sys.exit(main()) From 782d465771e94173508a1018991ff4d8813d4733 Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 20 Nov 2026 08:19:29 -0800 Subject: [PATCH 2/6] test(workflow-pins): add telemetry-cost-contract to the docs-drift ROWS table --- .github/workflow-pins/tests/test_inputs_docs_drift.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflow-pins/tests/test_inputs_docs_drift.py b/.github/workflow-pins/tests/test_inputs_docs_drift.py index 1e507a44..6908fad6 100644 --- a/.github/workflow-pins/tests/test_inputs_docs_drift.py +++ b/.github/workflow-pins/tests/test_inputs_docs_drift.py @@ -235,6 +235,14 @@ sentinel="workflows_ref", expect_header_with=False, ), + # Same shape as linear-ticket: the header comment documents the two caller + # FILES in prose (see "Caller pattern:") and leaves the `with:` fences to + # the guide. + Row( + workflow="telemetry-cost-contract", + sentinel="workflows_ref", + expect_header_with=False, + ), Row( workflow="pr-area-label", sentinel="workflows_ref", From 2c3996c78eb17149d4af73db62ad9ffbd8b4b4c3 Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 20 Nov 2026 16:21:35 -0800 Subject: [PATCH 3/6] fix(telemetry-cost-contract): close unfilled-template bypass in has_cost_impact_section Cursor review (High, 6/6 reviewers) on #477: the heading regex's \s* let match.end() land mid-line, so trailing heading text ("## Cost impact: TBD", "(required)", closing "##") was never excluded from the content check. The content check also counted raw HTML-comment placeholder text and fenced-block examples as real content, so a PR template with an untouched `` placeholder passed the gate with zero actual cost analysis. Anchor the heading match to the full line, require a real ATX space (`[ \t]+`, matching CommonMark), and strip fenced blocks (before heading-boundary detection, so a quoted heading inside a fence cannot be mistaken for the section boundary) and HTML comments before checking for non-blank content. 6 new regression tests; all 37 tests pass. --- scripts/telemetry-cost-contract/lib.py | 22 ++++++++++++---- .../telemetry-cost-contract/tests/test_lib.py | 25 +++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/scripts/telemetry-cost-contract/lib.py b/scripts/telemetry-cost-contract/lib.py index 9445d51b..6b3433e4 100644 --- a/scripts/telemetry-cost-contract/lib.py +++ b/scripts/telemetry-cost-contract/lib.py @@ -34,8 +34,11 @@ # headers and '@@' hunk headers, which must never be mistaken for added content. _ADDED_LINE_RE = re.compile(r"^\+(?!\+\+)(.*)$", re.MULTILINE) -_COST_IMPACT_HEADING_RE = re.compile(r"^#{1,6}\s*cost impact\b", re.IGNORECASE | re.MULTILINE) -_ANY_HEADING_RE = re.compile(r"^#{1,6}\s", re.MULTILINE) +_COST_IMPACT_HEADING_RE = re.compile( + r"^#{1,6}[ \t]+cost impact\b.*$", re.IGNORECASE | re.MULTILINE) +_ANY_HEADING_RE = re.compile(r"^#{1,6}[ \t]", re.MULTILINE) +_HTML_COMMENT_RE = re.compile(r"", re.DOTALL) +_FENCED_BLOCK_RE = re.compile(r"^```.*?^```", re.DOTALL | re.MULTILINE) def parse_patterns(raw: str) -> list[re.Pattern]: @@ -101,18 +104,27 @@ def first_match(files: list[dict], patterns: list[re.Pattern]) -> tuple[str, str def has_cost_impact_section(body: str) -> bool: """True iff ``body`` has a ``## Cost impact`` (any heading depth, case-insensitive) - heading followed by at least one non-blank line before the next heading or EOF. + heading followed by at least one non-blank, non-placeholder line before the next heading + or EOF. A bare heading with nothing under it (copy-pasted template, never filled in) does not count — TDD section 7.1 requires actual content (priced unit, baseline, protected signal), - not just the section's existence. + not just the section's existence. The heading match consumes the whole heading line (so + trailing text like ``: TBD`` or ``(required)`` is never mistaken for section content), and + the section body has HTML comments and fenced code blocks stripped before the + content check, so an untouched PR-template placeholder (```` or an + example inside a ``` fence) does not count either. """ match = _COST_IMPACT_HEADING_RE.search(body or "") if not match: return False - rest = (body or "")[match.end():] + # Strip fenced blocks before looking for the next heading: a heading-shaped line quoted + # inside a fence (e.g. an example of the template itself) must not be mistaken for the + # section boundary. + rest = _FENCED_BLOCK_RE.sub("", (body or "")[match.end():]) next_heading = _ANY_HEADING_RE.search(rest) section_body = rest[:next_heading.start()] if next_heading else rest + section_body = _HTML_COMMENT_RE.sub("", section_body) return any(line.strip() for line in section_body.splitlines()) diff --git a/scripts/telemetry-cost-contract/tests/test_lib.py b/scripts/telemetry-cost-contract/tests/test_lib.py index c410e75f..536e94a9 100644 --- a/scripts/telemetry-cost-contract/tests/test_lib.py +++ b/scripts/telemetry-cost-contract/tests/test_lib.py @@ -116,6 +116,31 @@ def test_empty_body(self): self.assertFalse(lib.has_cost_impact_section("")) self.assertFalse(lib.has_cost_impact_section(None)) + def test_unfilled_html_comment_placeholder_does_not_count(self): + body = "## Cost impact\n\n\n\n## Testing\n" + self.assertFalse(lib.has_cost_impact_section(body)) + + def test_multiline_html_comment_placeholder_does_not_count(self): + body = "## Cost impact\n\n\n\n## Testing\n" + self.assertFalse(lib.has_cost_impact_section(body)) + + def test_text_on_heading_line_does_not_count_as_content(self): + self.assertFalse(lib.has_cost_impact_section("## Cost impact: TBD\n\n## Testing\n")) + self.assertFalse(lib.has_cost_impact_section("## Cost impact (required)\n\n## Testing\n")) + self.assertFalse(lib.has_cost_impact_section("## Cost impact ##\n\n## Testing\n")) + + def test_example_inside_fenced_block_does_not_count(self): + body = "## Cost impact\n\n```\n## Cost impact\nBills $X/month.\n```\n\n## Testing\n" + self.assertFalse(lib.has_cost_impact_section(body)) + + def test_real_content_after_placeholder_comment_still_counts(self): + body = ("## Cost impact\n\n\n" + "Bills per event; forecast 1k/day.\n\n## Testing\n") + self.assertTrue(lib.has_cost_impact_section(body)) + + def test_heading_requires_space_not_just_any_whitespace(self): + self.assertFalse(lib.has_cost_impact_section("##\ncost impact\nsome text\n")) + class SoftFailEnabledTests(unittest.TestCase): def test_true_string(self): From 807edc782df67772068323e8eac451e62de8a530 Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 20 Nov 2026 20:21:18 -0800 Subject: [PATCH 4/6] fix(telemetry-cost-contract): close MULTILINE-anchor, truncated-diff and log-forging gaps - parse_patterns: compile caller patterns with re.MULTILINE so an anchored pattern (^\s*statsd\.) matches any added line in a file's joined blob, not only the first. - lib.unscannable_files + validate.py: a non-removed file with added lines but no renderable patch (GitHub's too-large-to-render omission) now fails closed as changed_files_unavailable instead of silently passing as 'no match'. - validate.py: log the PR-controlled filename via repr() so embedded newlines/control characters in a fork PR's filename can't forge ::stop-commands::/::add-mask:: lines in the privileged workflow_run job log (linear-ticket/validate.py avoids the same class of issue by logging only counts). Addresses the 3 Medium cursor-review findings on this PR at 782d465/2c3996c (diff-scan, truncation, log-injection). 7 new regression tests; 44/44 pass. --- scripts/telemetry-cost-contract/lib.py | 35 ++++++++++++++-- .../telemetry-cost-contract/tests/test_lib.py | 41 +++++++++++++++++++ scripts/telemetry-cost-contract/validate.py | 12 +++++- 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/scripts/telemetry-cost-contract/lib.py b/scripts/telemetry-cost-contract/lib.py index 6b3433e4..1a3d6507 100644 --- a/scripts/telemetry-cost-contract/lib.py +++ b/scripts/telemetry-cost-contract/lib.py @@ -58,7 +58,12 @@ def parse_patterns(raw: str) -> list[re.Pattern]: if not text or text.startswith("#"): continue try: - patterns.append(re.compile(text)) + # MULTILINE: a matched file's added lines are joined into one blob (see + # first_match) so one pattern.search() covers the whole file, but an anchored + # caller pattern like `^\s*statsd\.` must still match at the start of ANY added + # line, not only the first — without this flag `^`/`$` bind to the blob's start + # and end only, so every added line past the first is invisible to anchors. + patterns.append(re.compile(text, re.MULTILINE)) except re.error as exc: raise ValueError(f"invalid pattern {text!r}: {exc}") from exc if not patterns: @@ -84,9 +89,9 @@ def first_match(files: list[dict], patterns: list[re.Pattern]) -> tuple[str, str A file with no ``patch`` key (binary, or a diff too large for GitHub to render — GitHub omits `patch` past its ~1MB-per-file render ceiling and does not flag the omission) - contributes no lines and is silently skipped, matching the diff-added-lines-only scope: - this check is a code-review nudge, not a security boundary, so a binary/huge-diff file - simply cannot trip it either way. + contributes no lines and is silently skipped here. Call ``unscannable_files`` first and + fail closed on anything it returns — a large/padded file missing its patch must not get a + free pass just because this function cannot see its content. """ for entry in files or []: patch = entry.get("patch") if isinstance(entry, dict) else None @@ -102,6 +107,28 @@ def first_match(files: list[dict], patterns: list[re.Pattern]) -> tuple[str, str return None +def unscannable_files(files: list[dict]) -> list[str]: + """Filenames whose diff content this check could not see at all. + + GitHub omits ``patch`` both for genuinely binary files and for text diffs too large to + render, and the two look identical on the wire. A removed file or one with zero added + lines can only ever shrink cost, so it is excluded; anything else missing its patch with + ``additions > 0`` must be treated as unscannable rather than silently passed as "no + match" — a padded or oversized new telemetry module is exactly the case this gate exists + to catch, and ``first_match`` alone cannot distinguish it from a harmless binary asset. + """ + names = [] + for entry in files or []: + if not isinstance(entry, dict) or entry.get("patch"): + continue + if entry.get("status") == "removed": + continue + if (entry.get("additions") or 0) <= 0: + continue + names.append(entry.get("filename") or "") + return names + + def has_cost_impact_section(body: str) -> bool: """True iff ``body`` has a ``## Cost impact`` (any heading depth, case-insensitive) heading followed by at least one non-blank, non-placeholder line before the next heading diff --git a/scripts/telemetry-cost-contract/tests/test_lib.py b/scripts/telemetry-cost-contract/tests/test_lib.py index 536e94a9..34ffb432 100644 --- a/scripts/telemetry-cost-contract/tests/test_lib.py +++ b/scripts/telemetry-cost-contract/tests/test_lib.py @@ -88,6 +88,47 @@ def test_second_pattern_matches_second_file(self): def test_empty_files_list(self): self.assertIsNone(lib.first_match([], self.patterns)) + def test_anchored_pattern_matches_a_later_added_line(self): + # Regression: added lines are joined into one blob with "\n". Without re.MULTILINE on + # the caller's compiled pattern, `^` only binds to the blob's start, so an anchored + # pattern would see line one and miss every added line after it. + anchored = lib.parse_patterns(r"^\s*statsd\.Gauge\(") + files = [{"filename": "a.go", "patch": ( + "+fmt.Println(1)\n" + "+ statsd.Gauge(\"x\", 1)\n" + )}] + match = lib.first_match(files, anchored) + self.assertEqual(match, ("a.go", r"^\s*statsd\.Gauge\(")) + + +class UnscannableFilesTests(unittest.TestCase): + def test_missing_patch_with_additions_is_unscannable(self): + files = [{"filename": "huge.go", "additions": 500, "status": "modified"}] + self.assertEqual(lib.unscannable_files(files), ["huge.go"]) + + def test_missing_patch_but_removed_is_not_unscannable(self): + files = [{"filename": "gone.go", "additions": 0, "status": "removed"}] + self.assertEqual(lib.unscannable_files(files), []) + + def test_missing_patch_with_zero_additions_is_not_unscannable(self): + files = [{"filename": "binary.png", "additions": 0, "status": "modified"}] + self.assertEqual(lib.unscannable_files(files), []) + + def test_file_with_patch_is_not_unscannable(self): + files = [{"filename": "a.go", "patch": "+x\n", "additions": 1, "status": "modified"}] + self.assertEqual(lib.unscannable_files(files), []) + + def test_multiple_unscannable_files_in_order(self): + files = [ + {"filename": "a.go", "additions": 10, "status": "modified"}, + {"filename": "b.go", "patch": "+x\n", "additions": 1, "status": "modified"}, + {"filename": "c.go", "additions": 20, "status": "added"}, + ] + self.assertEqual(lib.unscannable_files(files), ["a.go", "c.go"]) + + def test_empty_files_list(self): + self.assertEqual(lib.unscannable_files([]), []) + class CostImpactSectionTests(unittest.TestCase): def test_present_with_content(self): diff --git a/scripts/telemetry-cost-contract/validate.py b/scripts/telemetry-cost-contract/validate.py index 8b66e4d5..fc09e5e2 100644 --- a/scripts/telemetry-cost-contract/validate.py +++ b/scripts/telemetry-cost-contract/validate.py @@ -329,6 +329,16 @@ def run(self, event: dict) -> int: "Failing closed.") return self.finish_fail("changed_files_unavailable", f"Changed-file lookup: {exc}.") + unscannable = lib.unscannable_files(files) + if unscannable: + error(f"{len(unscannable)} changed file(s) have added content but no renderable " + "diff patch (GitHub omits `patch` past its render ceiling); cannot rule " + "out a cost-sensitive call site. Failing closed.") + return self.finish_fail( + "changed_files_unavailable", + f"Unscannable file(s): {', '.join(f'`{n}`' for n in unscannable[:10])}" + + ("…" if len(unscannable) > 10 else "") + ".") + match = lib.first_match(files, self.patterns) if match is None: log(f"No cost-sensitive pattern matched across {len(files)} changed file(s) — pass.") @@ -337,7 +347,7 @@ def run(self, event: dict) -> int: "No new cost-sensitive telemetry call site detected") filename, pattern_text = match - log(f"Cost-sensitive pattern matched in {filename} (pattern: {pattern_text!r})") + log(f"Cost-sensitive pattern matched in {filename!r} (pattern: {pattern_text!r})") if lib.has_cost_impact_section(body): return self.finish_pass( f"A new telemetry call site was detected (matched `{pattern_text}` in " From 5bd3dcc69546486f689a15db3dff82ba0e02abb4 Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 20 Nov 2026 20:22:58 -0800 Subject: [PATCH 5/6] fix(telemetry-cost-contract): match each added line individually, not a joined blob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 807edc7's re.MULTILINE fix only solved the anchor half of the lib.py:98/103 finding. Per the review's own note, `\s`, `.` (with DOTALL) and negated character classes match a literal newline regardless of MULTILINE, so joining added lines into one blob still let a pattern span two unrelated added lines (or hunks) and match content nobody wrote on either line. Search each line on its own instead, as the finding suggested — a single line has no embedded newline for an anchor to miss or a cross-line pattern to exploit. Added the missing cross-line regression test; 45/45 pass. --- scripts/telemetry-cost-contract/lib.py | 21 ++++++++++--------- .../telemetry-cost-contract/tests/test_lib.py | 17 ++++++++++++--- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/scripts/telemetry-cost-contract/lib.py b/scripts/telemetry-cost-contract/lib.py index 1a3d6507..20b2402e 100644 --- a/scripts/telemetry-cost-contract/lib.py +++ b/scripts/telemetry-cost-contract/lib.py @@ -58,12 +58,7 @@ def parse_patterns(raw: str) -> list[re.Pattern]: if not text or text.startswith("#"): continue try: - # MULTILINE: a matched file's added lines are joined into one blob (see - # first_match) so one pattern.search() covers the whole file, but an anchored - # caller pattern like `^\s*statsd\.` must still match at the start of ANY added - # line, not only the first — without this flag `^`/`$` bind to the blob's start - # and end only, so every added line past the first is invisible to anchors. - patterns.append(re.compile(text, re.MULTILINE)) + patterns.append(re.compile(text)) except re.error as exc: raise ValueError(f"invalid pattern {text!r}: {exc}") from exc if not patterns: @@ -87,6 +82,15 @@ def first_match(files: list[dict], patterns: list[re.Pattern]) -> tuple[str, str """The first (filename, pattern-text) whose pattern matches an added line in some file's patch, in file order then pattern order; None if nothing matches. + Each added line is matched on its own (``any(p.search(line) for line in lines)``), never + joined into one multi-line blob: joining would make an anchored pattern like + `^\\s*statsd\\.` see only the file's first added line (`^`/`$` bind to the whole blob, not + each line, without `re.MULTILINE`), AND would let a pattern containing `\\s`, a negated + character class, or `.` with `re.DOTALL` span two unrelated added lines — or even two + different hunks — and match content nobody actually wrote. Per-line matching has neither + failure mode: a single line has no embedded newline for an anchor to miss or a + cross-line pattern to exploit. + A file with no ``patch`` key (binary, or a diff too large for GitHub to render — GitHub omits `patch` past its ~1MB-per-file render ceiling and does not flag the omission) contributes no lines and is silently skipped here. Call ``unscannable_files`` first and @@ -98,11 +102,8 @@ def first_match(files: list[dict], patterns: list[re.Pattern]) -> tuple[str, str if not patch: continue lines = added_lines(patch) - if not lines: - continue - blob = "\n".join(lines) for pattern in patterns: - if pattern.search(blob): + if any(pattern.search(line) for line in lines): return entry.get("filename") or "", pattern.pattern return None diff --git a/scripts/telemetry-cost-contract/tests/test_lib.py b/scripts/telemetry-cost-contract/tests/test_lib.py index 34ffb432..a20ee940 100644 --- a/scripts/telemetry-cost-contract/tests/test_lib.py +++ b/scripts/telemetry-cost-contract/tests/test_lib.py @@ -89,9 +89,9 @@ def test_empty_files_list(self): self.assertIsNone(lib.first_match([], self.patterns)) def test_anchored_pattern_matches_a_later_added_line(self): - # Regression: added lines are joined into one blob with "\n". Without re.MULTILINE on - # the caller's compiled pattern, `^` only binds to the blob's start, so an anchored - # pattern would see line one and miss every added line after it. + # Regression: each added line is matched on its own. If lines were joined into one + # blob, `^` would only bind to the blob's start (without re.MULTILINE), so an + # anchored pattern would see line one and miss every added line after it. anchored = lib.parse_patterns(r"^\s*statsd\.Gauge\(") files = [{"filename": "a.go", "patch": ( "+fmt.Println(1)\n" @@ -100,6 +100,17 @@ def test_anchored_pattern_matches_a_later_added_line(self): match = lib.first_match(files, anchored) self.assertEqual(match, ("a.go", r"^\s*statsd\.Gauge\(")) + def test_pattern_does_not_false_match_across_two_added_lines(self): + # Regression: joining added lines into one blob would let a pattern containing `\s` + # (which matches a literal newline) span two unrelated added lines and match content + # nobody actually wrote on one line. + spanning = lib.parse_patterns(r"foo\s+bar") + files = [{"filename": "a.go", "patch": ( + "+some_call(foo)\n" + "+bar_other_call()\n" + )}] + self.assertIsNone(lib.first_match(files, spanning)) + class UnscannableFilesTests(unittest.TestCase): def test_missing_patch_with_additions_is_unscannable(self): From 19bc86b857634950c159a465a7a5dcc6a615a2fb Mon Sep 17 00:00:00 2001 From: bymyself Date: Fri, 20 Nov 2026 20:24:28 -0800 Subject: [PATCH 6/6] fix(telemetry-cost-contract): escape filename for Markdown, not just the log Closes the second half of the log-injection finding: a backtick or newline in a PR-controlled filename broke out of the single-backtick code span in the marker comment and job summary, not just the privileged job log (which 807edc7 already fixed via repr()). md_code() escapes backticks and strips newlines before wrapping in backticks; both call sites that render a filename in Markdown now use it. --- scripts/telemetry-cost-contract/validate.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/telemetry-cost-contract/validate.py b/scripts/telemetry-cost-contract/validate.py index fc09e5e2..892bd926 100644 --- a/scripts/telemetry-cost-contract/validate.py +++ b/scripts/telemetry-cost-contract/validate.py @@ -62,6 +62,13 @@ def warning(msg: str) -> None: print(f"::warning::{msg}", file=sys.stderr, flush=True) +def md_code(text: str) -> str: + """Render ``text`` as a Markdown inline code span, safe even if it contains a backtick or + newline — either would otherwise let PR-controlled data (a filename) break out of the + span into arbitrary Markdown, links or @-mentions in the marker comment / job summary.""" + return "`" + text.replace("`", "'").replace("\n", " ").replace("\r", " ") + "`" + + def summary(msg: str) -> None: path = os.environ.get("GITHUB_STEP_SUMMARY") if not path: @@ -336,7 +343,7 @@ def run(self, event: dict) -> int: "out a cost-sensitive call site. Failing closed.") return self.finish_fail( "changed_files_unavailable", - f"Unscannable file(s): {', '.join(f'`{n}`' for n in unscannable[:10])}" + f"Unscannable file(s): {', '.join(md_code(n) for n in unscannable[:10])}" + ("…" if len(unscannable) > 10 else "") + ".") match = lib.first_match(files, self.patterns) @@ -351,12 +358,12 @@ def run(self, event: dict) -> int: if lib.has_cost_impact_section(body): return self.finish_pass( f"A new telemetry call site was detected (matched `{pattern_text}` in " - f"`{filename}`), and the PR body carries a `## Cost impact` section.", + f"{md_code(filename)}), and the PR body carries a `## Cost impact` section.", "Cost impact section present") return self.finish_fail( "missing_cost_impact", - f"Matched pattern `{pattern_text}` against an added line in `{filename}`.") + f"Matched pattern `{pattern_text}` against an added line in {md_code(filename)}.") def _resolve_pr(self, event: dict, head_sha: str) -> int | None: """Exactly one open PR. Same-repo runs carry workflow_run.pull_requests; fork runs do