From 9a947be9ff214394458222749189fe830a497b28 Mon Sep 17 00:00:00 2001 From: Martin Linzmayer Date: Wed, 18 Nov 2026 00:28:58 +0100 Subject: [PATCH 1/5] feat(ci): add tally benchmarks --- .gitattributes | 1 + .github/workflows/benchmarks.yml | 80 ++++++++++++++++++++++++++ benchmarks/CaptureExceptionBench.php | 85 ++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 benchmarks/CaptureExceptionBench.php diff --git a/.gitattributes b/.gitattributes index fb056e7a4..fae99b923 100644 --- a/.gitattributes +++ b/.gitattributes @@ -31,3 +31,4 @@ /mago.toml export-ignore /decision-model-linter export-ignore /warden.toml export-ignore +/benchmarks export-ignore diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 000000000..b761bfb09 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,80 @@ +name: Benchmarks + +on: + pull_request: + push: + branches: + - master + - release/** + +permissions: + contents: read + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + tally: + name: Tally + # Tally compares instruction counts with the numbers committed next to each benchmark. + # The counts depend on the runner image and the PHP build, so neither floats with `latest`. + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The token can comment on pull requests, so the code under test must not find it in .git/config. + persist-credentials: false + + - name: Setup PHP + uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # v2 + with: + php-version: '8.5' + extensions: posix + coverage: none + + # Tally needs PHP 8.2+, so it cannot be in composer.json, which has to install on PHP 7.2. + # It is required without the dev dependencies, because every installed package ends up in + # the modules list of an event and changes the counts. + - name: Install dependencies and Tally + run: composer require phpstan/tally:1.0.1 --update-no-dev --no-progress --no-interaction --prefer-dist + + # Exit code 1 means a benchmark is slower or could not be measured. That is reported in the + # job summary and the pull request comment, and does not fail the job. Exit code 2 means + # Tally itself could not run, which still fails, otherwise a broken setup would go unnoticed. + - name: Run benchmarks + run: | + vendor/bin/tally run benchmarks || [ $? -eq 1 ] + cp "$GITHUB_STEP_SUMMARY" "$RUNNER_TEMP/tally-report.md" + + # Pull requests from forks only get a read-only token, they have the report in the job summary. + # Dependabot's token is read-only too, so a failed comment does not fail the job either. + - name: Comment on the pull request + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + marker='' + report="$RUNNER_TEMP/tally-report.md" + comment="$RUNNER_TEMP/tally-comment.md" + { echo "$marker"; cat "$report"; } > "$comment" + + comment_id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" --paginate \ + --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1) + + if [ -n "$comment_id" ]; then + # Keep the existing comment up to date, also once the benchmarks are unchanged again. + gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$comment_id" -F body=@"$comment" --silent + elif grep -q '^> \[!' "$report"; then + # Tally starts a section with an alert for slower, faster, new, and not measured + # benchmarks. Without one, every benchmark is unchanged and there is nothing to say. + gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" -F body=@"$comment" --silent + fi diff --git a/benchmarks/CaptureExceptionBench.php b/benchmarks/CaptureExceptionBench.php new file mode 100644 index 000000000..006d81845 --- /dev/null +++ b/benchmarks/CaptureExceptionBench.php @@ -0,0 +1,85 @@ +httpClient = new CountingHttpClient(); + + init([ + 'dsn' => 'https://public@example.com/1', + 'http_client' => $this->httpClient, + ]); + + $this->exception = $this->createException(self::STACK_DEPTH); + } + + #[Warmup(1)] + public function benchCaptureException(): void + { + captureException($this->exception); + } + + /** + * Without this, an event dropped before the transport would be measured + * as a very fast capture. + */ + #[After] + public function assertEnvelopeWasSent(): void + { + if ($this->httpClient->requests === 0) { + throw new \LogicException('captureException() did not reach the HTTP client.'); + } + } + + private function createException(int $depth): \Throwable + { + if ($depth > 0) { + return $this->createException($depth - 1); + } + + return new \RuntimeException('Benchmark exception'); + } +} + +final class CountingHttpClient implements HttpClientInterface +{ + public int $requests = 0; + + public function sendRequest(Request $request, Options $options): Response + { + ++$this->requests; + + return new Response(200, [], ''); + } +} From 017144fdfa8b796af8f91bfe398c9faeabab1cb3 Mon Sep 17 00:00:00 2001 From: Martin Linzmayer Date: Wed, 18 Nov 2026 00:37:26 +0100 Subject: [PATCH 2/5] wip --- .github/workflows/benchmarks.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b761bfb09..8338c007b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -65,7 +65,9 @@ jobs: marker='' report="$RUNNER_TEMP/tally-report.md" comment="$RUNNER_TEMP/tally-comment.md" - { echo "$marker"; cat "$report"; } > "$comment" + # The comment is the headline and the table of all benchmarks, the last table of the + # report. The full report with the patch of new expected numbers is in the job summary. + { echo "$marker"; awk 'NR == 1 { print; print "" } /^
/ { table = 1 } table && /^\|/' "$report"; } > "$comment" comment_id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" --paginate \ --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1) From a8622ce90daef0c6a5c82eacf5ffa5c4626b27a5 Mon Sep 17 00:00:00 2001 From: Martin Linzmayer Date: Wed, 18 Nov 2026 00:49:00 +0100 Subject: [PATCH 3/5] wip --- .github/workflows/benchmarks.yml | 8 ++-- benchmarks/CaptureExceptionBench.php | 1 + benchmarks/Name.php | 17 +++++++++ benchmarks/pr-comment.php | 56 ++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 benchmarks/Name.php create mode 100644 benchmarks/pr-comment.php diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 8338c007b..649cc8181 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -52,6 +52,8 @@ jobs: run: | vendor/bin/tally run benchmarks || [ $? -eq 1 ] cp "$GITHUB_STEP_SUMMARY" "$RUNNER_TEMP/tally-report.md" + # Built here, because the next step has a token that code of the pull request must not run with. + php benchmarks/pr-comment.php "$RUNNER_TEMP/tally-report.md" > "$RUNNER_TEMP/tally-table.md" # Pull requests from forks only get a read-only token, they have the report in the job summary. # Dependabot's token is read-only too, so a failed comment does not fail the job either. @@ -65,9 +67,9 @@ jobs: marker='' report="$RUNNER_TEMP/tally-report.md" comment="$RUNNER_TEMP/tally-comment.md" - # The comment is the headline and the table of all benchmarks, the last table of the - # report. The full report with the patch of new expected numbers is in the job summary. - { echo "$marker"; awk 'NR == 1 { print; print "" } /^
/ { table = 1 } table && /^\|/' "$report"; } > "$comment" + # The comment is the headline and the table of all benchmarks. The full report with + # the patch of new expected numbers is in the job summary. + { echo "$marker"; cat "$RUNNER_TEMP/tally-table.md"; } > "$comment" comment_id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" --paginate \ --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1) diff --git a/benchmarks/CaptureExceptionBench.php b/benchmarks/CaptureExceptionBench.php index 006d81845..63052573c 100644 --- a/benchmarks/CaptureExceptionBench.php +++ b/benchmarks/CaptureExceptionBench.php @@ -44,6 +44,7 @@ public function setUp(): void $this->exception = $this->createException(self::STACK_DEPTH); } + #[Name('captureException with 20 recursive frames')] #[Warmup(1)] public function benchCaptureException(): void { diff --git a/benchmarks/Name.php b/benchmarks/Name.php new file mode 100644 index 000000000..8ee2fc2fb --- /dev/null +++ b/benchmarks/Name.php @@ -0,0 +1,17 @@ + + */ + +namespace Sentry\Benchmarks; + +require_once __DIR__ . '/../vendor/autoload.php'; +require_once __DIR__ . '/Name.php'; + +foreach (glob(__DIR__ . '/*Bench.php') ?: [] as $file) { + require_once $file; +} + +$lines = file($argv[1], \FILE_IGNORE_NEW_LINES); +if ($lines === false || $lines === []) { + fwrite(\STDERR, \sprintf("Cannot read the report from %s.\n", $argv[1])); + + exit(1); +} + +echo $lines[0], "\n\n"; + +// The table of all benchmarks is the one inside
, the last of the report. +$inTable = false; +foreach ($lines as $line) { + if (str_starts_with($line, '
')) { + $inTable = true; + } + + if (!$inTable || !str_starts_with($line, '|')) { + continue; + } + + // The status is the last column. + $line = preg_replace('/[^|]*\|$/', '', $line); + + echo preg_replace_callback( + '/^\| `([^`@]+)::(\w+)(@[^`]*)?` \|/', + static function (array $match): string { + $attributes = (new \ReflectionMethod($match[1], $match[2]))->getAttributes(Name::class); + if ($attributes === []) { + return $match[0]; + } + + return \sprintf('| %s%s |', $attributes[0]->newInstance()->name, $match[3] ?? ''); + }, + $line + ), "\n"; +} From 948b7db6a29ec78a88c051f570638e1ca8e368d6 Mon Sep 17 00:00:00 2001 From: Martin Linzmayer Date: Wed, 18 Nov 2026 11:11:12 +0100 Subject: [PATCH 4/5] wip --- benchmarks/pr-comment.php | 67 ++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/benchmarks/pr-comment.php b/benchmarks/pr-comment.php index f82245a1c..70677a4a8 100644 --- a/benchmarks/pr-comment.php +++ b/benchmarks/pr-comment.php @@ -4,7 +4,7 @@ /* * Turns Tally's Markdown report into the body of the pull request comment: the headline and - * the table of all benchmarks, with the names from #[Name] instead of the case IDs. + * a table with one row per benchmark, with the names from #[Name] instead of the case IDs. * * Usage: php benchmarks/pr-comment.php */ @@ -25,9 +25,27 @@ exit(1); } -echo $lines[0], "\n\n"; +/** + * The name from #[Name] of a case like `Class::method` or `Class::method@data set`. + */ +function displayName(string $cell): string +{ + if (preg_match('/^`([^`@]+)::(\w+)(@.*)?`$/', $cell, $match) !== 1 || !method_exists($match[1], $match[2])) { + return $cell; + } + + $attributes = (new \ReflectionMethod($match[1], $match[2]))->getAttributes(Name::class); + if ($attributes === []) { + return $cell; + } -// The table of all benchmarks is the one inside
, the last of the report. + return $attributes[0]->newInstance()->name . ($match[3] ?? ''); +} + +// The table of all benchmarks is the one inside
, the last of the report. It has a +// row per metric, with the case only in the first one: +// | Benchmark | Metric | Expected | Measured | Change | Allowed | Status | +$benchmarks = []; $inTable = false; foreach ($lines as $line) { if (str_starts_with($line, '
')) { @@ -38,19 +56,32 @@ continue; } - // The status is the last column. - $line = preg_replace('/[^|]*\|$/', '', $line); - - echo preg_replace_callback( - '/^\| `([^`@]+)::(\w+)(@[^`]*)?` \|/', - static function (array $match): string { - $attributes = (new \ReflectionMethod($match[1], $match[2]))->getAttributes(Name::class); - if ($attributes === []) { - return $match[0]; - } - - return \sprintf('| %s%s |', $attributes[0]->newInstance()->name, $match[3] ?? ''); - }, - $line - ), "\n"; + [$benchmark, $metric, $expected, $measured, $change, $allowed] = array_map('trim', explode('|', trim($line, '|'))); + if ($benchmark === 'Benchmark' || str_starts_with($benchmark, '-')) { + // the header and the separator below it + continue; + } + + if ($benchmark !== '') { + $benchmarks[] = ['name' => displayName($benchmark), 'metrics' => []]; + } + + $benchmarks[array_key_last($benchmarks)]['metrics'][$metric] = [ + $change === '-' ? $measured : \sprintf('%s (%s)', $measured, $change), + $allowed === '-' ? $expected : \sprintf('%s %s', $expected, $allowed), + ]; +} + +echo $lines[0], "\n\n"; +echo "| Benchmark | Instructions | Instructions expected | Cycles | Cycles expected | Memory | Memory expected |\n"; +echo "|---|--:|--:|--:|--:|--:|--:|\n"; + +foreach ($benchmarks as $benchmark) { + $cells = [$benchmark['name']]; + foreach (['instructions', 'cycles', 'memory'] as $metric) { + // a benchmark that could not be measured has no metrics + array_push($cells, ...($benchmark['metrics'][$metric] ?? ['-', '-'])); + } + + echo '| ', implode(' | ', $cells), " |\n"; } From be62a30edb4b4a4142d071985afc6e15edc84172 Mon Sep 17 00:00:00 2001 From: Martin Linzmayer Date: Wed, 18 Nov 2026 13:19:19 +0100 Subject: [PATCH 5/5] add label to commit benchmarks --- .github/workflows/benchmarks-accept.yml | 83 +++++++++++++++++++++++++ .github/workflows/benchmarks.yml | 7 ++- 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/benchmarks-accept.yml diff --git a/.github/workflows/benchmarks-accept.yml b/.github/workflows/benchmarks-accept.yml new file mode 100644 index 000000000..451be2d67 --- /dev/null +++ b/.github/workflows/benchmarks-accept.yml @@ -0,0 +1,83 @@ +name: Accept benchmarks + +# Adding the `accept-benchmarks` label to a pull request measures its benchmarks again and +# commits the measured numbers to the pull request's branch as the expected ones. +# Only people with triage access or more can add a label. +on: + pull_request: + types: + - labeled + +permissions: + contents: read + +jobs: + # Runs the code of the pull request and its dependencies, so it only gets a read-only token + # and hands the changed expectations to the next job as a patch. + measure: + name: Measure + # The token of a pull request from a fork cannot push, its numbers are accepted locally. + if: github.event.label.name == 'accept-benchmarks' && github.event.pull_request.head.repo.full_name == github.repository + # Same runner image, PHP, and dependencies as benchmarks.yml, because the counts depend on them. + runs-on: ubuntu-24.04 + outputs: + patch: ${{ steps.accept.outputs.patch }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup PHP + uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # v2 + with: + php-version: '8.5' + extensions: posix + coverage: none + + - name: Install dependencies and Tally + run: composer require phpstan/tally:1.0.1 --update-no-dev --no-progress --no-interaction --prefer-dist + + - name: Measure and accept + id: accept + run: | + vendor/bin/tally run benchmarks --output="$RUNNER_TEMP/tally-results" || [ $? -eq 1 ] + vendor/bin/tally accept "$RUNNER_TEMP/tally-results" benchmarks + echo "patch=$(git diff -- benchmarks | base64 --wrap=0)" >> "$GITHUB_OUTPUT" + + # Has the token that can push, and runs nothing from the pull request. + commit: + name: Commit + needs: measure + if: always() && needs.measure.result != 'skipped' + runs-on: ubuntu-24.04 + permissions: + contents: write + pull-requests: write + steps: + # Also when measuring failed, so that adding the label again starts another attempt. + - name: Remove the label + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh api --method DELETE "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels/accept-benchmarks" --silent + + - name: Checkout + if: needs.measure.outputs.patch != '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + + # The push is rejected when the branch got new commits since the label was added. + # A push with this token does not start workflows, so CI only runs again with the next push. + - name: Commit the expected numbers + if: needs.measure.outputs.patch != '' + env: + PATCH: ${{ needs.measure.outputs.patch }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: | + echo "$PATCH" | base64 --decode | git apply --include='benchmarks/*' + git add benchmarks + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit --message='chore(benchmarks): accept new expected numbers' + git push origin "HEAD:refs/heads/$HEAD_REF" diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 649cc8181..1946d16cf 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -69,7 +69,12 @@ jobs: comment="$RUNNER_TEMP/tally-comment.md" # The comment is the headline and the table of all benchmarks. The full report with # the patch of new expected numbers is in the job summary. - { echo "$marker"; cat "$RUNNER_TEMP/tally-table.md"; } > "$comment" + { + echo "$marker" + cat "$RUNNER_TEMP/tally-table.md" + echo + echo 'Add the `accept-benchmarks` label to commit the measured numbers as the expected ones.' + } > "$comment" comment_id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" --paginate \ --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1)