diff --git a/.gitattributes b/.gitattributes index fb056e7a4..fae99b923 100644 --- a/.gitattributes +++ b/.gitattributes @@ -31,3 +31,4 @@ /mago.toml export-ignore /decision-model-linter export-ignore /warden.toml export-ignore +/benchmarks export-ignore diff --git a/.github/workflows/benchmarks-accept.yml b/.github/workflows/benchmarks-accept.yml new file mode 100644 index 000000000..451be2d67 --- /dev/null +++ b/.github/workflows/benchmarks-accept.yml @@ -0,0 +1,83 @@ +name: Accept benchmarks + +# Adding the `accept-benchmarks` label to a pull request measures its benchmarks again and +# commits the measured numbers to the pull request's branch as the expected ones. +# Only people with triage access or more can add a label. +on: + pull_request: + types: + - labeled + +permissions: + contents: read + +jobs: + # Runs the code of the pull request and its dependencies, so it only gets a read-only token + # and hands the changed expectations to the next job as a patch. + measure: + name: Measure + # The token of a pull request from a fork cannot push, its numbers are accepted locally. + if: github.event.label.name == 'accept-benchmarks' && github.event.pull_request.head.repo.full_name == github.repository + # Same runner image, PHP, and dependencies as benchmarks.yml, because the counts depend on them. + runs-on: ubuntu-24.04 + outputs: + patch: ${{ steps.accept.outputs.patch }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup PHP + uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # v2 + with: + php-version: '8.5' + extensions: posix + coverage: none + + - name: Install dependencies and Tally + run: composer require phpstan/tally:1.0.1 --update-no-dev --no-progress --no-interaction --prefer-dist + + - name: Measure and accept + id: accept + run: | + vendor/bin/tally run benchmarks --output="$RUNNER_TEMP/tally-results" || [ $? -eq 1 ] + vendor/bin/tally accept "$RUNNER_TEMP/tally-results" benchmarks + echo "patch=$(git diff -- benchmarks | base64 --wrap=0)" >> "$GITHUB_OUTPUT" + + # Has the token that can push, and runs nothing from the pull request. + commit: + name: Commit + needs: measure + if: always() && needs.measure.result != 'skipped' + runs-on: ubuntu-24.04 + permissions: + contents: write + pull-requests: write + steps: + # Also when measuring failed, so that adding the label again starts another attempt. + - name: Remove the label + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh api --method DELETE "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/labels/accept-benchmarks" --silent + + - name: Checkout + if: needs.measure.outputs.patch != '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + + # The push is rejected when the branch got new commits since the label was added. + # A push with this token does not start workflows, so CI only runs again with the next push. + - name: Commit the expected numbers + if: needs.measure.outputs.patch != '' + env: + PATCH: ${{ needs.measure.outputs.patch }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: | + echo "$PATCH" | base64 --decode | git apply --include='benchmarks/*' + git add benchmarks + git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \ + commit --message='chore(benchmarks): accept new expected numbers' + git push origin "HEAD:refs/heads/$HEAD_REF" diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 000000000..1946d16cf --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,89 @@ +name: Benchmarks + +on: + pull_request: + push: + branches: + - master + - release/** + +permissions: + contents: read + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + tally: + name: Tally + # Tally compares instruction counts with the numbers committed next to each benchmark. + # The counts depend on the runner image and the PHP build, so neither floats with `latest`. + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The token can comment on pull requests, so the code under test must not find it in .git/config. + persist-credentials: false + + - name: Setup PHP + uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # v2 + with: + php-version: '8.5' + extensions: posix + coverage: none + + # Tally needs PHP 8.2+, so it cannot be in composer.json, which has to install on PHP 7.2. + # It is required without the dev dependencies, because every installed package ends up in + # the modules list of an event and changes the counts. + - name: Install dependencies and Tally + run: composer require phpstan/tally:1.0.1 --update-no-dev --no-progress --no-interaction --prefer-dist + + # Exit code 1 means a benchmark is slower or could not be measured. That is reported in the + # job summary and the pull request comment, and does not fail the job. Exit code 2 means + # Tally itself could not run, which still fails, otherwise a broken setup would go unnoticed. + - name: Run benchmarks + run: | + vendor/bin/tally run benchmarks || [ $? -eq 1 ] + cp "$GITHUB_STEP_SUMMARY" "$RUNNER_TEMP/tally-report.md" + # Built here, because the next step has a token that code of the pull request must not run with. + php benchmarks/pr-comment.php "$RUNNER_TEMP/tally-report.md" > "$RUNNER_TEMP/tally-table.md" + + # Pull requests from forks only get a read-only token, they have the report in the job summary. + # Dependabot's token is read-only too, so a failed comment does not fail the job either. + - name: Comment on the pull request + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + marker='' + report="$RUNNER_TEMP/tally-report.md" + comment="$RUNNER_TEMP/tally-comment.md" + # The comment is the headline and the table of all benchmarks. The full report with + # the patch of new expected numbers is in the job summary. + { + echo "$marker" + cat "$RUNNER_TEMP/tally-table.md" + echo + echo 'Add the `accept-benchmarks` label to commit the measured numbers as the expected ones.' + } > "$comment" + + comment_id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" --paginate \ + --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -n 1) + + if [ -n "$comment_id" ]; then + # Keep the existing comment up to date, also once the benchmarks are unchanged again. + gh api --method PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$comment_id" -F body=@"$comment" --silent + elif grep -q '^> \[!' "$report"; then + # Tally starts a section with an alert for slower, faster, new, and not measured + # benchmarks. Without one, every benchmark is unchanged and there is nothing to say. + gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" -F body=@"$comment" --silent + fi diff --git a/benchmarks/CaptureExceptionBench.php b/benchmarks/CaptureExceptionBench.php new file mode 100644 index 000000000..63052573c --- /dev/null +++ b/benchmarks/CaptureExceptionBench.php @@ -0,0 +1,86 @@ +httpClient = new CountingHttpClient(); + + init([ + 'dsn' => 'https://public@example.com/1', + 'http_client' => $this->httpClient, + ]); + + $this->exception = $this->createException(self::STACK_DEPTH); + } + + #[Name('captureException with 20 recursive frames')] + #[Warmup(1)] + public function benchCaptureException(): void + { + captureException($this->exception); + } + + /** + * Without this, an event dropped before the transport would be measured + * as a very fast capture. + */ + #[After] + public function assertEnvelopeWasSent(): void + { + if ($this->httpClient->requests === 0) { + throw new \LogicException('captureException() did not reach the HTTP client.'); + } + } + + private function createException(int $depth): \Throwable + { + if ($depth > 0) { + return $this->createException($depth - 1); + } + + return new \RuntimeException('Benchmark exception'); + } +} + +final class CountingHttpClient implements HttpClientInterface +{ + public int $requests = 0; + + public function sendRequest(Request $request, Options $options): Response + { + ++$this->requests; + + return new Response(200, [], ''); + } +} diff --git a/benchmarks/Name.php b/benchmarks/Name.php new file mode 100644 index 000000000..8ee2fc2fb --- /dev/null +++ b/benchmarks/Name.php @@ -0,0 +1,17 @@ + + */ + +namespace Sentry\Benchmarks; + +require_once __DIR__ . '/../vendor/autoload.php'; +require_once __DIR__ . '/Name.php'; + +foreach (glob(__DIR__ . '/*Bench.php') ?: [] as $file) { + require_once $file; +} + +$lines = file($argv[1], \FILE_IGNORE_NEW_LINES); +if ($lines === false || $lines === []) { + fwrite(\STDERR, \sprintf("Cannot read the report from %s.\n", $argv[1])); + + exit(1); +} + +/** + * The name from #[Name] of a case like `Class::method` or `Class::method@data set`. + */ +function displayName(string $cell): string +{ + if (preg_match('/^`([^`@]+)::(\w+)(@.*)?`$/', $cell, $match) !== 1 || !method_exists($match[1], $match[2])) { + return $cell; + } + + $attributes = (new \ReflectionMethod($match[1], $match[2]))->getAttributes(Name::class); + if ($attributes === []) { + return $cell; + } + + return $attributes[0]->newInstance()->name . ($match[3] ?? ''); +} + +// The table of all benchmarks is the one inside
, the last of the report. It has a +// row per metric, with the case only in the first one: +// | Benchmark | Metric | Expected | Measured | Change | Allowed | Status | +$benchmarks = []; +$inTable = false; +foreach ($lines as $line) { + if (str_starts_with($line, '
')) { + $inTable = true; + } + + if (!$inTable || !str_starts_with($line, '|')) { + continue; + } + + [$benchmark, $metric, $expected, $measured, $change, $allowed] = array_map('trim', explode('|', trim($line, '|'))); + if ($benchmark === 'Benchmark' || str_starts_with($benchmark, '-')) { + // the header and the separator below it + continue; + } + + if ($benchmark !== '') { + $benchmarks[] = ['name' => displayName($benchmark), 'metrics' => []]; + } + + $benchmarks[array_key_last($benchmarks)]['metrics'][$metric] = [ + $change === '-' ? $measured : \sprintf('%s (%s)', $measured, $change), + $allowed === '-' ? $expected : \sprintf('%s %s', $expected, $allowed), + ]; +} + +echo $lines[0], "\n\n"; +echo "| Benchmark | Instructions | Instructions expected | Cycles | Cycles expected | Memory | Memory expected |\n"; +echo "|---|--:|--:|--:|--:|--:|--:|\n"; + +foreach ($benchmarks as $benchmark) { + $cells = [$benchmark['name']]; + foreach (['instructions', 'cycles', 'memory'] as $metric) { + // a benchmark that could not be measured has no metrics + array_push($cells, ...($benchmark['metrics'][$metric] ?? ['-', '-'])); + } + + echo '| ', implode(' | ', $cells), " |\n"; +}