Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 137 additions & 0 deletions .github/workflows/mutation-testing.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
name: Mutation Testing

# Says which of the tests a pull request adds could go again without anything going unnoticed. It never
# fails a pull request: the findings are warnings on the tests, a job summary and a comment.
# See scripts/mutation-testing/run.php.
#
# It only runs on pull requests that have the "mutation-testing" label. Every test it reports still has to
# be read next to the test it names, which is worth the time of whoever wrote the tests, not of every
# reviewer.

on:
pull_request:
types: [opened, synchronize, reopened, labeled]
paths:
- 'tests/**'
- 'scripts/mutation-testing/**'
- '.github/workflows/mutation-testing.yml'

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
new-tests:
name: New tests
if: ${{ contains(github.event.pull_request.labels.*.name, 'mutation-testing') }}
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
summary: ${{ steps.summary.outputs.summary }}

steps:
# The pull request's merge commit and its first parent, the base branch, are all the comparison needs.
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 2

- name: Setup PHP
uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # v2
with:
php-version: '8.5'
coverage: xdebug
extensions: excimer
tools: infection:0.35.10

- name: Determine Composer cache directory
id: composer-cache
run: echo "directory=$(composer config cache-dir)" >> "$GITHUB_OUTPUT"

- name: Cache Composer dependencies
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5
with:
path: ${{ steps.composer-cache.outputs.directory }}
key: ${{ runner.os }}-8.5-composer-highest-${{ hashFiles('**/composer.lock') }}
restore-keys: ${{ runner.os }}-8.5-highest-composer-

# The same dependencies as the "Tests" job of ci.yml with the highest versions.
- name: Remove unused dependencies
run: composer remove carthage-software/mago phpstan/phpstan friendsofphp/php-cs-fixer --dev --no-interaction --no-update

- name: Remove OpenTelemetry dependencies
run: composer remove open-telemetry/api open-telemetry/context open-telemetry/exporter-otlp open-telemetry/gen-otlp-protobuf open-telemetry/sem-conv open-telemetry/sdk --dev --no-interaction --no-update

- name: Set phpunit/phpunit version constraint
run: composer require phpunit/phpunit:'^9.6.34' --dev --no-interaction --no-update

- name: Install dependencies
run: composer update --no-progress --no-interaction --prefer-dist

# Tests that are not judged within the budget are reported as not measured. A crash or a timeout
# here never fails the check.
- name: Run the new tests against mutants
id: run
continue-on-error: true
timeout-minutes: 25
run: php scripts/mutation-testing/run.php --base=HEAD^1 --out="$RUNNER_TEMP/mutation-testing" --workers="$(nproc)" --budget=900

- name: Write the job summary
id: summary
if: always()
env:
RUN_OUTCOME: ${{ steps.run.outcome }}
run: |
summary="$RUNNER_TEMP/mutation-testing/summary.md"
if [[ "$RUN_OUTCOME" != success || ! -f "$summary" ]]; then
mkdir -p "$(dirname "$summary")"
printf '## New tests under mutation testing\n\nThe run did not finish, so the new tests were not measured. The job log has the details.\n' > "$summary"
fi
tee -a "$GITHUB_STEP_SUMMARY" < "$summary"
{ echo 'summary<<MUTATION_TESTING_EOF'; cat "$summary"; echo 'MUTATION_TESTING_EOF'; } >> "$GITHUB_OUTPUT"

- name: Upload the report
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: mutation-testing
path: |
${{ runner.temp }}/mutation-testing/summary.md
${{ runner.temp }}/mutation-testing/report.json
if-no-files-found: ignore

comment:
name: Comment the tests that could go
needs: new-tests
# Pull requests from forks get a read-only token, so they only get the warnings and the job summary.
if: ${{ !cancelled() && needs.new-tests.result != 'skipped' && github.event.pull_request.head.repo.full_name == github.repository }}
runs-on: ubuntu-latest
# The job above runs the pull request's code with a read-only token. This one runs none of it.
permissions:
pull-requests: write

steps:
# Only starts a comment when a test is reported, but keeps an existing one up to date, so a test
# that was removed or improved no longer shows.
- name: Comment on the pull request
env:
GH_TOKEN: ${{ github.token }}
PR: ${{ github.event.pull_request.number }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SUMMARY: ${{ needs.new-tests.outputs.summary }}
run: |
[[ -n "$SUMMARY" ]] || exit 0
marker='<!-- mutation-testing -->'
body="$marker
$SUMMARY

[Run]($RUN_URL)"
id=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" --paginate --jq ".[] | select(.body | startswith(\"$marker\")) | .id" | head -1)
if [[ -n "$id" ]]; then
gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$id" -f body="$body" > /dev/null
elif grep -q '^### ' <<< "$SUMMARY"; then
gh pr comment "$PR" --repo "$GITHUB_REPOSITORY" --body "$body"
fi
12 changes: 12 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@

- Add tests with every behavior change. This is a library repo with broad
compatibility and regression coverage.
- Every new test must catch a bug that no existing test catches. Read the
tests that already cover the code first, and name that bug before adding the
test. If you cannot name one, do not add the test.
- Add a data-provider row only for input that takes a different branch than
the existing rows, not for another value of the same kind.
- Do not add tests for plain getters and setters. Options are covered by their
case in `OptionsTest::optionsDataProvider()`.
- Every SDK option/default must have a valid case in
`OptionsTest::optionsDataProvider()`. Keep the exhaustive option-coverage and
default-validation tests passing when adding or changing options.
Expand Down Expand Up @@ -107,3 +114,8 @@
jobs for FrankenPHP and RoadRunner.
- `.github/workflows/static-analysis.yaml` runs PHP-CS-Fixer, PHPStan, and
Psalm on single recent PHP versions rather than across the full test matrix.
- `.github/workflows/mutation-testing.yml` only runs on pull requests with the
`mutation-testing` label. It comments which new tests notice nothing that
another test does not notice too. Add the label when a pull request adds
tests, then remove each reported test or say in the pull request why it
stays. It never fails a pull request.
212 changes: 212 additions & 0 deletions scripts/mutation-testing/MutationTestingListener.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,212 @@
<?php

declare(strict_types=1);

use PHPUnit\Framework\AssertionFailedError;
use PHPUnit\Framework\ErrorTestCase;
use PHPUnit\Framework\Test;
use PHPUnit\Framework\TestCase;
use PHPUnit\Framework\TestListener;
use PHPUnit\Framework\TestSuite;
use PHPUnit\Framework\Warning;
use PHPUnit\Framework\WarningTestCase;
use PHPUnit\Runner\Filter\Factory;
use PHPUnit\Runner\PhptTestCase;

require_once __DIR__ . '/SelectedTestsFilter.php';

/**
* The part of scripts/mutation-testing/run.php that lives inside a PHPUnit run. It does nothing unless
* one of these environment variables is set:
*
* MT_LIST Write every test of the suite to this file, one JSON object per line, and exit without
* running a test.
* MT_ONLY Run only the tests whose ids are listed in this file, one per line.
* MT_LOG Append the outcome of every test to this file. Every line is flushed at once, so when
* the process dies in a test the log still says which one was running:
* B the run started
* S <id> a test started
* R <id> <status> <s> a test ended: P passed, F failed, E error, O anything else
* X PHPUnit could not build a test, e.g. a data provider threw
* Z the run ended
*
* A test id is "Class::method with data set #0", the id code coverage uses as well, or the path of a
* PHPT file relative to the working directory.
*
* PHPUnit 9 deprecates TestListener in favour of its hooks, but a hook only gets the name of a test.
* Listing the tests and selecting some of them needs the suite, which only a listener is given.
*/
final class MutationTestingListener implements TestListener
{
/**
* @var resource|null
*/
private $log;

/**
* @var int
*/
private $depth = 0;

/**
* @var string
*/
private $status = 'P';

public static function id(Test $test): string
{
if ($test instanceof PhptTestCase) {
$root = getcwd() . '/';
$name = $test->getName();

return strpos($name, $root) === 0 ? substr($name, strlen($root)) : $name;
}

if ($test instanceof TestCase) {
return str_replace(["\r\n", "\n", "\r", "\t"], ['\\n', '\\n', '\\n', ' '], get_class($test) . '::' . $test->getName());
}

return get_class($test);
}

public function startTestSuite(TestSuite $suite): void
{
if ($this->depth++ > 0) {
return;
}

$list = getenv('MT_LIST');

if (is_string($list) && $list !== '') {
$this->writeList($suite, $list);

exit(0);
}

$only = getenv('MT_ONLY');

if (is_string($only) && $only !== '') {
$factory = new Factory();
$factory->addFilter(new ReflectionClass(SelectedTestsFilter::class), $only);
$suite->injectFilter($factory);
}

$log = getenv('MT_LOG');

if (is_string($log) && $log !== '') {
$this->log = fopen($log, 'a');
$this->write('B');
}
}

public function endTestSuite(TestSuite $suite): void
{
if (--$this->depth === 0) {
$this->write('Z');
}
}

public function startTest(Test $test): void
{
$this->status = 'P';

if (!$test instanceof ErrorTestCase && !$test instanceof WarningTestCase) {
$this->write("S\t" . self::id($test));
}
}

public function endTest(Test $test, float $time): void
{
if ($test instanceof ErrorTestCase) {
$this->write('X');
} elseif (!$test instanceof WarningTestCase) {
$this->write(sprintf("R\t%s\t%s\t%.4F", self::id($test), $this->status, $time));
}
}

public function addError(Test $test, Throwable $t, float $time): void
{
$this->status = 'E';
}

public function addWarning(Test $test, Warning $e, float $time): void
{
// A warning does not fail a run, so it does not count as noticing a mutant.
}

public function addFailure(Test $test, AssertionFailedError $e, float $time): void
{
if ($this->status !== 'E') {
$this->status = 'F';
}
}

public function addIncompleteTest(Test $test, Throwable $t, float $time): void
{
$this->other();
}

public function addRiskyTest(Test $test, Throwable $t, float $time): void
{
$this->other();
}

public function addSkippedTest(Test $test, Throwable $t, float $time): void
{
$this->other();
}

private function other(): void
{
if ($this->status === 'P') {
$this->status = 'O';
}
}

private function write(string $line): void
{
if ($this->log !== null) {
fwrite($this->log, $line . "\n");
fflush($this->log);
}
}

private function writeList(TestSuite $suite, string $path): void
{
$root = getcwd() . '/';
$lines = '';

foreach (new RecursiveIteratorIterator($suite->getIterator()) as $test) {
if ($test instanceof PhptTestCase) {
$lines .= json_encode(['id' => self::id($test), 'file' => self::id($test), 'line' => 1]) . "\n";

continue;
}

if (!$test instanceof TestCase || $test instanceof ErrorTestCase || $test instanceof WarningTestCase) {
continue;
}

$class = new ReflectionClass($test);
$method = $class->getMethod($test->getName(false));
$entry = [
'id' => self::id($test),
'method' => $class->getName() . '::' . $method->getName(),
'file' => substr((string) $class->getFileName(), strlen($root)),
// An inherited test is reported on the class that inherits it.
'line' => $method->getFileName() === $class->getFileName() ? $method->getStartLine() : $class->getStartLine(),
];

if ($test->usesDataProvider()) {
$entry['dataSet'] = $test->dataName();
// Tells an inserted data set from the ones that only moved down by one. The paths
// differ between the two checkouts that are compared.
$entry['dataHash'] = md5(str_replace($root, '', print_r($test->getProvidedData(), true)));
}

$lines .= json_encode($entry, \JSON_INVALID_UTF8_SUBSTITUTE) . "\n";
}

file_put_contents($path, $lines);
}
}
Loading
Loading