Feat stop all tests on error

.mocharc.js

@@ -3,6 +3,6 @@
 module.exports = {
     recursive: true,
     extension: [".js", ".ts"],
-    ignore: ["./test/browser-env/**", "**/report/**", "**/basic-report/**"],
+    ignore: ["./test/browser-env/**", "**/report/**", "**/basic-report/**", "**/exit-on-error/fixtures/**"],
     require: ["./test/setup", "./test/assert-ext", "./test/ts-node", "tsconfig-paths/register"],
 };

docs/features.md

@@ -29,6 +29,9 @@ To prevent incidental fails, Testplane retries a failed test before marking it a
 
 :warning: Testplane reruns tests in a new browser session to exclude situations when the browser environment is the cause of the failure.
 
+### Stops on infrastructure errors
+See [Stop a run on matching errors](exit-on-error.md) for configuring named error filters and run-wide failure thresholds.
+
 ### Executes separate tests
 Sometimes you only need to run specific tests, not all the tests in a set. Testplane makes this possible. You can specify the path to the test file
 ```

src/config/defaults.js

@@ -84,6 +84,10 @@ module.exports = {
     testsPerWorker: Infinity,
     retry: 0,
     shouldRetry: null,
+    exitOnError: {
+        enabled: false,
+        filters: [],
+    },
     mochaOpts: {
         slow: 10000,
         timeout: 60000,

src/config/options.js

@@ -127,6 +127,39 @@ const rootSection = section(
 
         plugins: options.anyObject(),
 
+        exitOnError: section({
+            enabled: options.boolean("exitOnError.enabled"),
+            filters: option({
+                defaultValue: defaults.exitOnError.filters,
+                validate: value => {
+                    if (!Array.isArray(value)) {
+                        throw new Error('"exitOnError.filters" must be an array of named rules');
+                    }
+
+                    const names = new Set();
+                    for (const [index, rule] of value.entries()) {
+                        const path = `exitOnError.filters[${index}]`;
+                        if (!_.isPlainObject(rule)) {
+                            throw new Error(`"${path}" must be an object`);
+                        }
+                        if (typeof rule.name !== "string" || !rule.name.trim()) {
+                            throw new Error(`"${path}.name" must be a non-empty string`);
+                        }
+                        if (names.has(rule.name)) {
+                            throw new Error(`"${path}.name" must be unique: duplicate rule name "${rule.name}"`);
+                        }
+                        names.add(rule.name);
+                        if (typeof rule.filter !== "function" && !_.isRegExp(rule.filter)) {
+                            throw new Error(`"${path}.filter" must be a function or RegExp`);
+                        }
+                        if (!Number.isSafeInteger(rule.limit) || rule.limit <= 0) {
+                            throw new Error(`"${path}.limit" must be a positive safe integer`);
+                        }
+                    }
+                },
+            }),
+        }),
+
         lastFailed: section({
             only: options.boolean("lastFailed.only"),
             input: option({

src/config/types.ts

@@ -374,6 +374,18 @@ export interface BrowserDownloadMirrors {
 
 export type BrowserDownloadMirrorsInput = Partial<BrowserDownloadMirrors>;
 
+export interface ExitOnErrorRule {
+    /** Unique, non-empty rule name used in error messages. */
+    name: string;
+    filter: RegExp | ((errorText: string) => boolean);
+    limit: number;
+}
+
+export interface ExitOnErrorConfig {
+    enabled: boolean;
+    filters: ExitOnErrorRule[];
+}
+
 export interface CommonConfig {
     configPath?: string;
     automationProtocol: "webdriver";
@@ -545,6 +557,7 @@ export type ConfigInputData = Partial<PartialCommonConfig> & {
     beforeAll?: HookType;
     afterAll?: HookType;
     profiler?: Partial<ProfilerConfig>;
+    exitOnError?: Partial<ExitOnErrorConfig>;
 };
 
 export type ConfigInput = ConfigInputData | (() => ConfigInputData) | (() => Promise<ConfigInputData>);
@@ -558,6 +571,7 @@ export interface ConfigParsed extends CommonConfig {
     beforeAll?: HookType;
     afterAll?: HookType;
     profiler: ProfilerConfig;
+    exitOnError: ExitOnErrorConfig;
 }
 
 export interface RuntimeConfig {

src/index.ts

@@ -35,7 +35,14 @@ export type {
 } from "./types";
 export type { Config, ProfilerConfig, ProfilerLevel } from "./config";
 export { TimeTravelMode, SelectivityMode } from "./config";
-export type { ConfigInput, AssertViewOpts, BrowserDownloadMirrors, BrowserDownloadMirrorsInput } from "./config/types";
+export type {
+    ConfigInput,
+    AssertViewOpts,
+    BrowserDownloadMirrors,
+    BrowserDownloadMirrorsInput,
+    ExitOnErrorConfig,
+    ExitOnErrorRule,
+} from "./config/types";
 export type { ProfilerResultV1 } from "./profiler/schema";
 export type {
     TestCollection,

src/runner/browser-runner.ts

@@ -12,6 +12,7 @@ import { Workers } from "./index";
 import type { Test } from "../types";
 import { noopProfilerRuntime } from "../profiler/runtime/noop";
 import type { ProfilerRuntimeLike } from "../profiler/runtime/types";
+import type { CheckTestError } from "./exit-on-error";
 
 export interface BrowserRunner {
     on(event: InterceptedEvent, handler: (test: Test) => void): this;
@@ -34,6 +35,7 @@ export class BrowserRunner extends CancelableEmitter {
         browserPool: BrowserPool,
         workers: Workers,
         profiler: ProfilerRuntimeLike = noopProfilerRuntime,
+        private checkTestError?: CheckTestError,
     ) {
         super();
         this._browserId = browserId;
@@ -72,7 +74,7 @@ export class BrowserRunner extends CancelableEmitter {
             version: test.browserVersion,
             pool: this.browserPool,
         });
-        const runner = TestRunner.create(test, this.config, browserAgent, this.profiler);
+        const runner = TestRunner.create(test, this.config, browserAgent, this.profiler, this.checkTestError);
 
         if (this.cancelError && typeof runner.cancel === "function") {
             runner.cancel(this.cancelError);

src/runner/exit-on-error.ts

@@ -0,0 +1,63 @@
+import type { ExitOnErrorConfig } from "../config/types";
+import type { Test } from "../types";
+
+export type CheckTestError = (test: Test) => Error | undefined;
+
+/** Shared by all browsers for a single run. */
+export class ExitOnError {
+    private stopped = false;
+    private readonly rules;
+
+    constructor(config: ExitOnErrorConfig) {
+        this.rules = config.enabled
+            ? config.filters.map(({ name, filter, limit }) => ({
+                  name,
+                  // Do not mutate the user's RegExp (including its lastIndex).
+                  filter: typeof filter === "function" ? filter : new RegExp(filter.source, filter.flags),
+                  limit,
+                  count: 0,
+              }))
+            : [];
+    }
+
+    check(test: Test): Error | undefined {
+        if (this.stopped || !test.err) {
+            return undefined;
+        }
+
+        const errorText = test.err.message;
+
+        for (const rule of this.rules) {
+            let matches: boolean;
+            try {
+                if (typeof rule.filter === "function") {
+                    matches = rule.filter(errorText);
+                    if (typeof matches !== "boolean") {
+                        // Also consume rejections from an accidentally async filter.
+                        void Promise.resolve(matches).catch(() => {});
+                        throw new TypeError("filter must synchronously return a boolean");
+                    }
+                } else {
+                    rule.filter.lastIndex = 0;
+                    matches = rule.filter.test(errorText);
+                }
+            } catch (cause) {
+                this.stopped = true;
+                return new Error(`exitOnError filter "${rule.name}" failed`, { cause });
+            }
+
+            if (matches) {
+                rule.count++;
+                if (rule.count >= rule.limit) {
+                    this.stopped = true;
+                    return new Error(
+                        `exitOnError rule "${rule.name}" reached its limit of ${rule.limit} matching errors: ${errorText}`,
+                        { cause: test.err },
+                    );
+                }
+            }
+        }
+
+        return undefined;
+    }
+}

src/runner/index.ts

@@ -25,6 +25,7 @@ import { Test } from "../types";
 import { SelectivityRunner } from "../browser/cdp/selectivity/runner";
 import { noopProfilerRuntime } from "../profiler/runtime/noop";
 import type { ProfilerRuntimeLike } from "../profiler/runtime/types";
+import { ExitOnError } from "./exit-on-error";
 
 interface WorkerMethods {
     runTest: typeof runTest;
@@ -59,11 +60,13 @@ export class MainRunner extends RunnableEmitter {
     protected workers: Workers | null;
     protected profiler: ProfilerRuntimeLike;
     private _cancelError: Error | null;
+    private exitOnError: ExitOnError;
 
     constructor(config: Config, interceptors: Interceptor[], profiler: ProfilerRuntimeLike = noopProfilerRuntime) {
         super();
 
         this.config = config;
+        this.exitOnError = new ExitOnError(config.exitOnError);
         this.interceptors = interceptors;
         this.browserPool = null;
 
@@ -167,7 +170,24 @@ export class MainRunner extends RunnableEmitter {
     }
 
     protected _createBrowserRunner(browserId: string): BrowserRunner {
-        const runner = BrowserRunner.create(browserId, this.config, this.browserPool, this.workers, this.profiler);
+        const runner = BrowserRunner.create(
+            browserId,
+            this.config,
+            this.browserPool,
+            this.workers,
+            this.profiler,
+            (test: Test) => {
+                if (this.cancelled) {
+                    return undefined;
+                }
+
+                const error = this.exitOnError.check(test);
+                if (error) {
+                    this.emit(MasterEvents.ERROR, error);
+                }
+                return error;
+            },
+        );
 
         if (this._cancelError) {
             runner.cancel(this._cancelError);

src/runner/test-runner/index.ts

@@ -4,6 +4,7 @@ import { Config } from "../../config";
 import type { Test } from "../../types";
 import type BrowserAgent from "../browser-agent";
 import type { ProfilerRuntimeLike } from "../../profiler/runtime/types";
+import type { CheckTestError } from "../exit-on-error";
 
 export type TestRunner = SkippedTestRunner | InsistantTestRunner;
 
@@ -12,8 +13,9 @@ export const create = function (
     config: Config,
     browserAgent: BrowserAgent,
     profiler?: ProfilerRuntimeLike,
+    checkTestError?: CheckTestError,
 ): TestRunner {
     return test.pending || test.disabled
         ? SkippedTestRunner.create(test)
-        : InsistantTestRunner.create(test, config, browserAgent, profiler);
+        : InsistantTestRunner.create(test, config, browserAgent, profiler, checkTestError);
 };

src/runner/test-runner/insistant-test-runner.js

@@ -10,14 +10,15 @@ const { passthroughEvent } = require("../../events/utils");
 const { NoRefImageError } = require("../../browser/commands/assert-view/errors/no-ref-image-error");
 
 module.exports = class InsistantTestRunner extends RunnableEmitter {
-    constructor(test, config, browserAgent, profiler) {
+    constructor(test, config, browserAgent, profiler, checkTestError) {
         super();
 
         this._test = test;
         this._config = config;
         this._browserConfig = config.forBrowser(browserAgent.browserId);
         this._browserAgent = browserAgent;
         this._profiler = profiler;
+        this._checkTestError = checkTestError;
 
         this._retriesPerformed = 0;
         this._cancelled = false;
@@ -34,6 +35,12 @@ module.exports = class InsistantTestRunner extends RunnableEmitter {
         const runner = RegularTestRunner.create(this._test, browserAgent, this._profiler).on(
             MasterEvents.TEST_FAIL,
             data => {
+                if (!this._cancelled) {
+                    const error = this._checkTestError?.(data);
+                    if (error) {
+                        this.cancel(error);
+                    }
+                }
                 if (this._shouldRetry(data)) {
                     this.emit(MasterEvents.RETRY, _.extend(data, { retriesLeft: this._retriesLeft }));
                     retry = true;

test/integration/exit-on-error/exit-on-error.test.ts

@@ -0,0 +1,54 @@
+import { strict as assert } from "node:assert";
+import fs from "node:fs/promises";
+import http from "node:http";
+import os from "node:os";
+import path from "node:path";
+import type { AddressInfo } from "node:net";
+import execa from "execa";
+
+describe("exitOnError integration", function () {
+    this.timeout(60000);
+
+    it("stops the CLI on a real HTTP 429 without test retries and cancels the remaining tests", async () => {
+        const directory = await fs.mkdtemp(path.join(os.tmpdir(), "testplane-exit-on-error-"));
+        let requests = 0;
+        const server = http.createServer((_req, res) => {
+            requests++;
+            res.writeHead(429, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ value: { error: "session not created", message: "Too Many Requests" } }));
+        });
+
+        try {
+            await new Promise<void>((resolve, reject) => {
+                server.once("error", reject);
+                server.listen(0, "127.0.0.1", resolve);
+            });
+            const { port } = server.address() as AddressInfo;
+            const config = path.join(directory, "testplane.config.js");
+            const tests = path.join(directory, "tests.js");
+            await fs.cp(path.join(__dirname, "fixtures"), directory, { recursive: true });
+
+            const result = await execa(process.execPath, [path.resolve("bin/testplane"), tests, "--config", config], {
+                cwd: directory,
+                env: { ["testplane_grid_url"]: `http://127.0.0.1:${port}/` },
+                reject: false,
+                timeout: 45000,
+            });
+
+            assert.equal(result.exitCode, 1, result.all || result.stderr);
+            assert.equal(result.timedOut, false);
+            assert.match(result.stderr, /exitOnError rule "quota" reached its limit of 1 matching errors/);
+            assert.equal((result.stderr.match(/Terminating on critical error:/g) || []).length, 1);
+            const events = result.stdout.match(/EXIT_ON_ERROR_EVENTS:(.*)/);
+            assert.ok(events, result.stdout);
+            assert.deepEqual(JSON.parse(events[1]), { retries: 0, failures: 4, errors: 1 });
+            // Four HTTP attempts precede the first test failure. The pool can start
+            // one queued request before that failure reaches the runner and cancels it.
+            assert.ok(requests >= 4 && requests <= 5, `unexpected number of HTTP requests: ${requests}`);
+        } finally {
+            server.closeAllConnections();
+            await new Promise<void>(resolve => server.close(() => resolve()));
+            await fs.rm(directory, { recursive: true, force: true });
+        }
+    });
+});

test/integration/exit-on-error/fixtures/events.js

@@ -0,0 +1,9 @@
+module.exports = testplane => {
+    const counts = { retries: 0, failures: 0, errors: 0 };
+    testplane.on(testplane.events.RETRY, () => counts.retries++);
+    testplane.on(testplane.events.TEST_FAIL, () => counts.failures++);
+    testplane.on(testplane.events.ERROR, () => counts.errors++);
+    testplane.on(testplane.events.RUNNER_END, () => {
+        console.log("EXIT_ON_ERROR_EVENTS:" + JSON.stringify(counts));
+    });
+};

test/integration/exit-on-error/fixtures/testplane.config.js

@@ -0,0 +1,11 @@
+module.exports = {
+    retry: 5,
+    sessionsPerBrowser: 1,
+    system: { parallelLimit: 1 },
+    plugins: { [require.resolve("./events")]: {} },
+    exitOnError: {
+        enabled: true,
+        filters: [{ name: "quota", filter: /Too Many Requests/i, limit: 1 }],
+    },
+    browsers: { chrome: { desiredCapabilities: { browserName: "chrome" } } },
+};

test/integration/exit-on-error/fixtures/tests.js

@@ -0,0 +1,5 @@
+describe("quota", () => {
+    for (let i = 0; i < 4; i++) {
+        it("test " + i, () => {});
+    }
+});