diff --git a/packages/brain/src/createErrorFactory.ts b/packages/brain/src/createErrorFactory.ts index 918d064f..eaf250e0 100644 --- a/packages/brain/src/createErrorFactory.ts +++ b/packages/brain/src/createErrorFactory.ts @@ -1,7 +1,7 @@ export function createErrorFactory(errorName: string) { return class CustomError extends Error { - constructor(message: string) { - super(message); + constructor(message: string, options?: ErrorOptions) { + super(message, options); this.name = errorName; // Set the error name provided to the factory Error.captureStackTrace(this, CustomError); // Capture the stack trace and omit the constructor frame } diff --git a/packages/brain/src/modules/apiClients/error.ts b/packages/brain/src/modules/apiClients/error.ts index fed2abb5..9cb33825 100644 --- a/packages/brain/src/modules/apiClients/error.ts +++ b/packages/brain/src/modules/apiClients/error.ts @@ -10,3 +10,9 @@ export class HttpApiError extends ApiError { super(message); } } + +/** True when a request got an HTTP response, i.e. the server is reachable even if it refused the request. */ +export function receivedHttpResponse(error: unknown): boolean { + for (let e = error; e instanceof Error; e = e.cause) if (e instanceof HttpApiError) return true; + return false; +} diff --git a/packages/brain/src/modules/apiClients/validator/error.ts b/packages/brain/src/modules/apiClients/validator/error.ts index 2a9773f7..6308bae9 100644 --- a/packages/brain/src/modules/apiClients/validator/error.ts +++ b/packages/brain/src/modules/apiClients/validator/error.ts @@ -1,8 +1,8 @@ import { ApiError } from "../error.js"; export class ValidatorApiError extends ApiError { - constructor(message: string) { - super(message); + constructor(message: string, options?: ErrorOptions) { + super(message, options); this.name = "ValidatorApiError"; // Override the name if needed } } diff --git a/packages/brain/src/modules/apiClients/validator/index.ts b/packages/brain/src/modules/apiClients/validator/index.ts index 92705633..cafe1da7 100644 --- a/packages/brain/src/modules/apiClients/validator/index.ts +++ b/packages/brain/src/modules/apiClients/validator/index.ts @@ -85,7 +85,10 @@ export class ValidatorApi extends StandardApi { error instanceof HttpApiError ? `HTTP ${error.statusCode}${[405, 501].includes(error.statusCode) ? " (builder configuration is unsupported)" : error.statusCode === 404 ? " (endpoint unavailable or validator key not found)" : ""}` : error.message; - return new ValidatorApiError(`Error ${action} builder configuration for ${pubkey}: ${reason}`); + // Keep whether the client answered (for reload to stop on an unreachable client), never the body. + const cause = + error instanceof HttpApiError ? new HttpApiError(error.statusCode, `HTTP ${error.statusCode}`) : error; + return new ValidatorApiError(`Error ${action} builder configuration for ${pubkey}: ${reason}`, { cause }); } /** @@ -120,16 +123,22 @@ export class ValidatorApi extends StandardApi { * Sets the validator client fee recipient mapping which will then update the beacon node.. * https://ethereum.github.io/keymanager-APIs/#/Fee%20Recipient/setFeeRecipient */ - public async setFeeRecipient(newFeeRecipient: string, publicKey: string): Promise { + public async setFeeRecipient( + newFeeRecipient: string, + publicKey: string, + { timeout }: { timeout?: number } = {} + ): Promise { try { await this.request({ method: "POST", endpoint: path.join(this.validatorEndpoint, prefix0xPubkey(publicKey), "feerecipient"), - body: JSON.stringify({ ethaddress: newFeeRecipient }) + body: JSON.stringify({ ethaddress: newFeeRecipient }), + timeout }); } catch (e) { throw new ValidatorApiError( - `Error setting fee recipient for pubkey ${publicKey} to ${newFeeRecipient} on validator. ${e.message}` + `Error setting fee recipient for pubkey ${publicKey} to ${newFeeRecipient} on validator. ${e.message}`, + { cause: e } ); } } diff --git a/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts b/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts new file mode 100644 index 00000000..05fe908b --- /dev/null +++ b/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts @@ -0,0 +1,43 @@ +import type { BrainDataBase } from "../../db/index.js"; +import type { PubkeyDetails } from "../../db/types.js"; +import { withValidatorOperationLock } from "../../db/withValidatorOperationLock.js"; + +/** + * How long one slice of a per-key reload step may hold the validator operation lock. User operations + * queued meanwhile wait for at most this plus one in-flight client request, far below their wait budget. + */ +export const RELOAD_SLICE_MAX_HOLD_MS = 1000; + +export type SliceStep = (pubkey: string, details: PubkeyDetails) => Promise<"continue" | "stop">; + +/** + * Apply `step` to each key, holding the lock only for short time slices so user operations can run in + * between. Every slice re-reads the database inside the lock, so it always applies the settings stored + * now, never a value a user operation has since replaced, and skips keys deleted meanwhile. A step that + * returns "stop" ends the whole run; the next reload pass starts over. + * + * @returns "stopped" when a step stopped the run, so callers can skip later steps against the same client + */ +export async function forEachInLockedSlices( + brainDb: BrainDataBase, + pubkeys: readonly string[], + step: SliceStep, + { maxHoldMs = RELOAD_SLICE_MAX_HOLD_MS }: { maxHoldMs?: number } = {} +): Promise<"completed" | "stopped"> { + let next = 0; + let stopped = false; + while (next < pubkeys.length && !stopped) { + await withValidatorOperationLock(brainDb, async () => { + const data = brainDb.getData(); + const deadline = Date.now() + maxHoldMs; + // At least one key per slice, so every slice makes progress. + do { + const pubkey = pubkeys[next++]; + const details = data[pubkey]; + if (!details || details.importPending) continue; + if ((await step(pubkey, details)) === "stop") stopped = true; + } while (next < pubkeys.length && !stopped && Date.now() < deadline); + }); + } + return stopped ? "stopped" : "completed"; +} diff --git a/packages/brain/src/modules/cron/reloadValidators/index.ts b/packages/brain/src/modules/cron/reloadValidators/index.ts index f2af6a26..ac769cc9 100644 --- a/packages/brain/src/modules/cron/reloadValidators/index.ts +++ b/packages/brain/src/modules/cron/reloadValidators/index.ts @@ -21,6 +21,8 @@ import { postValidatorsBuilderConfigsFromDb } from "./postValidatorsBuilderConfi * - DELETE to validator API pubkeys that are in validator API and not in DB * - POST to validator API fee recipients that are in DB and not in validator API * + * Key membership (steps 0-5) is reconciled under one lock; per-key settings are then applied in short + * locked slices that re-read the database, so user operations never wait for a whole pass. */ /** Steps that tidy up rather than configure: log a failure and carry on with the pass. */ async function continueOnFailure(step: string, run: () => Promise): Promise { @@ -37,72 +39,84 @@ export async function reloadValidators( validatorApi: ValidatorApi, brainDb: BrainDataBase ): Promise { - return withValidatorOperationLock(brainDb, async () => { - try { - logger.debug(`${logPrefix}Reloading data...`); + try { + logger.debug(`${logPrefix}Reloading data...`); + // Steps 0-5 reconcile key membership with a few bulk calls under one lock. + const activePubkeys = await withValidatorOperationLock(brainDb, () => + syncValidatorKeys(signerApi, signerUrl, validatorApi, brainDb) + ); - // 0. Check signer API upcheck endpoint - await signerApi.upcheck(); + // 6. POST to validator API fee recipients that are in DB in validator API. One request per key, so + // it holds the lock in short slices: a long pass on a large node must not lock out user operations. + const fees = await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys }); - // 1. GET data - const dbPubkeys = Object.keys(brainDb.getData()); - const signerPubkeys = (await signerApi.listRemoteKeys()).data.map((keystore) => keystore.validating_pubkey); + // Independently retry managed configuration after key imports and fee destinations. Skip it when the + // client stopped responding: every request would wait for its timeout and delay the next pass. + if (featureFlags.builderConfig) { + if (fees === "stopped") logger.debug(`${logPrefix}Skipping builder configuration: validator client unavailable`); + else await postValidatorsBuilderConfigsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys }); + } + // Repairs such as restored fee recipients change no database record. If an open UI last saw a + // problem, announce the pass so it lists again (and checks the fee recipients by GET). + if (takeListRecheck(brainDb)) brainDb.notifyChanged(); + logger.debug(`${logPrefix}Finished reloading data`); + } catch (e) { + logger.error(`${logPrefix}Error reloading data`, e); + } +} - // 2. DELETE from signer API pubkeys that are not in DB. Keys whose deletion fails stay listed, so - // the database keeps them and the next pass retries. - await continueOnFailure("Deleting signer keys not in the database", () => - deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys }) - ); +/** Steps 0-5. Returns the keys the signer holds, in database spelling, for per-key configuration. */ +async function syncValidatorKeys( + signerApi: Web3SignerApi, + signerUrl: string, + validatorApi: ValidatorApi, + brainDb: BrainDataBase +): Promise { + // 0. Check signer API upcheck endpoint + await signerApi.upcheck(); - // 3. DELETE from DB pubkeys that are not in signer API - await deleteDbPubkeysNotInSigner({ brainDb, dbPubkeys, signerPubkeys }); + // 1. GET data + const dbPubkeys = Object.keys(brainDb.getData()); + const signerPubkeys = (await signerApi.listRemoteKeys()).data.map((keystore) => keystore.validating_pubkey); - const signerKeys = new Set(signerPubkeys.map((pubkey) => pubkey.toLowerCase())); - const data = brainDb.getData(); - const confirmedImports = dbPubkeys.filter( - (pubkey) => data[pubkey].importPending && signerKeys.has(pubkey.toLowerCase()) - ); - if (confirmedImports.length > 0) - brainDb.updateValidators(confirmedImports.map((pubkey) => ({ pubkey, importPending: undefined }))); - // Preserve pending records, but do not activate keys the signer has not confirmed. - const activeData = Object.fromEntries( - Object.entries(brainDb.getData()).filter(([pubkey]) => signerKeys.has(pubkey.toLowerCase())) - ); + // 2. DELETE from signer API pubkeys that are not in DB. Keys whose deletion fails stay listed, so + // the database keeps them and the next pass retries. + await continueOnFailure("Deleting signer keys not in the database", () => + deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys }) + ); - const validatorPubkeys = (await validatorApi.getRemoteKeys()).data.map((keystore) => keystore.pubkey); - const validatorKeys = new Set(validatorPubkeys.map((pubkey) => pubkey.toLowerCase())); - const dbKeys = new Set(dbPubkeys.map((pubkey) => pubkey.toLowerCase())); + // 3. DELETE from DB pubkeys that are not in signer API + await deleteDbPubkeysNotInSigner({ brainDb, dbPubkeys, signerPubkeys }); - // 4. POST to validator API pubkeys that are in DB and not in validator API - const brainDbPubkeysToAdd = Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())); - await continueOnFailure("Adding keys to the validator client", () => - postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys }) - ); + const signerKeys = new Set(signerPubkeys.map((pubkey) => pubkey.toLowerCase())); + const data = brainDb.getData(); + const confirmedImports = dbPubkeys.filter( + (pubkey) => data[pubkey]?.importPending && signerKeys.has(pubkey.toLowerCase()) + ); + if (confirmedImports.length > 0) + brainDb.updateValidators(confirmedImports.map((pubkey) => ({ pubkey, importPending: undefined }))); + // Preserve pending records, but do not activate keys the signer has not confirmed. + const activePubkeys = Object.keys(brainDb.getData()).filter((pubkey) => signerKeys.has(pubkey.toLowerCase())); - // 5. DELETE to validator API pubkeys that are in validator API and not in DB - const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())); - // Cleanup must not stop desired validators from being configured; failed removals stay listed and - // are retried next pass. - await continueOnFailure("Removing obsolete keys from the validator client", () => - deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove }) - ); - // Client membership is shown in the UI; repairs here change it without touching the database. - if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged(); + const validatorPubkeys = (await validatorApi.getRemoteKeys()).data.map((keystore) => keystore.pubkey); + const validatorKeys = new Set(validatorPubkeys.map((pubkey) => pubkey.toLowerCase())); + const dbKeys = new Set(dbPubkeys.map((pubkey) => pubkey.toLowerCase())); - // 6. POST to validator API fee recipients that are in DB in validator API - await postValidatorsFeeRecipientsFromDb({ - validatorApi, - dbData: activeData - }); + // 4. POST to validator API pubkeys that are in DB and not in validator API + const brainDbPubkeysToAdd = activePubkeys.filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())); + await continueOnFailure("Adding keys to the validator client", () => + postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys }) + ); - // Independently retry managed configuration after key imports and fee destinations. - if (featureFlags.builderConfig) await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData }); - // Repairs such as restored fee recipients change no database record. If an open UI last saw a - // problem, announce the pass so it lists again (and checks the fee recipients by GET). - if (takeListRecheck(brainDb)) brainDb.notifyChanged(); - logger.debug(`${logPrefix}Finished reloading data`); - } catch (e) { - logger.error(`${logPrefix}Error reloading data`, e); - } - }); + // 5. DELETE to validator API pubkeys that are in validator API and not in DB + const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())); + // Cleanup must not stop desired validators from being configured; failed removals stay listed and + // are retried next pass. + await continueOnFailure("Removing obsolete keys from the validator client", () => + deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove }) + ); + // Client membership is shown in the UI; repairs here change it without touching the database. + if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged(); + + return activePubkeys; } diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts index 8f51fe1c..017d65e2 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts @@ -1,8 +1,10 @@ import { BuilderConfig } from "@stakingbrain/common"; import { ValidatorApi } from "../../apiClients/index.js"; -import { StakingBrainDb } from "../../db/types.js"; +import { receivedHttpResponse } from "../../apiClients/error.js"; +import { BrainDataBase } from "../../db/index.js"; import logger from "../../logger/index.js"; import { logPrefix } from "./logPrefix.js"; +import { forEachInLockedSlices } from "./forEachInLockedSlices.js"; export async function applyValidatorBuilderConfig( validatorApi: ValidatorApi, @@ -13,17 +15,33 @@ export async function applyValidatorBuilderConfig( else if (config !== undefined) await validatorApi.setBuilderConfig(config, pubkey); } -/** Replay original overrides, including reset intent. Never copy resolved GET values. */ +/** + * Replay original overrides, including reset intent. Never copy resolved GET values. + * Like fee recipients, an HTTP error concerns one key; no response at all stops the step. + */ export async function postValidatorsBuilderConfigsFromDb({ validatorApi, - dbData + brainDb, + pubkeys }: { validatorApi: ValidatorApi; - dbData: StakingBrainDb; -}): Promise { - for (const [pubkey, { builderConfig }] of Object.entries(dbData)) { - await applyValidatorBuilderConfig(validatorApi, pubkey, builderConfig).catch((error) => - logger.error(`${logPrefix}Error applying builder configuration for ${pubkey}`, error) - ); - } + brainDb: BrainDataBase; + pubkeys: readonly string[]; +}): Promise<"completed" | "stopped"> { + return forEachInLockedSlices(brainDb, pubkeys, async (pubkey, { builderConfig }) => { + try { + await applyValidatorBuilderConfig(validatorApi, pubkey, builderConfig); + return "continue"; + } catch (error) { + if (!receivedHttpResponse(error)) { + logger.error( + `${logPrefix}Validator client did not respond while applying builder configuration; retrying next pass`, + error + ); + return "stop"; + } + logger.error(`${logPrefix}Error applying builder configuration for ${pubkey}`, error); + return "continue"; + } + }); } diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts index 6fc9c7f5..e38e3d5b 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts @@ -1,25 +1,56 @@ import { ValidatorApi } from "../../apiClients/index.js"; -import { StakingBrainDb } from "../../db/types.js"; +import { receivedHttpResponse } from "../../apiClients/error.js"; +import { BrainDataBase } from "../../db/index.js"; import logger from "../../logger/index.js"; import { logPrefix } from "./logPrefix.js"; +import { forEachInLockedSlices } from "./forEachInLockedSlices.js"; + +/** A healthy client answers a fee recipient write quickly; a hung one must not stall the pass for long. */ +export const FEE_RECIPIENT_TIMEOUT_MS = 10 * 1000; /** - * Post in the validator API fee recipients that are in the DB in the validator API + * Post to the validator client the fee recipient stored for each key. + * + * An HTTP error concerns one key, so the pass carries on with the rest. A request that gets no response + * (timeout, refused or reset connection) means the client is unavailable: stop, rather than wait on every + * remaining key, and let the next pass start over. + * + * @returns "stopped" when the client did not respond, so reload skips its other per-key steps */ export async function postValidatorsFeeRecipientsFromDb({ validatorApi, - dbData + brainDb, + pubkeys, + timeoutMs = FEE_RECIPIENT_TIMEOUT_MS, + maxHoldMs }: { validatorApi: ValidatorApi; - dbData: StakingBrainDb; -}): Promise { - for (const [pubkey, { feeRecipient }] of Object.entries(dbData)) { - await validatorApi - .setFeeRecipient(feeRecipient, pubkey) - .catch((e) => + brainDb: BrainDataBase; + pubkeys: readonly string[]; + timeoutMs?: number; + maxHoldMs?: number; +}): Promise<"completed" | "stopped"> { + return forEachInLockedSlices( + brainDb, + pubkeys, + async (pubkey, { feeRecipient }) => { + try { + await validatorApi.setFeeRecipient(feeRecipient, pubkey, { timeout: timeoutMs }); + return "continue"; + } catch (e) { + if (!receivedHttpResponse(e)) { + logger.error( + `${logPrefix}Validator client did not respond while posting fee recipients; retrying all keys next pass`, + e + ); + return "stop"; + } logger.error( `${logPrefix}Error posting fee recipient ${feeRecipient} for pubkey ${pubkey} to validator API: ${e}` - ) - ); - } + ); + return "continue"; + } + }, + { maxHoldMs } + ); } diff --git a/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts b/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts new file mode 100644 index 00000000..74d5a4a5 --- /dev/null +++ b/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts @@ -0,0 +1,216 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { once } from "node:events"; +import { inspect } from "node:util"; +import { BuilderConfig, featureFlags, Network } from "@stakingbrain/common"; +import { BrainDataBase } from "../../../src/modules/db/index.js"; +import { CronJob } from "../../../src/modules/cron/cron.js"; +import { ValidatorApi, Web3SignerApi } from "../../../src/modules/apiClients/index.js"; +import { ApiError } from "../../../src/modules/apiClients/error.js"; +import { ValidatorApiError } from "../../../src/modules/apiClients/validator/error.js"; +import { updateValidators } from "../../../src/modules/apiServers/ui/calls/updateValidators.js"; +import { reloadValidators } from "../../../src/modules/cron/reloadValidators/index.js"; +import { postValidatorsFeeRecipientsFromDb } from "../../../src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.js"; +import { postValidatorsBuilderConfigsFromDb } from "../../../src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.js"; +import { RELOAD_SLICE_MAX_HOLD_MS } from "../../../src/modules/cron/reloadValidators/forEachInLockedSlices.js"; +import logger from "../../../src/modules/logger/index.js"; + +const pubkeyAt = (i: number) => "0x" + i.toString(16).padStart(96, "0"); +const recipient = "0x" + "ab".repeat(20); +const newRecipient = "0x" + "cd".repeat(20); +const builderConfig: BuilderConfig = { builders: [{ url: "https://builder.example" }] }; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +describe("Reload holds the validator operation lock in short slices", () => { + let directory: string; + let brainDb: BrainDataBase; + + const seed = (count: number, extra: { builderConfig?: BuilderConfig } = {}): string[] => { + const keys = Array.from({ length: count }, (_, i) => pubkeyAt(i + 1)); + brainDb.addValidators({ + validators: Object.fromEntries( + keys.map((key) => [key, { tag: "solo" as const, feeRecipient: recipient, automaticImport: false, ...extra }]) + ) + }); + return keys; + }; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-slices-")); + brainDb = new BrainDataBase(path.join(directory, "db.json")); + sinon.stub(logger, "error"); + sinon.stub(logger, "debug"); + sinon.stub(logger, "info"); + sinon.stub(logger, "warn"); + }); + + afterEach(() => { + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("lets a user fee update run mid-pass and never re-sends the replaced value", async function () { + this.timeout(20 * 1000); + // 800 keys at 5 ms per write: a pass far longer than one slice, as on a large node. + const keys = seed(800); + const target = keys[keys.length - 1]; + const signerApi = sinon.createStubInstance(Web3SignerApi); + const validatorApi = sinon.createStubInstance(ValidatorApi); + signerApi.upcheck.resolves(); + signerApi.listRemoteKeys.resolves({ + data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + validatorApi.getRemoteKeys.resolves({ data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "" })) }); + validatorApi.setFeeRecipient.callsFake(() => sleep(5)); + + const started = Date.now(); + const pass = reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + await sleep(300); + const updateStarted = Date.now(); + await updateValidators({ + brainDb, + validatorApi, + signerUrl: "http://signer", + reloadValidatorsCronTask: sinon.createStubInstance(CronJob), + customValidatorUpdateRequest: [{ pubkey: target, feeRecipient: newRecipient }] + }); + const waited = Date.now() - updateStarted; + await pass; + + // The pass was still running when the update finished, so the update did not wait for all of it. + expect(Date.now() - started).to.be.greaterThan(waited + 1000); + expect(waited).to.be.lessThan(RELOAD_SLICE_MAX_HOLD_MS + 500); + // Every key was configured by the pass. + const reloadCalls = validatorApi.setFeeRecipient.getCalls().filter((call) => call.args[2] !== undefined); + expect(new Set(reloadCalls.map((call) => call.args[1])).size).to.equal(keys.length); + // The pass reached the target after the update and sent the new value, not its stale snapshot. + const targetCalls = reloadCalls.filter((call) => call.args[1] === target); + expect(targetCalls.map((call) => call.args[0])).to.deep.equal([newRecipient]); + expect(brainDb.getData()[target].feeRecipient).to.equal(newRecipient); + }); + + it("skips keys deleted while the pass is running", async () => { + const keys = seed(3); + const validatorApi = sinon.createStubInstance(ValidatorApi); + validatorApi.setFeeRecipient.onFirstCall().callsFake(async () => brainDb.deleteValidators([keys[2]])); + validatorApi.setFeeRecipient.resolves(); + // One key per slice, so the deletion lands between slices. + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: keys, maxHoldMs: 0 }); + expect(validatorApi.setFeeRecipient.getCalls().map((call) => call.args[1])).to.deep.equal(keys.slice(0, 2)); + }); + + it("skips builder configuration when the validator client stopped responding to fee recipients", async () => { + const keys = seed(3, { builderConfig }); + const signerApi = sinon.createStubInstance(Web3SignerApi); + const validatorApi = sinon.createStubInstance(ValidatorApi); + signerApi.upcheck.resolves(); + signerApi.listRemoteKeys.resolves({ + data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + validatorApi.getRemoteKeys.resolves({ data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "" })) }); + // What the real client throws when a request gets no response. + validatorApi.setFeeRecipient.rejects(new ValidatorApiError("fee", { cause: new ApiError("timed out") })); + validatorApi.setBuilderConfig.resolves(); + featureFlags.builderConfig = true; + try { + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(validatorApi.setFeeRecipient.callCount).to.equal(1); + expect(validatorApi.setBuilderConfig.called).to.equal(false); + + // A responsive client gets both phases again on the next pass. + validatorApi.setFeeRecipient.resolves(); + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(validatorApi.setBuilderConfig.callCount).to.equal(keys.length); + } finally { + featureFlags.builderConfig = false; + } + }); + + describe("against a real validator client connection", () => { + let server: http.Server; + let validatorApi: ValidatorApi; + let requests: string[]; + let respond: (req: http.IncomingMessage, res: http.ServerResponse) => void; + + beforeEach(async () => { + requests = []; + server = http.createServer((req, res) => { + requests.push(req.url ?? ""); + req.resume(); + respond(req, res); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const { port } = server.address() as AddressInfo; + validatorApi = new ValidatorApi({ baseUrl: `http://127.0.0.1:${port}` }, Network.Mainnet); + }); + + afterEach(() => { + server.closeAllConnections(); + server.close(); + }); + + it("stops at the first request that times out and retries every key next pass", async () => { + const keys = seed(3); + respond = () => undefined; // never answers + const started = Date.now(); + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: keys, timeoutMs: 100 }); + expect(Date.now() - started).to.be.lessThan(2000); + expect(requests).to.have.length(1); + + respond = (_req, res) => res.writeHead(202).end(); + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: keys, timeoutMs: 100 }); + expect(requests).to.have.length(1 + keys.length); + }); + + it("keeps going past HTTP errors, which concern only one key", async () => { + const keys = seed(3); + respond = (req, res) => res.writeHead(req.url?.includes(keys[0]) ? 404 : 202).end(); + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: keys }); + expect(requests).to.have.length(keys.length); + }); + + it("stops at the first refused connection", async () => { + const keys = seed(3); + const { port } = server.address() as AddressInfo; + server.close(); + await once(server, "close"); + const closedApi = new ValidatorApi({ baseUrl: `http://127.0.0.1:${port}` }, Network.Mainnet); + const spy = sinon.spy(closedApi, "setFeeRecipient"); + await postValidatorsFeeRecipientsFromDb({ validatorApi: closedApi, brainDb, pubkeys: keys }); + expect(spy.callCount).to.equal(1); + server.listen(0, "127.0.0.1"); // let afterEach close it cleanly + await once(server, "listening"); + }); + + it("stops builder configuration at the first refused connection", async () => { + const keys = seed(3, { builderConfig }); + const { port } = server.address() as AddressInfo; + server.close(); + await once(server, "close"); + const closedApi = new ValidatorApi({ baseUrl: `http://127.0.0.1:${port}` }, Network.Mainnet); + const spy = sinon.spy(closedApi, "setBuilderConfig"); + expect(await postValidatorsBuilderConfigsFromDb({ validatorApi: closedApi, brainDb, pubkeys: keys })).to.equal( + "stopped" + ); + expect(spy.callCount).to.equal(1); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + }); + + it("keeps applying builder configuration past HTTP errors without logging response bodies", async () => { + const keys = seed(3, { builderConfig }); + respond = (req, res) => + req.url?.includes(keys[0]) ? res.writeHead(500).end("echoed-secret-authorization") : res.writeHead(200).end(); + expect(await postValidatorsBuilderConfigsFromDb({ validatorApi, brainDb, pubkeys: keys })).to.equal("completed"); + expect(requests).to.have.length(keys.length); + const logged = (logger.error as sinon.SinonStub).getCalls().map((call) => inspect(call.args, { depth: 10 })); + expect(logged.join("\n")).to.include("HTTP 500").and.not.include("echoed-secret"); + }); + }); +});