From 0f8265eac61f0b3f0c74f0d4cfd3f9e80d3258c4 Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:05:01 +0100 Subject: [PATCH 1/2] fix: hold the reload lock in short slices and stop on an unresponsive client The reload pass held the validator operation lock for its whole run, including one fee recipient POST per key. On large nodes a pass can exceed the 30s wait budget of user operations, which then failed as busy (HTTP 503 on the Launchpad API). Key membership (steps 0-5) is still reconciled under one lock. Fee recipients and builder configs are then applied in slices that hold the lock for at most 1s and re-read the database inside the lock, so they never re-send a value a user has since replaced and skip keys deleted mid-pass. Reload fee recipient writes now time out after 10s. A request with no HTTP response (timeout, refused or reset) stops the pass; the next pass retries every key. HTTP errors concern one key and the pass continues, as before. --- packages/brain/src/createErrorFactory.ts | 4 +- .../brain/src/modules/apiClients/error.ts | 6 + .../src/modules/apiClients/validator/error.ts | 4 +- .../src/modules/apiClients/validator/index.ts | 12 +- .../reloadValidators/forEachInLockedSlices.ts | 40 +++++ .../modules/cron/reloadValidators/index.ts | 127 +++++++------- .../postValidatorsBuilderConfigsFromDb.ts | 14 +- .../postValidatorsFeeRecipientsFromDb.ts | 51 ++++-- .../modules/reloadLockSlices.unit.test.ts | 159 ++++++++++++++++++ 9 files changed, 336 insertions(+), 81 deletions(-) create mode 100644 packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts create mode 100644 packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts diff --git a/packages/brain/src/createErrorFactory.ts b/packages/brain/src/createErrorFactory.ts index 918d064f..eaf250e0 100644 --- a/packages/brain/src/createErrorFactory.ts +++ b/packages/brain/src/createErrorFactory.ts @@ -1,7 +1,7 @@ export function createErrorFactory(errorName: string) { return class CustomError extends Error { - constructor(message: string) { - super(message); + constructor(message: string, options?: ErrorOptions) { + super(message, options); this.name = errorName; // Set the error name provided to the factory Error.captureStackTrace(this, CustomError); // Capture the stack trace and omit the constructor frame } diff --git a/packages/brain/src/modules/apiClients/error.ts b/packages/brain/src/modules/apiClients/error.ts index fed2abb5..9cb33825 100644 --- a/packages/brain/src/modules/apiClients/error.ts +++ b/packages/brain/src/modules/apiClients/error.ts @@ -10,3 +10,9 @@ export class HttpApiError extends ApiError { super(message); } } + +/** True when a request got an HTTP response, i.e. the server is reachable even if it refused the request. */ +export function receivedHttpResponse(error: unknown): boolean { + for (let e = error; e instanceof Error; e = e.cause) if (e instanceof HttpApiError) return true; + return false; +} diff --git a/packages/brain/src/modules/apiClients/validator/error.ts b/packages/brain/src/modules/apiClients/validator/error.ts index 2a9773f7..6308bae9 100644 --- a/packages/brain/src/modules/apiClients/validator/error.ts +++ b/packages/brain/src/modules/apiClients/validator/error.ts @@ -1,8 +1,8 @@ import { ApiError } from "../error.js"; export class ValidatorApiError extends ApiError { - constructor(message: string) { - super(message); + constructor(message: string, options?: ErrorOptions) { + super(message, options); this.name = "ValidatorApiError"; // Override the name if needed } } diff --git a/packages/brain/src/modules/apiClients/validator/index.ts b/packages/brain/src/modules/apiClients/validator/index.ts index 92705633..e9d7f5d1 100644 --- a/packages/brain/src/modules/apiClients/validator/index.ts +++ b/packages/brain/src/modules/apiClients/validator/index.ts @@ -120,16 +120,22 @@ export class ValidatorApi extends StandardApi { * Sets the validator client fee recipient mapping which will then update the beacon node.. * https://ethereum.github.io/keymanager-APIs/#/Fee%20Recipient/setFeeRecipient */ - public async setFeeRecipient(newFeeRecipient: string, publicKey: string): Promise { + public async setFeeRecipient( + newFeeRecipient: string, + publicKey: string, + { timeout }: { timeout?: number } = {} + ): Promise { try { await this.request({ method: "POST", endpoint: path.join(this.validatorEndpoint, prefix0xPubkey(publicKey), "feerecipient"), - body: JSON.stringify({ ethaddress: newFeeRecipient }) + body: JSON.stringify({ ethaddress: newFeeRecipient }), + timeout }); } catch (e) { throw new ValidatorApiError( - `Error setting fee recipient for pubkey ${publicKey} to ${newFeeRecipient} on validator. ${e.message}` + `Error setting fee recipient for pubkey ${publicKey} to ${newFeeRecipient} on validator. ${e.message}`, + { cause: e } ); } } diff --git a/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts b/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts new file mode 100644 index 00000000..27017489 --- /dev/null +++ b/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts @@ -0,0 +1,40 @@ +import type { BrainDataBase } from "../../db/index.js"; +import type { PubkeyDetails } from "../../db/types.js"; +import { withValidatorOperationLock } from "../../db/withValidatorOperationLock.js"; + +/** + * How long one slice of a per-key reload step may hold the validator operation lock. User operations + * queued meanwhile wait for at most this plus one in-flight client request, far below their wait budget. + */ +export const RELOAD_SLICE_MAX_HOLD_MS = 1000; + +export type SliceStep = (pubkey: string, details: PubkeyDetails) => Promise<"continue" | "stop">; + +/** + * Apply `step` to each key, holding the lock only for short time slices so user operations can run in + * between. Every slice re-reads the database inside the lock, so it always applies the settings stored + * now, never a value a user operation has since replaced, and skips keys deleted meanwhile. A step that + * returns "stop" ends the whole run; the next reload pass starts over. + */ +export async function forEachInLockedSlices( + brainDb: BrainDataBase, + pubkeys: readonly string[], + step: SliceStep, + { maxHoldMs = RELOAD_SLICE_MAX_HOLD_MS }: { maxHoldMs?: number } = {} +): Promise { + let next = 0; + let stopped = false; + while (next < pubkeys.length && !stopped) { + await withValidatorOperationLock(brainDb, async () => { + const data = brainDb.getData(); + const deadline = Date.now() + maxHoldMs; + // At least one key per slice, so every slice makes progress. + do { + const pubkey = pubkeys[next++]; + const details = data[pubkey]; + if (!details || details.importPending) continue; + if ((await step(pubkey, details)) === "stop") stopped = true; + } while (next < pubkeys.length && !stopped && Date.now() < deadline); + }); + } +} diff --git a/packages/brain/src/modules/cron/reloadValidators/index.ts b/packages/brain/src/modules/cron/reloadValidators/index.ts index f2af6a26..086baae1 100644 --- a/packages/brain/src/modules/cron/reloadValidators/index.ts +++ b/packages/brain/src/modules/cron/reloadValidators/index.ts @@ -21,6 +21,8 @@ import { postValidatorsBuilderConfigsFromDb } from "./postValidatorsBuilderConfi * - DELETE to validator API pubkeys that are in validator API and not in DB * - POST to validator API fee recipients that are in DB and not in validator API * + * Key membership (steps 0-5) is reconciled under one lock; per-key settings are then applied in short + * locked slices that re-read the database, so user operations never wait for a whole pass. */ /** Steps that tidy up rather than configure: log a failure and carry on with the pass. */ async function continueOnFailure(step: string, run: () => Promise): Promise { @@ -37,72 +39,81 @@ export async function reloadValidators( validatorApi: ValidatorApi, brainDb: BrainDataBase ): Promise { - return withValidatorOperationLock(brainDb, async () => { - try { - logger.debug(`${logPrefix}Reloading data...`); + try { + logger.debug(`${logPrefix}Reloading data...`); + // Steps 0-5 reconcile key membership with a few bulk calls under one lock. + const activePubkeys = await withValidatorOperationLock(brainDb, () => + syncValidatorKeys(signerApi, signerUrl, validatorApi, brainDb) + ); + + // 6. POST to validator API fee recipients that are in DB in validator API. One request per key, so + // it holds the lock in short slices: a long pass on a large node must not lock out user operations. + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys }); - // 0. Check signer API upcheck endpoint - await signerApi.upcheck(); + // Independently retry managed configuration after key imports and fee destinations. + if (featureFlags.builderConfig) + await postValidatorsBuilderConfigsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys }); + // Repairs such as restored fee recipients change no database record. If an open UI last saw a + // problem, announce the pass so it lists again (and checks the fee recipients by GET). + if (takeListRecheck(brainDb)) brainDb.notifyChanged(); + logger.debug(`${logPrefix}Finished reloading data`); + } catch (e) { + logger.error(`${logPrefix}Error reloading data`, e); + } +} - // 1. GET data - const dbPubkeys = Object.keys(brainDb.getData()); - const signerPubkeys = (await signerApi.listRemoteKeys()).data.map((keystore) => keystore.validating_pubkey); +/** Steps 0-5. Returns the keys the signer holds, in database spelling, for per-key configuration. */ +async function syncValidatorKeys( + signerApi: Web3SignerApi, + signerUrl: string, + validatorApi: ValidatorApi, + brainDb: BrainDataBase +): Promise { + // 0. Check signer API upcheck endpoint + await signerApi.upcheck(); - // 2. DELETE from signer API pubkeys that are not in DB. Keys whose deletion fails stay listed, so - // the database keeps them and the next pass retries. - await continueOnFailure("Deleting signer keys not in the database", () => - deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys }) - ); + // 1. GET data + const dbPubkeys = Object.keys(brainDb.getData()); + const signerPubkeys = (await signerApi.listRemoteKeys()).data.map((keystore) => keystore.validating_pubkey); - // 3. DELETE from DB pubkeys that are not in signer API - await deleteDbPubkeysNotInSigner({ brainDb, dbPubkeys, signerPubkeys }); + // 2. DELETE from signer API pubkeys that are not in DB. Keys whose deletion fails stay listed, so + // the database keeps them and the next pass retries. + await continueOnFailure("Deleting signer keys not in the database", () => + deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys }) + ); - const signerKeys = new Set(signerPubkeys.map((pubkey) => pubkey.toLowerCase())); - const data = brainDb.getData(); - const confirmedImports = dbPubkeys.filter( - (pubkey) => data[pubkey].importPending && signerKeys.has(pubkey.toLowerCase()) - ); - if (confirmedImports.length > 0) - brainDb.updateValidators(confirmedImports.map((pubkey) => ({ pubkey, importPending: undefined }))); - // Preserve pending records, but do not activate keys the signer has not confirmed. - const activeData = Object.fromEntries( - Object.entries(brainDb.getData()).filter(([pubkey]) => signerKeys.has(pubkey.toLowerCase())) - ); + // 3. DELETE from DB pubkeys that are not in signer API + await deleteDbPubkeysNotInSigner({ brainDb, dbPubkeys, signerPubkeys }); - const validatorPubkeys = (await validatorApi.getRemoteKeys()).data.map((keystore) => keystore.pubkey); - const validatorKeys = new Set(validatorPubkeys.map((pubkey) => pubkey.toLowerCase())); - const dbKeys = new Set(dbPubkeys.map((pubkey) => pubkey.toLowerCase())); + const signerKeys = new Set(signerPubkeys.map((pubkey) => pubkey.toLowerCase())); + const data = brainDb.getData(); + const confirmedImports = dbPubkeys.filter( + (pubkey) => data[pubkey]?.importPending && signerKeys.has(pubkey.toLowerCase()) + ); + if (confirmedImports.length > 0) + brainDb.updateValidators(confirmedImports.map((pubkey) => ({ pubkey, importPending: undefined }))); + // Preserve pending records, but do not activate keys the signer has not confirmed. + const activePubkeys = Object.keys(brainDb.getData()).filter((pubkey) => signerKeys.has(pubkey.toLowerCase())); - // 4. POST to validator API pubkeys that are in DB and not in validator API - const brainDbPubkeysToAdd = Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())); - await continueOnFailure("Adding keys to the validator client", () => - postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys }) - ); + const validatorPubkeys = (await validatorApi.getRemoteKeys()).data.map((keystore) => keystore.pubkey); + const validatorKeys = new Set(validatorPubkeys.map((pubkey) => pubkey.toLowerCase())); + const dbKeys = new Set(dbPubkeys.map((pubkey) => pubkey.toLowerCase())); - // 5. DELETE to validator API pubkeys that are in validator API and not in DB - const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())); - // Cleanup must not stop desired validators from being configured; failed removals stay listed and - // are retried next pass. - await continueOnFailure("Removing obsolete keys from the validator client", () => - deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove }) - ); - // Client membership is shown in the UI; repairs here change it without touching the database. - if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged(); + // 4. POST to validator API pubkeys that are in DB and not in validator API + const brainDbPubkeysToAdd = activePubkeys.filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase())); + await continueOnFailure("Adding keys to the validator client", () => + postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys }) + ); - // 6. POST to validator API fee recipients that are in DB in validator API - await postValidatorsFeeRecipientsFromDb({ - validatorApi, - dbData: activeData - }); + // 5. DELETE to validator API pubkeys that are in validator API and not in DB + const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase())); + // Cleanup must not stop desired validators from being configured; failed removals stay listed and + // are retried next pass. + await continueOnFailure("Removing obsolete keys from the validator client", () => + deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove }) + ); + // Client membership is shown in the UI; repairs here change it without touching the database. + if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged(); - // Independently retry managed configuration after key imports and fee destinations. - if (featureFlags.builderConfig) await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData }); - // Repairs such as restored fee recipients change no database record. If an open UI last saw a - // problem, announce the pass so it lists again (and checks the fee recipients by GET). - if (takeListRecheck(brainDb)) brainDb.notifyChanged(); - logger.debug(`${logPrefix}Finished reloading data`); - } catch (e) { - logger.error(`${logPrefix}Error reloading data`, e); - } - }); + return activePubkeys; } diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts index 8f51fe1c..49cfbea7 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts @@ -1,8 +1,9 @@ import { BuilderConfig } from "@stakingbrain/common"; import { ValidatorApi } from "../../apiClients/index.js"; -import { StakingBrainDb } from "../../db/types.js"; +import { BrainDataBase } from "../../db/index.js"; import logger from "../../logger/index.js"; import { logPrefix } from "./logPrefix.js"; +import { forEachInLockedSlices } from "./forEachInLockedSlices.js"; export async function applyValidatorBuilderConfig( validatorApi: ValidatorApi, @@ -16,14 +17,17 @@ export async function applyValidatorBuilderConfig( /** Replay original overrides, including reset intent. Never copy resolved GET values. */ export async function postValidatorsBuilderConfigsFromDb({ validatorApi, - dbData + brainDb, + pubkeys }: { validatorApi: ValidatorApi; - dbData: StakingBrainDb; + brainDb: BrainDataBase; + pubkeys: readonly string[]; }): Promise { - for (const [pubkey, { builderConfig }] of Object.entries(dbData)) { + await forEachInLockedSlices(brainDb, pubkeys, async (pubkey, { builderConfig }) => { await applyValidatorBuilderConfig(validatorApi, pubkey, builderConfig).catch((error) => logger.error(`${logPrefix}Error applying builder configuration for ${pubkey}`, error) ); - } + return "continue"; + }); } diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts index 6fc9c7f5..631a3575 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts @@ -1,25 +1,54 @@ import { ValidatorApi } from "../../apiClients/index.js"; -import { StakingBrainDb } from "../../db/types.js"; +import { receivedHttpResponse } from "../../apiClients/error.js"; +import { BrainDataBase } from "../../db/index.js"; import logger from "../../logger/index.js"; import { logPrefix } from "./logPrefix.js"; +import { forEachInLockedSlices } from "./forEachInLockedSlices.js"; + +/** A healthy client answers a fee recipient write quickly; a hung one must not stall the pass for long. */ +export const FEE_RECIPIENT_TIMEOUT_MS = 10 * 1000; /** - * Post in the validator API fee recipients that are in the DB in the validator API + * Post to the validator client the fee recipient stored for each key. + * + * An HTTP error concerns one key, so the pass carries on with the rest. A request that gets no response + * (timeout, refused or reset connection) means the client is unavailable: stop, rather than wait on every + * remaining key, and let the next pass start over. */ export async function postValidatorsFeeRecipientsFromDb({ validatorApi, - dbData + brainDb, + pubkeys, + timeoutMs = FEE_RECIPIENT_TIMEOUT_MS, + maxHoldMs }: { validatorApi: ValidatorApi; - dbData: StakingBrainDb; + brainDb: BrainDataBase; + pubkeys: readonly string[]; + timeoutMs?: number; + maxHoldMs?: number; }): Promise { - for (const [pubkey, { feeRecipient }] of Object.entries(dbData)) { - await validatorApi - .setFeeRecipient(feeRecipient, pubkey) - .catch((e) => + await forEachInLockedSlices( + brainDb, + pubkeys, + async (pubkey, { feeRecipient }) => { + try { + await validatorApi.setFeeRecipient(feeRecipient, pubkey, { timeout: timeoutMs }); + return "continue"; + } catch (e) { + if (!receivedHttpResponse(e)) { + logger.error( + `${logPrefix}Validator client did not respond while posting fee recipients; retrying all keys next pass`, + e + ); + return "stop"; + } logger.error( `${logPrefix}Error posting fee recipient ${feeRecipient} for pubkey ${pubkey} to validator API: ${e}` - ) - ); - } + ); + return "continue"; + } + }, + { maxHoldMs } + ); } diff --git a/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts b/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts new file mode 100644 index 00000000..6a238c8a --- /dev/null +++ b/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts @@ -0,0 +1,159 @@ +import { expect } from "chai"; +import sinon from "sinon"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { once } from "node:events"; +import { Network } from "@stakingbrain/common"; +import { BrainDataBase } from "../../../src/modules/db/index.js"; +import { CronJob } from "../../../src/modules/cron/cron.js"; +import { ValidatorApi, Web3SignerApi } from "../../../src/modules/apiClients/index.js"; +import { updateValidators } from "../../../src/modules/apiServers/ui/calls/updateValidators.js"; +import { reloadValidators } from "../../../src/modules/cron/reloadValidators/index.js"; +import { postValidatorsFeeRecipientsFromDb } from "../../../src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.js"; +import { RELOAD_SLICE_MAX_HOLD_MS } from "../../../src/modules/cron/reloadValidators/forEachInLockedSlices.js"; +import logger from "../../../src/modules/logger/index.js"; + +const pubkeyAt = (i: number) => "0x" + i.toString(16).padStart(96, "0"); +const recipient = "0x" + "ab".repeat(20); +const newRecipient = "0x" + "cd".repeat(20); +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +describe("Reload holds the validator operation lock in short slices", () => { + let directory: string; + let brainDb: BrainDataBase; + + const seed = (count: number): string[] => { + const keys = Array.from({ length: count }, (_, i) => pubkeyAt(i + 1)); + brainDb.addValidators({ + validators: Object.fromEntries( + keys.map((key) => [key, { tag: "solo" as const, feeRecipient: recipient, automaticImport: false }]) + ) + }); + return keys; + }; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "stakingbrain-slices-")); + brainDb = new BrainDataBase(path.join(directory, "db.json")); + sinon.stub(logger, "error"); + sinon.stub(logger, "debug"); + sinon.stub(logger, "info"); + sinon.stub(logger, "warn"); + }); + + afterEach(() => { + sinon.restore(); + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("lets a user fee update run mid-pass and never re-sends the replaced value", async function () { + this.timeout(20 * 1000); + // 800 keys at 5 ms per write: a pass far longer than one slice, as on a large node. + const keys = seed(800); + const target = keys[keys.length - 1]; + const signerApi = sinon.createStubInstance(Web3SignerApi); + const validatorApi = sinon.createStubInstance(ValidatorApi); + signerApi.upcheck.resolves(); + signerApi.listRemoteKeys.resolves({ + data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + validatorApi.getRemoteKeys.resolves({ data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "" })) }); + validatorApi.setFeeRecipient.callsFake(() => sleep(5)); + + const started = Date.now(); + const pass = reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + await sleep(300); + const updateStarted = Date.now(); + await updateValidators({ + brainDb, + validatorApi, + signerUrl: "http://signer", + reloadValidatorsCronTask: sinon.createStubInstance(CronJob), + customValidatorUpdateRequest: [{ pubkey: target, feeRecipient: newRecipient }] + }); + const waited = Date.now() - updateStarted; + await pass; + + // The pass was still running when the update finished, so the update did not wait for all of it. + expect(Date.now() - started).to.be.greaterThan(waited + 1000); + expect(waited).to.be.lessThan(RELOAD_SLICE_MAX_HOLD_MS + 500); + // Every key was configured by the pass. + const reloadCalls = validatorApi.setFeeRecipient.getCalls().filter((call) => call.args[2] !== undefined); + expect(new Set(reloadCalls.map((call) => call.args[1])).size).to.equal(keys.length); + // The pass reached the target after the update and sent the new value, not its stale snapshot. + const targetCalls = reloadCalls.filter((call) => call.args[1] === target); + expect(targetCalls.map((call) => call.args[0])).to.deep.equal([newRecipient]); + expect(brainDb.getData()[target].feeRecipient).to.equal(newRecipient); + }); + + it("skips keys deleted while the pass is running", async () => { + const keys = seed(3); + const validatorApi = sinon.createStubInstance(ValidatorApi); + validatorApi.setFeeRecipient.onFirstCall().callsFake(async () => brainDb.deleteValidators([keys[2]])); + validatorApi.setFeeRecipient.resolves(); + // One key per slice, so the deletion lands between slices. + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: keys, maxHoldMs: 0 }); + expect(validatorApi.setFeeRecipient.getCalls().map((call) => call.args[1])).to.deep.equal(keys.slice(0, 2)); + }); + + describe("against a real validator client connection", () => { + let server: http.Server; + let validatorApi: ValidatorApi; + let requests: string[]; + let respond: (req: http.IncomingMessage, res: http.ServerResponse) => void; + + beforeEach(async () => { + requests = []; + server = http.createServer((req, res) => { + requests.push(req.url ?? ""); + req.resume(); + respond(req, res); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const { port } = server.address() as AddressInfo; + validatorApi = new ValidatorApi({ baseUrl: `http://127.0.0.1:${port}` }, Network.Mainnet); + }); + + afterEach(() => { + server.closeAllConnections(); + server.close(); + }); + + it("stops at the first request that times out and retries every key next pass", async () => { + const keys = seed(3); + respond = () => undefined; // never answers + const started = Date.now(); + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: keys, timeoutMs: 100 }); + expect(Date.now() - started).to.be.lessThan(2000); + expect(requests).to.have.length(1); + + respond = (_req, res) => res.writeHead(202).end(); + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: keys, timeoutMs: 100 }); + expect(requests).to.have.length(1 + keys.length); + }); + + it("keeps going past HTTP errors, which concern only one key", async () => { + const keys = seed(3); + respond = (req, res) => res.writeHead(req.url?.includes(keys[0]) ? 404 : 202).end(); + await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: keys }); + expect(requests).to.have.length(keys.length); + }); + + it("stops at the first refused connection", async () => { + const keys = seed(3); + const { port } = server.address() as AddressInfo; + server.close(); + await once(server, "close"); + const closedApi = new ValidatorApi({ baseUrl: `http://127.0.0.1:${port}` }, Network.Mainnet); + const spy = sinon.spy(closedApi, "setFeeRecipient"); + await postValidatorsFeeRecipientsFromDb({ validatorApi: closedApi, brainDb, pubkeys: keys }); + expect(spy.callCount).to.equal(1); + server.listen(0, "127.0.0.1"); // let afterEach close it cleanly + await once(server, "listening"); + }); + }); +}); From 3b7fe6443433c694f0e315f26ead863cc9107e84 Mon Sep 17 00:00:00 2001 From: 3alpha <15694175+3alpha@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:29:27 +0100 Subject: [PATCH 2/2] fix: skip builder configuration when the validator client stops responding When the fee recipient step stopped because the validator client gave no response, reload still applied every builder configuration, each waiting up to 10s. With the builder feature enabled this could delay the next pass for hours on large nodes. The slice helper now reports whether a step stopped the run. Reload skips the builder phase when the fee step stopped, and the builder step itself stops at the first request without a response. Builder errors keep a cause with only the HTTP status, never the response body, so echoed authorization data stays out of logs. --- .../src/modules/apiClients/validator/index.ts | 5 +- .../reloadValidators/forEachInLockedSlices.ts | 5 +- .../modules/cron/reloadValidators/index.ts | 11 ++-- .../postValidatorsBuilderConfigsFromDb.ts | 28 ++++++--- .../postValidatorsFeeRecipientsFromDb.ts | 6 +- .../modules/reloadLockSlices.unit.test.ts | 63 ++++++++++++++++++- 6 files changed, 100 insertions(+), 18 deletions(-) diff --git a/packages/brain/src/modules/apiClients/validator/index.ts b/packages/brain/src/modules/apiClients/validator/index.ts index e9d7f5d1..cafe1da7 100644 --- a/packages/brain/src/modules/apiClients/validator/index.ts +++ b/packages/brain/src/modules/apiClients/validator/index.ts @@ -85,7 +85,10 @@ export class ValidatorApi extends StandardApi { error instanceof HttpApiError ? `HTTP ${error.statusCode}${[405, 501].includes(error.statusCode) ? " (builder configuration is unsupported)" : error.statusCode === 404 ? " (endpoint unavailable or validator key not found)" : ""}` : error.message; - return new ValidatorApiError(`Error ${action} builder configuration for ${pubkey}: ${reason}`); + // Keep whether the client answered (for reload to stop on an unreachable client), never the body. + const cause = + error instanceof HttpApiError ? new HttpApiError(error.statusCode, `HTTP ${error.statusCode}`) : error; + return new ValidatorApiError(`Error ${action} builder configuration for ${pubkey}: ${reason}`, { cause }); } /** diff --git a/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts b/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts index 27017489..05fe908b 100644 --- a/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts +++ b/packages/brain/src/modules/cron/reloadValidators/forEachInLockedSlices.ts @@ -15,13 +15,15 @@ export type SliceStep = (pubkey: string, details: PubkeyDetails) => Promise<"con * between. Every slice re-reads the database inside the lock, so it always applies the settings stored * now, never a value a user operation has since replaced, and skips keys deleted meanwhile. A step that * returns "stop" ends the whole run; the next reload pass starts over. + * + * @returns "stopped" when a step stopped the run, so callers can skip later steps against the same client */ export async function forEachInLockedSlices( brainDb: BrainDataBase, pubkeys: readonly string[], step: SliceStep, { maxHoldMs = RELOAD_SLICE_MAX_HOLD_MS }: { maxHoldMs?: number } = {} -): Promise { +): Promise<"completed" | "stopped"> { let next = 0; let stopped = false; while (next < pubkeys.length && !stopped) { @@ -37,4 +39,5 @@ export async function forEachInLockedSlices( } while (next < pubkeys.length && !stopped && Date.now() < deadline); }); } + return stopped ? "stopped" : "completed"; } diff --git a/packages/brain/src/modules/cron/reloadValidators/index.ts b/packages/brain/src/modules/cron/reloadValidators/index.ts index 086baae1..ac769cc9 100644 --- a/packages/brain/src/modules/cron/reloadValidators/index.ts +++ b/packages/brain/src/modules/cron/reloadValidators/index.ts @@ -48,11 +48,14 @@ export async function reloadValidators( // 6. POST to validator API fee recipients that are in DB in validator API. One request per key, so // it holds the lock in short slices: a long pass on a large node must not lock out user operations. - await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys }); + const fees = await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys }); - // Independently retry managed configuration after key imports and fee destinations. - if (featureFlags.builderConfig) - await postValidatorsBuilderConfigsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys }); + // Independently retry managed configuration after key imports and fee destinations. Skip it when the + // client stopped responding: every request would wait for its timeout and delay the next pass. + if (featureFlags.builderConfig) { + if (fees === "stopped") logger.debug(`${logPrefix}Skipping builder configuration: validator client unavailable`); + else await postValidatorsBuilderConfigsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys }); + } // Repairs such as restored fee recipients change no database record. If an open UI last saw a // problem, announce the pass so it lists again (and checks the fee recipients by GET). if (takeListRecheck(brainDb)) brainDb.notifyChanged(); diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts index 49cfbea7..017d65e2 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.ts @@ -1,5 +1,6 @@ import { BuilderConfig } from "@stakingbrain/common"; import { ValidatorApi } from "../../apiClients/index.js"; +import { receivedHttpResponse } from "../../apiClients/error.js"; import { BrainDataBase } from "../../db/index.js"; import logger from "../../logger/index.js"; import { logPrefix } from "./logPrefix.js"; @@ -14,7 +15,10 @@ export async function applyValidatorBuilderConfig( else if (config !== undefined) await validatorApi.setBuilderConfig(config, pubkey); } -/** Replay original overrides, including reset intent. Never copy resolved GET values. */ +/** + * Replay original overrides, including reset intent. Never copy resolved GET values. + * Like fee recipients, an HTTP error concerns one key; no response at all stops the step. + */ export async function postValidatorsBuilderConfigsFromDb({ validatorApi, brainDb, @@ -23,11 +27,21 @@ export async function postValidatorsBuilderConfigsFromDb({ validatorApi: ValidatorApi; brainDb: BrainDataBase; pubkeys: readonly string[]; -}): Promise { - await forEachInLockedSlices(brainDb, pubkeys, async (pubkey, { builderConfig }) => { - await applyValidatorBuilderConfig(validatorApi, pubkey, builderConfig).catch((error) => - logger.error(`${logPrefix}Error applying builder configuration for ${pubkey}`, error) - ); - return "continue"; +}): Promise<"completed" | "stopped"> { + return forEachInLockedSlices(brainDb, pubkeys, async (pubkey, { builderConfig }) => { + try { + await applyValidatorBuilderConfig(validatorApi, pubkey, builderConfig); + return "continue"; + } catch (error) { + if (!receivedHttpResponse(error)) { + logger.error( + `${logPrefix}Validator client did not respond while applying builder configuration; retrying next pass`, + error + ); + return "stop"; + } + logger.error(`${logPrefix}Error applying builder configuration for ${pubkey}`, error); + return "continue"; + } }); } diff --git a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts index 631a3575..e38e3d5b 100644 --- a/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts +++ b/packages/brain/src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.ts @@ -14,6 +14,8 @@ export const FEE_RECIPIENT_TIMEOUT_MS = 10 * 1000; * An HTTP error concerns one key, so the pass carries on with the rest. A request that gets no response * (timeout, refused or reset connection) means the client is unavailable: stop, rather than wait on every * remaining key, and let the next pass start over. + * + * @returns "stopped" when the client did not respond, so reload skips its other per-key steps */ export async function postValidatorsFeeRecipientsFromDb({ validatorApi, @@ -27,8 +29,8 @@ export async function postValidatorsFeeRecipientsFromDb({ pubkeys: readonly string[]; timeoutMs?: number; maxHoldMs?: number; -}): Promise { - await forEachInLockedSlices( +}): Promise<"completed" | "stopped"> { + return forEachInLockedSlices( brainDb, pubkeys, async (pubkey, { feeRecipient }) => { diff --git a/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts b/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts index 6a238c8a..74d5a4a5 100644 --- a/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts +++ b/packages/brain/test/unit/modules/reloadLockSlices.unit.test.ts @@ -6,30 +6,35 @@ import path from "node:path"; import http from "node:http"; import type { AddressInfo } from "node:net"; import { once } from "node:events"; -import { Network } from "@stakingbrain/common"; +import { inspect } from "node:util"; +import { BuilderConfig, featureFlags, Network } from "@stakingbrain/common"; import { BrainDataBase } from "../../../src/modules/db/index.js"; import { CronJob } from "../../../src/modules/cron/cron.js"; import { ValidatorApi, Web3SignerApi } from "../../../src/modules/apiClients/index.js"; +import { ApiError } from "../../../src/modules/apiClients/error.js"; +import { ValidatorApiError } from "../../../src/modules/apiClients/validator/error.js"; import { updateValidators } from "../../../src/modules/apiServers/ui/calls/updateValidators.js"; import { reloadValidators } from "../../../src/modules/cron/reloadValidators/index.js"; import { postValidatorsFeeRecipientsFromDb } from "../../../src/modules/cron/reloadValidators/postValidatorsFeeRecipientsFromDb.js"; +import { postValidatorsBuilderConfigsFromDb } from "../../../src/modules/cron/reloadValidators/postValidatorsBuilderConfigsFromDb.js"; import { RELOAD_SLICE_MAX_HOLD_MS } from "../../../src/modules/cron/reloadValidators/forEachInLockedSlices.js"; import logger from "../../../src/modules/logger/index.js"; const pubkeyAt = (i: number) => "0x" + i.toString(16).padStart(96, "0"); const recipient = "0x" + "ab".repeat(20); const newRecipient = "0x" + "cd".repeat(20); +const builderConfig: BuilderConfig = { builders: [{ url: "https://builder.example" }] }; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); describe("Reload holds the validator operation lock in short slices", () => { let directory: string; let brainDb: BrainDataBase; - const seed = (count: number): string[] => { + const seed = (count: number, extra: { builderConfig?: BuilderConfig } = {}): string[] => { const keys = Array.from({ length: count }, (_, i) => pubkeyAt(i + 1)); brainDb.addValidators({ validators: Object.fromEntries( - keys.map((key) => [key, { tag: "solo" as const, feeRecipient: recipient, automaticImport: false }]) + keys.map((key) => [key, { tag: "solo" as const, feeRecipient: recipient, automaticImport: false, ...extra }]) ) }); return keys; @@ -99,6 +104,33 @@ describe("Reload holds the validator operation lock in short slices", () => { expect(validatorApi.setFeeRecipient.getCalls().map((call) => call.args[1])).to.deep.equal(keys.slice(0, 2)); }); + it("skips builder configuration when the validator client stopped responding to fee recipients", async () => { + const keys = seed(3, { builderConfig }); + const signerApi = sinon.createStubInstance(Web3SignerApi); + const validatorApi = sinon.createStubInstance(ValidatorApi); + signerApi.upcheck.resolves(); + signerApi.listRemoteKeys.resolves({ + data: keys.map((validating_pubkey) => ({ validating_pubkey, derivation_path: "", readonly: false })) + }); + validatorApi.getRemoteKeys.resolves({ data: keys.map((pubkey) => ({ pubkey, readonly: false, url: "" })) }); + // What the real client throws when a request gets no response. + validatorApi.setFeeRecipient.rejects(new ValidatorApiError("fee", { cause: new ApiError("timed out") })); + validatorApi.setBuilderConfig.resolves(); + featureFlags.builderConfig = true; + try { + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(validatorApi.setFeeRecipient.callCount).to.equal(1); + expect(validatorApi.setBuilderConfig.called).to.equal(false); + + // A responsive client gets both phases again on the next pass. + validatorApi.setFeeRecipient.resolves(); + await reloadValidators(signerApi, "http://signer", validatorApi, brainDb); + expect(validatorApi.setBuilderConfig.callCount).to.equal(keys.length); + } finally { + featureFlags.builderConfig = false; + } + }); + describe("against a real validator client connection", () => { let server: http.Server; let validatorApi: ValidatorApi; @@ -155,5 +187,30 @@ describe("Reload holds the validator operation lock in short slices", () => { server.listen(0, "127.0.0.1"); // let afterEach close it cleanly await once(server, "listening"); }); + + it("stops builder configuration at the first refused connection", async () => { + const keys = seed(3, { builderConfig }); + const { port } = server.address() as AddressInfo; + server.close(); + await once(server, "close"); + const closedApi = new ValidatorApi({ baseUrl: `http://127.0.0.1:${port}` }, Network.Mainnet); + const spy = sinon.spy(closedApi, "setBuilderConfig"); + expect(await postValidatorsBuilderConfigsFromDb({ validatorApi: closedApi, brainDb, pubkeys: keys })).to.equal( + "stopped" + ); + expect(spy.callCount).to.equal(1); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + }); + + it("keeps applying builder configuration past HTTP errors without logging response bodies", async () => { + const keys = seed(3, { builderConfig }); + respond = (req, res) => + req.url?.includes(keys[0]) ? res.writeHead(500).end("echoed-secret-authorization") : res.writeHead(200).end(); + expect(await postValidatorsBuilderConfigsFromDb({ validatorApi, brainDb, pubkeys: keys })).to.equal("completed"); + expect(requests).to.have.length(keys.length); + const logged = (logger.error as sinon.SinonStub).getCalls().map((call) => inspect(call.args, { depth: 10 })); + expect(logged.join("\n")).to.include("HTTP 500").and.not.include("echoed-secret"); + }); }); });