Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/brain/src/createErrorFactory.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
export function createErrorFactory(errorName: string) {
return class CustomError extends Error {
constructor(message: string) {
super(message);
constructor(message: string, options?: ErrorOptions) {
super(message, options);
this.name = errorName; // Set the error name provided to the factory
Error.captureStackTrace(this, CustomError); // Capture the stack trace and omit the constructor frame
}
Expand Down
6 changes: 6 additions & 0 deletions packages/brain/src/modules/apiClients/error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,9 @@ export class HttpApiError extends ApiError {
super(message);
}
}

/** True when a request got an HTTP response, i.e. the server is reachable even if it refused the request. */
export function receivedHttpResponse(error: unknown): boolean {
for (let e = error; e instanceof Error; e = e.cause) if (e instanceof HttpApiError) return true;
return false;
}
4 changes: 2 additions & 2 deletions packages/brain/src/modules/apiClients/validator/error.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { ApiError } from "../error.js";

export class ValidatorApiError extends ApiError {
constructor(message: string) {
super(message);
constructor(message: string, options?: ErrorOptions) {
super(message, options);
this.name = "ValidatorApiError"; // Override the name if needed
}
}
17 changes: 13 additions & 4 deletions packages/brain/src/modules/apiClients/validator/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,10 @@ export class ValidatorApi extends StandardApi {
error instanceof HttpApiError
? `HTTP ${error.statusCode}${[405, 501].includes(error.statusCode) ? " (builder configuration is unsupported)" : error.statusCode === 404 ? " (endpoint unavailable or validator key not found)" : ""}`
: error.message;
return new ValidatorApiError(`Error ${action} builder configuration for ${pubkey}: ${reason}`);
// Keep whether the client answered (for reload to stop on an unreachable client), never the body.
const cause =
error instanceof HttpApiError ? new HttpApiError(error.statusCode, `HTTP ${error.statusCode}`) : error;
return new ValidatorApiError(`Error ${action} builder configuration for ${pubkey}: ${reason}`, { cause });
}

/**
Expand Down Expand Up @@ -120,16 +123,22 @@ export class ValidatorApi extends StandardApi {
* Sets the validator client fee recipient mapping which will then update the beacon node..
* https://ethereum.github.io/keymanager-APIs/#/Fee%20Recipient/setFeeRecipient
*/
public async setFeeRecipient(newFeeRecipient: string, publicKey: string): Promise<void> {
public async setFeeRecipient(
newFeeRecipient: string,
publicKey: string,
{ timeout }: { timeout?: number } = {}
): Promise<void> {
try {
await this.request({
method: "POST",
endpoint: path.join(this.validatorEndpoint, prefix0xPubkey(publicKey), "feerecipient"),
body: JSON.stringify({ ethaddress: newFeeRecipient })
body: JSON.stringify({ ethaddress: newFeeRecipient }),
timeout
});
} catch (e) {
throw new ValidatorApiError(
`Error setting fee recipient for pubkey ${publicKey} to ${newFeeRecipient} on validator. ${e.message}`
`Error setting fee recipient for pubkey ${publicKey} to ${newFeeRecipient} on validator. ${e.message}`,
{ cause: e }
);
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import type { BrainDataBase } from "../../db/index.js";
import type { PubkeyDetails } from "../../db/types.js";
import { withValidatorOperationLock } from "../../db/withValidatorOperationLock.js";

/**
* How long one slice of a per-key reload step may hold the validator operation lock. User operations
* queued meanwhile wait for at most this plus one in-flight client request, far below their wait budget.
*/
export const RELOAD_SLICE_MAX_HOLD_MS = 1000;

export type SliceStep = (pubkey: string, details: PubkeyDetails) => Promise<"continue" | "stop">;

/**
* Apply `step` to each key, holding the lock only for short time slices so user operations can run in
* between. Every slice re-reads the database inside the lock, so it always applies the settings stored
* now, never a value a user operation has since replaced, and skips keys deleted meanwhile. A step that
* returns "stop" ends the whole run; the next reload pass starts over.
*
* @returns "stopped" when a step stopped the run, so callers can skip later steps against the same client
*/
export async function forEachInLockedSlices(
brainDb: BrainDataBase,
pubkeys: readonly string[],
step: SliceStep,
{ maxHoldMs = RELOAD_SLICE_MAX_HOLD_MS }: { maxHoldMs?: number } = {}
): Promise<"completed" | "stopped"> {
let next = 0;
let stopped = false;
while (next < pubkeys.length && !stopped) {
await withValidatorOperationLock(brainDb, async () => {
const data = brainDb.getData();
const deadline = Date.now() + maxHoldMs;
// At least one key per slice, so every slice makes progress.
do {
const pubkey = pubkeys[next++];
const details = data[pubkey];
if (!details || details.importPending) continue;
if ((await step(pubkey, details)) === "stop") stopped = true;
} while (next < pubkeys.length && !stopped && Date.now() < deadline);
});
}
return stopped ? "stopped" : "completed";
}
130 changes: 72 additions & 58 deletions packages/brain/src/modules/cron/reloadValidators/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import { postValidatorsBuilderConfigsFromDb } from "./postValidatorsBuilderConfi
* - DELETE to validator API pubkeys that are in validator API and not in DB
* - POST to validator API fee recipients that are in DB and not in validator API
*
* Key membership (steps 0-5) is reconciled under one lock; per-key settings are then applied in short
* locked slices that re-read the database, so user operations never wait for a whole pass.
*/
/** Steps that tidy up rather than configure: log a failure and carry on with the pass. */
async function continueOnFailure(step: string, run: () => Promise<void>): Promise<void> {
Expand All @@ -37,72 +39,84 @@ export async function reloadValidators(
validatorApi: ValidatorApi,
brainDb: BrainDataBase
): Promise<void> {
return withValidatorOperationLock(brainDb, async () => {
try {
logger.debug(`${logPrefix}Reloading data...`);
try {
logger.debug(`${logPrefix}Reloading data...`);
// Steps 0-5 reconcile key membership with a few bulk calls under one lock.
const activePubkeys = await withValidatorOperationLock(brainDb, () =>
syncValidatorKeys(signerApi, signerUrl, validatorApi, brainDb)
);

// 0. Check signer API upcheck endpoint
await signerApi.upcheck();
// 6. POST to validator API fee recipients that are in DB in validator API. One request per key, so
// it holds the lock in short slices: a long pass on a large node must not lock out user operations.
const fees = await postValidatorsFeeRecipientsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys });

// 1. GET data
const dbPubkeys = Object.keys(brainDb.getData());
const signerPubkeys = (await signerApi.listRemoteKeys()).data.map((keystore) => keystore.validating_pubkey);
// Independently retry managed configuration after key imports and fee destinations. Skip it when the
// client stopped responding: every request would wait for its timeout and delay the next pass.
if (featureFlags.builderConfig) {
if (fees === "stopped") logger.debug(`${logPrefix}Skipping builder configuration: validator client unavailable`);
else await postValidatorsBuilderConfigsFromDb({ validatorApi, brainDb, pubkeys: activePubkeys });
}
// Repairs such as restored fee recipients change no database record. If an open UI last saw a
// problem, announce the pass so it lists again (and checks the fee recipients by GET).
if (takeListRecheck(brainDb)) brainDb.notifyChanged();
logger.debug(`${logPrefix}Finished reloading data`);
} catch (e) {
logger.error(`${logPrefix}Error reloading data`, e);
}
}

// 2. DELETE from signer API pubkeys that are not in DB. Keys whose deletion fails stay listed, so
// the database keeps them and the next pass retries.
await continueOnFailure("Deleting signer keys not in the database", () =>
deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys })
);
/** Steps 0-5. Returns the keys the signer holds, in database spelling, for per-key configuration. */
async function syncValidatorKeys(
signerApi: Web3SignerApi,
signerUrl: string,
validatorApi: ValidatorApi,
brainDb: BrainDataBase
): Promise<string[]> {
// 0. Check signer API upcheck endpoint
await signerApi.upcheck();

// 3. DELETE from DB pubkeys that are not in signer API
await deleteDbPubkeysNotInSigner({ brainDb, dbPubkeys, signerPubkeys });
// 1. GET data
const dbPubkeys = Object.keys(brainDb.getData());
const signerPubkeys = (await signerApi.listRemoteKeys()).data.map((keystore) => keystore.validating_pubkey);

const signerKeys = new Set(signerPubkeys.map((pubkey) => pubkey.toLowerCase()));
const data = brainDb.getData();
const confirmedImports = dbPubkeys.filter(
(pubkey) => data[pubkey].importPending && signerKeys.has(pubkey.toLowerCase())
);
if (confirmedImports.length > 0)
brainDb.updateValidators(confirmedImports.map((pubkey) => ({ pubkey, importPending: undefined })));
// Preserve pending records, but do not activate keys the signer has not confirmed.
const activeData = Object.fromEntries(
Object.entries(brainDb.getData()).filter(([pubkey]) => signerKeys.has(pubkey.toLowerCase()))
);
// 2. DELETE from signer API pubkeys that are not in DB. Keys whose deletion fails stay listed, so
// the database keeps them and the next pass retries.
await continueOnFailure("Deleting signer keys not in the database", () =>
deleteSignerPubkeysNotInDb({ signerApi, signerPubkeys, dbPubkeys })
);

const validatorPubkeys = (await validatorApi.getRemoteKeys()).data.map((keystore) => keystore.pubkey);
const validatorKeys = new Set(validatorPubkeys.map((pubkey) => pubkey.toLowerCase()));
const dbKeys = new Set(dbPubkeys.map((pubkey) => pubkey.toLowerCase()));
// 3. DELETE from DB pubkeys that are not in signer API
await deleteDbPubkeysNotInSigner({ brainDb, dbPubkeys, signerPubkeys });

// 4. POST to validator API pubkeys that are in DB and not in validator API
const brainDbPubkeysToAdd = Object.keys(activeData).filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase()));
await continueOnFailure("Adding keys to the validator client", () =>
postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys })
);
const signerKeys = new Set(signerPubkeys.map((pubkey) => pubkey.toLowerCase()));
const data = brainDb.getData();
const confirmedImports = dbPubkeys.filter(
(pubkey) => data[pubkey]?.importPending && signerKeys.has(pubkey.toLowerCase())
);
if (confirmedImports.length > 0)
brainDb.updateValidators(confirmedImports.map((pubkey) => ({ pubkey, importPending: undefined })));
// Preserve pending records, but do not activate keys the signer has not confirmed.
const activePubkeys = Object.keys(brainDb.getData()).filter((pubkey) => signerKeys.has(pubkey.toLowerCase()));

// 5. DELETE to validator API pubkeys that are in validator API and not in DB
const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase()));
// Cleanup must not stop desired validators from being configured; failed removals stay listed and
// are retried next pass.
await continueOnFailure("Removing obsolete keys from the validator client", () =>
deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove })
);
// Client membership is shown in the UI; repairs here change it without touching the database.
if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged();
const validatorPubkeys = (await validatorApi.getRemoteKeys()).data.map((keystore) => keystore.pubkey);
const validatorKeys = new Set(validatorPubkeys.map((pubkey) => pubkey.toLowerCase()));
const dbKeys = new Set(dbPubkeys.map((pubkey) => pubkey.toLowerCase()));

// 6. POST to validator API fee recipients that are in DB in validator API
await postValidatorsFeeRecipientsFromDb({
validatorApi,
dbData: activeData
});
// 4. POST to validator API pubkeys that are in DB and not in validator API
const brainDbPubkeysToAdd = activePubkeys.filter((pubkey) => !validatorKeys.has(pubkey.toLowerCase()));
await continueOnFailure("Adding keys to the validator client", () =>
postValidatorPubkeysFromDb({ validatorApi, signerUrl, brainDbPubkeysToAdd, validatorPubkeys })
);

// Independently retry managed configuration after key imports and fee destinations.
if (featureFlags.builderConfig) await postValidatorsBuilderConfigsFromDb({ validatorApi, dbData: activeData });
// Repairs such as restored fee recipients change no database record. If an open UI last saw a
// problem, announce the pass so it lists again (and checks the fee recipients by GET).
if (takeListRecheck(brainDb)) brainDb.notifyChanged();
logger.debug(`${logPrefix}Finished reloading data`);
} catch (e) {
logger.error(`${logPrefix}Error reloading data`, e);
}
});
// 5. DELETE to validator API pubkeys that are in validator API and not in DB
const validatorPubkeysToRemove = validatorPubkeys.filter((pubkey) => !dbKeys.has(pubkey.toLowerCase()));
// Cleanup must not stop desired validators from being configured; failed removals stay listed and
// are retried next pass.
await continueOnFailure("Removing obsolete keys from the validator client", () =>
deleteValidatorPubkeysNotInDB({ validatorApi, validatorPubkeys, validatorPubkeysToRemove })
);
// Client membership is shown in the UI; repairs here change it without touching the database.
if (brainDbPubkeysToAdd.length > 0 || validatorPubkeysToRemove.length > 0) brainDb.notifyChanged();

return activePubkeys;
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { BuilderConfig } from "@stakingbrain/common";
import { ValidatorApi } from "../../apiClients/index.js";
import { StakingBrainDb } from "../../db/types.js";
import { receivedHttpResponse } from "../../apiClients/error.js";
import { BrainDataBase } from "../../db/index.js";
import logger from "../../logger/index.js";
import { logPrefix } from "./logPrefix.js";
import { forEachInLockedSlices } from "./forEachInLockedSlices.js";

export async function applyValidatorBuilderConfig(
validatorApi: ValidatorApi,
Expand All @@ -13,17 +15,33 @@ export async function applyValidatorBuilderConfig(
else if (config !== undefined) await validatorApi.setBuilderConfig(config, pubkey);
}

/** Replay original overrides, including reset intent. Never copy resolved GET values. */
/**
* Replay original overrides, including reset intent. Never copy resolved GET values.
* Like fee recipients, an HTTP error concerns one key; no response at all stops the step.
*/
export async function postValidatorsBuilderConfigsFromDb({
validatorApi,
dbData
brainDb,
pubkeys
}: {
validatorApi: ValidatorApi;
dbData: StakingBrainDb;
}): Promise<void> {
for (const [pubkey, { builderConfig }] of Object.entries(dbData)) {
await applyValidatorBuilderConfig(validatorApi, pubkey, builderConfig).catch((error) =>
logger.error(`${logPrefix}Error applying builder configuration for ${pubkey}`, error)
);
}
brainDb: BrainDataBase;
pubkeys: readonly string[];
}): Promise<"completed" | "stopped"> {
return forEachInLockedSlices(brainDb, pubkeys, async (pubkey, { builderConfig }) => {
try {
await applyValidatorBuilderConfig(validatorApi, pubkey, builderConfig);
return "continue";
} catch (error) {
if (!receivedHttpResponse(error)) {
logger.error(
`${logPrefix}Validator client did not respond while applying builder configuration; retrying next pass`,
error
);
return "stop";
}
logger.error(`${logPrefix}Error applying builder configuration for ${pubkey}`, error);
return "continue";
}
});
}
Original file line number Diff line number Diff line change
@@ -1,25 +1,56 @@
import { ValidatorApi } from "../../apiClients/index.js";
import { StakingBrainDb } from "../../db/types.js";
import { receivedHttpResponse } from "../../apiClients/error.js";
import { BrainDataBase } from "../../db/index.js";
import logger from "../../logger/index.js";
import { logPrefix } from "./logPrefix.js";
import { forEachInLockedSlices } from "./forEachInLockedSlices.js";

/** A healthy client answers a fee recipient write quickly; a hung one must not stall the pass for long. */
export const FEE_RECIPIENT_TIMEOUT_MS = 10 * 1000;

/**
* Post in the validator API fee recipients that are in the DB in the validator API
* Post to the validator client the fee recipient stored for each key.
*
* An HTTP error concerns one key, so the pass carries on with the rest. A request that gets no response
* (timeout, refused or reset connection) means the client is unavailable: stop, rather than wait on every
* remaining key, and let the next pass start over.
*
* @returns "stopped" when the client did not respond, so reload skips its other per-key steps
*/
export async function postValidatorsFeeRecipientsFromDb({
validatorApi,
dbData
brainDb,
pubkeys,
timeoutMs = FEE_RECIPIENT_TIMEOUT_MS,
maxHoldMs
}: {
validatorApi: ValidatorApi;
dbData: StakingBrainDb;
}): Promise<void> {
for (const [pubkey, { feeRecipient }] of Object.entries(dbData)) {
await validatorApi
.setFeeRecipient(feeRecipient, pubkey)
.catch((e) =>
brainDb: BrainDataBase;
pubkeys: readonly string[];
timeoutMs?: number;
maxHoldMs?: number;
}): Promise<"completed" | "stopped"> {
return forEachInLockedSlices(
brainDb,
pubkeys,
async (pubkey, { feeRecipient }) => {
try {
await validatorApi.setFeeRecipient(feeRecipient, pubkey, { timeout: timeoutMs });
return "continue";
} catch (e) {
if (!receivedHttpResponse(e)) {
logger.error(
`${logPrefix}Validator client did not respond while posting fee recipients; retrying all keys next pass`,
e
);
return "stop";
}
logger.error(
`${logPrefix}Error posting fee recipient ${feeRecipient} for pubkey ${pubkey} to validator API: ${e}`
)
);
}
);
return "continue";
}
},
{ maxHoldMs }
);
}
Loading
Loading