diff --git a/packages/grpc-js/src/internal-channel.ts b/packages/grpc-js/src/internal-channel.ts index f42fe37e8..036591ca2 100644 --- a/packages/grpc-js/src/internal-channel.ts +++ b/packages/grpc-js/src/internal-channel.ts @@ -863,7 +863,11 @@ export class InternalChannel { getConnectivityState(tryToConnect: boolean) { const connectivityState = this.connectivityState; - if (tryToConnect) { + /* close() reports SHUTDOWN before returning, and it does so + * synchronously. Watchers such as Client.waitForReady call back into + * getConnectivityState(true) from that notification. exitIdle() after + * destroy() would start a new resolver on a closed channel (#3128). */ + if (tryToConnect && connectivityState !== ConnectivityState.SHUTDOWN) { this.resolvingLoadBalancer.exitIdle(); this.lastActivityTimestamp = Date.now(); this.maybeStartIdleTimer(); diff --git a/packages/grpc-js/src/resolver-dns.ts b/packages/grpc-js/src/resolver-dns.ts index 5245fe126..027a1e8d0 100644 --- a/packages/grpc-js/src/resolver-dns.ts +++ b/packages/grpc-js/src/resolver-dns.ts @@ -68,6 +68,12 @@ class DnsResolver implements Resolver { private defaultResolutionError: StatusObject; private backoff: BackoffTimeout; private continueResolving = false; + /** + * Set false by destroy(). updateResolution() sets it true again. + * In-flight lookups that settle after destroy, including a listener + * that destroys the resolver, must not arm another attempt. + */ + private resolutionEnabled = true; private nextResolutionTimer: NodeJS.Timeout; private isNextResolutionTimerRunning = false; private isServiceConfigEnabled = true; @@ -189,10 +195,14 @@ class DnsResolver implements Resolver { * revert to an effectively blank one. */ this.latestLookupResult = null; const hostname: string = this.dnsHostname; - this.pendingLookupPromise = this.lookup(hostname); - this.pendingLookupPromise.then( + const lookupPromise = this.lookup(hostname); + this.pendingLookupPromise = lookupPromise; + lookupPromise.then( addressList => { - if (this.pendingLookupPromise === null) { + /* Compare identity, not just null. destroy() clears the field, + * and a later updateResolution() may already have stored a new + * promise. A superseded lookup must not report or retry. */ + if (this.pendingLookupPromise !== lookupPromise) { return; } this.pendingLookupPromise = null; @@ -222,7 +232,7 @@ class DnsResolver implements Resolver { this.handleHealthStatus(healthStatus); }, err => { - if (this.pendingLookupPromise === null) { + if (this.pendingLookupPromise !== lookupPromise) { return; } trace( @@ -232,13 +242,16 @@ class DnsResolver implements Resolver { (err as Error).message ); this.pendingLookupPromise = null; + /* The min-interval timer applies only to successful resolutions. + * Leaving it armed here would race the backoff timer. */ this.stopNextResolutionTimer(); - this.listener( + const healthStatus = this.listener( statusOrFromError(this.defaultResolutionError), {}, this.latestServiceConfigResult, '' - ) + ); + this.onLookupFailure(healthStatus); } ); /* If there already is a still-pending TXT resolution, we can just use @@ -311,8 +324,53 @@ class DnsResolver implements Resolver { this.backoff.stop(); this.backoff.reset(); } else { + this.requestAnotherResolution(); + } + } + + /** + * A lookup rejection is a resolution failure even when the listener + * accepts it. pick_first accepts an error endpoint list by entering + * TRANSIENT_FAILURE and returning true. handleHealthStatus(true) would + * stop the backoff, so the name would never be tried again (#3128). + * A false result uses the same retry as a rejected successful result. + * An unparseable target never reaches this path. + */ + private onLookupFailure(healthStatus: boolean) { + if (!this.resolutionEnabled) { + return; + } + trace( + 'Retrying failed DNS resolution for target ' + + uriToString(this.target) + + ' (listener accepted=' + + healthStatus + + ')' + ); + if (healthStatus) { + this.requestAnotherResolution(); + } else { + this.handleHealthStatus(healthStatus); + } + } + + /** + * Schedule exactly one follow-up attempt on the backoff timer that + * startResolutionWithBackoff already armed. If that timer already + * fired (the lookup outlived it), start the next attempt now. A second + * call while a timer or a lookup is in flight only sets + * continueResolving, so repeated updateResolution calls cannot pile up + * timers (#2080 / #2098). + */ + private requestAnotherResolution() { + if (!this.resolutionEnabled || this.pendingLookupPromise !== null) { + return; + } + if (this.backoff.isRunning() || this.isNextResolutionTimerRunning) { this.continueResolving = true; + return; } + this.startResolutionWithBackoff(); } private async lookup(hostname: string): Promise { @@ -375,15 +433,20 @@ class DnsResolver implements Resolver { } private startResolutionWithBackoff() { - if (this.pendingLookupPromise === null) { - this.continueResolving = false; - this.backoff.runOnce(); - this.startNextResolutionTimer(); - this.startResolution(); + if (!this.resolutionEnabled || this.pendingLookupPromise !== null) { + return; } + /* Clear per attempt so a timer that fires during this attempt does + * not start another one unless a new request arrives (#2098). */ + this.continueResolving = false; + this.backoff.runOnce(); + this.startNextResolutionTimer(); + this.startResolution(); } updateResolution() { + /* destroy() is a reset. The next request must be able to resolve. */ + this.resolutionEnabled = true; /* If there is a pending lookup, just let it finish. Otherwise, if the * nextResolutionTimer or backoff timer is running, set the * continueResolving flag to resolve when whichever of those timers @@ -413,6 +476,7 @@ class DnsResolver implements Resolver { * will be ignored. */ destroy() { + this.resolutionEnabled = false; this.continueResolving = false; this.backoff.reset(); this.backoff.stop(); diff --git a/packages/grpc-js/test/test-dns-resolver-retry.ts b/packages/grpc-js/test/test-dns-resolver-retry.ts new file mode 100644 index 000000000..92a8f3a93 --- /dev/null +++ b/packages/grpc-js/test/test-dns-resolver-retry.ts @@ -0,0 +1,332 @@ +/* + * Copyright 2026 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import * as assert from 'assert'; +import { promises as dnsPromises } from 'dns'; +import * as grpc from '../src'; +import { ConnectivityState } from '../src/connectivity-state'; +import { TestClient, TestServer } from './common'; + +const HOST = 'late.invalid'; + +/** + * Controllable stand-in for dns.promises.lookup. Offline and deterministic. + * The resolver imports that same promises object and reads `.lookup` at + * call time, so replacing the method is enough. + */ +class LookupController { + fail = true; + lookupCount = 0; + inFlight = 0; + maxInFlight = 0; + private original = dnsPromises.lookup; + + install() { + dnsPromises.lookup = ((hostname: string) => { + this.lookupCount += 1; + this.inFlight += 1; + this.maxInFlight = Math.max(this.maxInFlight, this.inFlight); + return new Promise((resolve, reject) => { + setImmediate(() => { + this.inFlight -= 1; + if (this.fail) { + reject(new Error(`ENOTFOUND ${hostname}`)); + return; + } + resolve([{ address: '127.0.0.1', family: 4 }]); + }); + }); + }) as typeof dnsPromises.lookup; + } + + restore() { + dnsPromises.lookup = this.original; + } + + waitForCount(count: number, timeoutMs: number): Promise { + const start = Date.now(); + return new Promise((resolve, reject) => { + const check = () => { + if (this.lookupCount >= count) { + resolve(); + return; + } + if (Date.now() - start > timeoutMs) { + reject( + new Error( + `timed out waiting for ${count} lookups, saw ${this.lookupCount}` + ) + ); + return; + } + setTimeout(check, 10); + }; + check(); + }); + } +} + +const channelOptions = { + 'grpc.initial_reconnect_backoff_ms': 20, + 'grpc.max_reconnect_backoff_ms': 40, + 'grpc.service_config_disable_resolution': 1, + 'grpc.enable_http_proxy': 0, + 'grpc.enable_channelz': 0, +}; + +describe('DNS resolver retry on failure', () => { + let server: TestServer; + let port: number; + let controller: LookupController; + let client: TestClient | null = null; + + before(async () => { + server = new TestServer(false); + await server.start(); + port = Number(server.getTarget().split(':')[1]); + }); + + beforeEach(() => { + controller = new LookupController(); + controller.install(); + }); + + afterEach(() => { + client?.close(); + client = null; + controller.restore(); + }); + + after(() => { + server.shutdown(); + }); + + function target(): string { + return `dns:${HOST}:${port}`; + } + + function openClient(extra?: grpc.ChannelOptions): TestClient { + client = new TestClient(target(), false, { + ...channelOptions, + ...extra, + }); + return client; + } + + /* TestClient keeps its Client private. Connectivity tests need the channel. */ + function getChannel(opened: TestClient): grpc.Channel { + return ( + opened as unknown as { client: { getChannel(): grpc.Channel } } + ).client.getChannel(); + } + + it('reaches READY via watchConnectivityState after DNS starts succeeding', function (done) { + this.timeout(4000); + const opened = openClient(); + const channel = getChannel(opened); + const deadline = Date.now() + 3000; + const watch = () => { + const state = channel.getConnectivityState(true); + if (state === ConnectivityState.READY) { + try { + assert.ok(controller.lookupCount >= 2); + assert.strictEqual(controller.maxInFlight, 1); + done(); + } catch (error) { + done(error); + } + return; + } + channel.watchConnectivityState(state, deadline, (err?: Error) => { + if (err) { + done(err); + return; + } + watch(); + }); + }; + watch(); + controller.waitForCount(2, 2000).then( + () => { + controller.fail = false; + }, + error => done(error) + ); + }); + + it('reaches READY via waitForReady after DNS starts succeeding', function (done) { + this.timeout(4000); + const opened = openClient(); + opened.waitForReady(Date.now() + 3000, (err?: Error) => { + try { + assert.ifError(err); + assert.strictEqual( + getChannel(opened).getConnectivityState(false), + ConnectivityState.READY + ); + assert.ok(controller.lookupCount >= 2); + done(); + } catch (error) { + done(error); + } + }); + controller.waitForCount(2, 2000).then( + () => { + controller.fail = false; + }, + error => done(error) + ); + }); + + it('completes one queued wait-for-ready RPC without a later RPC to kick resolution', function (done) { + this.timeout(4000); + const opened = openClient(); + let calls = 0; + opened.sendRequestWithMetadata( + new grpc.Metadata({ waitForReady: true }), + (err?: grpc.ServiceError) => { + calls += 1; + try { + assert.ifError(err); + assert.strictEqual(calls, 1); + assert.ok(controller.lookupCount >= 2); + assert.strictEqual(controller.maxInFlight, 1); + done(); + } catch (error) { + done(error); + } + } + ); + controller.waitForCount(2, 2000).then( + () => { + controller.fail = false; + }, + error => done(error) + ); + }); + + it('stops retries when the channel goes IDLE', function (done) { + this.timeout(5000); + const opened = openClient({ 'grpc.client_idle_timeout_ms': 1000 }); + const channel = getChannel(opened); + channel.getConnectivityState(true); + const started = Date.now(); + const waitForIdle = () => { + if (channel.getConnectivityState(false) === ConnectivityState.IDLE) { + const frozen = controller.lookupCount; + assert.ok(frozen >= 2, `lookupCount=${frozen}`); + setTimeout(() => { + try { + assert.strictEqual(controller.lookupCount, frozen); + assert.strictEqual( + channel.getConnectivityState(false), + ConnectivityState.IDLE + ); + done(); + } catch (error) { + done(error); + } + }, 300); + return; + } + if (Date.now() - started > 3500) { + done( + new Error( + `channel did not go IDLE (lookups=${controller.lookupCount})` + ) + ); + return; + } + setTimeout(waitForIdle, 50); + }; + setTimeout(waitForIdle, 200); + }); + + it('does not restart resolution when a SHUTDOWN watcher calls getConnectivityState(true)', function (done) { + this.timeout(3000); + const opened = openClient(); + const channel = getChannel(opened); + let connectAttemptsAfterShutdown = 0; + const watch = () => { + let state: ConnectivityState; + try { + state = channel.getConnectivityState(false); + } catch (error) { + done(error); + return; + } + if (state === ConnectivityState.SHUTDOWN) { + const before = controller.lookupCount; + /* Same call Client.waitForReady makes from a synchronous watcher. */ + channel.getConnectivityState(true); + connectAttemptsAfterShutdown = controller.lookupCount - before; + return; + } + channel.watchConnectivityState(state, Infinity, () => { + watch(); + }); + }; + watch(); + channel.getConnectivityState(true); + controller.waitForCount(2, 2000).then( + () => { + const atClose = controller.lookupCount; + opened.close(); + client = null; + setTimeout(() => { + try { + assert.strictEqual(connectAttemptsAfterShutdown, 0); + assert.strictEqual(controller.lookupCount, atClose); + assert.strictEqual(controller.inFlight, 0); + done(); + } catch (error) { + done(error); + } + }, 250); + }, + error => done(error) + ); + }); + + it('does not restart resolution when waitForReady observes close()', function (done) { + this.timeout(3000); + const opened = openClient(); + let readyError: Error | null = null; + opened.waitForReady(Date.now() + 3000, (err?: Error) => { + if (!err) { + readyError = new Error('waitForReady resolved after close'); + } + }); + controller.waitForCount(2, 2000).then( + () => { + const atClose = controller.lookupCount; + opened.close(); + client = null; + setTimeout(() => { + try { + assert.strictEqual(readyError, null); + assert.strictEqual(controller.lookupCount, atClose); + done(); + } catch (error) { + done(error); + } + }, 250); + }, + error => done(error) + ); + }); +}); diff --git a/packages/grpc-js/test/test-resolver.ts b/packages/grpc-js/test/test-resolver.ts index 39fc96607..1828dd7a4 100644 --- a/packages/grpc-js/test/test-resolver.ts +++ b/packages/grpc-js/test/test-resolver.ts @@ -18,6 +18,7 @@ // Allow `any` data type for testing runtime type checking. // tslint:disable no-any import * as assert from 'assert'; +import { promises as dnsPromises } from 'dns'; import * as resolverManager from '../src/resolver'; import * as resolver_dns from '../src/resolver-dns'; import * as resolver_uds from '../src/resolver-uds'; @@ -470,36 +471,145 @@ describe('Name Resolver', () => { done(); }, 10_000); }).timeout(15_000); - it('should not keep repeating failed resolutions', done => { + /* #2105 pinned "don't repeat failed resolutions" after the #2080 + * double-scheduling leak. #3128 reverses that: a failed lookup retries + * on the existing backoff until destroy(). #2098 still guarantees one + * timer per attempt, including when updateResolution() is repeated. */ + describe('failed lookup backoff', () => { + let originalLookup: typeof dnsPromises.lookup | null = null; + beforeEach(function () { + if (GRPC_NODE_USE_ALTERNATIVE_RESOLVER) { + this.skip(); + } + originalLookup = dnsPromises.lookup; + }); + afterEach(() => { + if (originalLookup) { + dnsPromises.lookup = originalLookup; + originalLookup = null; + } + }); + it('retries failed DNS lookups with backoff until destroyed', done => { + const target = resolverManager.mapUriDefaultScheme( + parseUri('late.invalid')! + )!; + let resultCount = 0; + let inFlight = 0; + let maxInFlight = 0; + dnsPromises.lookup = ((hostname: string) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + return new Promise((_resolve, reject) => { + setImmediate(() => { + inFlight -= 1; + reject(new Error(`ENOTFOUND ${hostname}`)); + }); + }); + }) as typeof dnsPromises.lookup; + const resolver = resolverManager.createResolver( + target, + (maybeEndpointList: StatusOr) => { + assert.strictEqual(maybeEndpointList.ok, false); + resultCount += 1; + if (resultCount === 1) { + process.nextTick(() => resolver.updateResolution()); + } + /* Accepting the error must not cancel the backoff. */ + return true; + }, + { + 'grpc.initial_reconnect_backoff_ms': 30, + 'grpc.max_reconnect_backoff_ms': 30, + 'grpc.service_config_disable_resolution': 1, + } + ); + resolver.updateResolution(); + const hammer = setInterval(() => { + resolver.updateResolution(); + }, 5); + setTimeout(() => { + clearInterval(hammer); + const settledCount = resultCount; + resolver.destroy(); + setTimeout(() => { + try { + assert.ok( + settledCount >= 4, + `expected retries, resultCount=${settledCount}` + ); + assert.ok( + settledCount <= 40, + `retries were not backed off, resultCount=${settledCount}` + ); + assert.strictEqual(maxInFlight, 1); + assert.strictEqual(resultCount, settledCount); + done(); + } catch (error) { + done(error); + } + }, 200); + }, 700); + }).timeout(3000); + it('does not report a lookup that settles after destroy', done => { + const target = resolverManager.mapUriDefaultScheme( + parseUri('late.invalid')! + )!; + let resultCount = 0; + dnsPromises.lookup = (() => { + return new Promise((_resolve, reject) => { + setTimeout(() => { + reject(new Error('ENOTFOUND late.invalid')); + }, 40); + }); + }) as typeof dnsPromises.lookup; + const resolver = resolverManager.createResolver( + target, + () => { + resultCount += 1; + return false; + }, + { + 'grpc.initial_reconnect_backoff_ms': 10, + 'grpc.max_reconnect_backoff_ms': 10, + 'grpc.service_config_disable_resolution': 1, + } + ); + resolver.updateResolution(); + setTimeout(() => resolver.destroy(), 10); + setTimeout(() => { + try { + assert.strictEqual(resultCount, 0); + done(); + } catch (error) { + done(error); + } + }, 120); + }).timeout(2000); + }); + it('does not retry an unparseable DNS target', function (done) { const target = resolverManager.mapUriDefaultScheme( - parseUri('host.invalid')! + parseUri('dns:bad:port')! )!; let resultCount = 0; const resolver = resolverManager.createResolver( target, - ( - maybeEndpointList: StatusOr, - attributes: { [key: string]: unknown}, - serviceConfig: StatusOr | null, - resolutionNote: string - ) => { - console.log(`Enpoint list: ${JSON.stringify(maybeEndpointList)}`); - assert(!maybeEndpointList.ok); + (maybeEndpointList: StatusOr) => { + assert.strictEqual(maybeEndpointList.ok, false); resultCount += 1; - console.log(`resultCount: ${resultCount}`); - if (resultCount === 1) { - process.nextTick(() => resolver.updateResolution()); - } - return true; + return false; }, - {} + { + 'grpc.initial_reconnect_backoff_ms': 20, + 'grpc.max_reconnect_backoff_ms': 20, + } ); resolver.updateResolution(); setTimeout(() => { - assert.strictEqual(resultCount, 2, `resultCount ${resultCount} !== 2`); + resolver.destroy(); + assert.strictEqual(resultCount, 1); done(); - }, 10_000); - }).timeout(15_000); + }, 200); + }).timeout(2000); }); describe('UDS Names', () => { it('Should handle a relative Unix Domain Socket name', done => {