From 46a33d4ce07963e9ea00d5f75dc06c654d0d9a8a Mon Sep 17 00:00:00 2001 From: Anikeya Date: Thu, 12 Nov 2026 13:57:18 +0100 Subject: [PATCH] Stop the client retrying a 429 response forever The retry counter in the client's pagination loops was reset at the start of every pass of `while next_url:`, so a provider that kept returning 429 Too Many Requests was retried forever and `max_attempts` was never reached. Initialise the counter once before the loop and reset it only after a page succeeds, in both the sync and async paths. Add sync and async tests with an HTTP client that always returns 429. Co-Authored-By: Claude Opus 5.5 --- optimade/client/client.py | 6 ++-- tests/server/test_client.py | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/optimade/client/client.py b/optimade/client/client.py index 197f998f4..32f5b1437 100644 --- a/optimade/client/client.py +++ b/optimade/client/client.py @@ -965,12 +965,12 @@ async def _get_one_async( results = QueryResults() number_of_requests = 0 total_data_available: int | None = None + attempts = 0 try: async with self._http_client(headers=self.headers) as client: # type: ignore[union-attr,call-arg,misc] while next_url: number_of_requests += 1 - attempts = 0 try: if self.verbosity: self._progress.print( @@ -1011,6 +1011,7 @@ async def _get_one_async( await asyncio.sleep(request_delay or 1) continue + attempts = 0 results.update(page_results) if not paginate: @@ -1070,6 +1071,7 @@ def _get_one( results = QueryResults() number_of_requests: int = 0 total_data_available: int | None = None + attempts = 0 try: with self._http_client() as client: # type: ignore[misc] client.headers.update(self.headers) @@ -1080,7 +1082,6 @@ def _get_one( while next_url: number_of_requests += 1 - attempts = 0 try: if self.verbosity: self._progress.print( @@ -1120,6 +1121,7 @@ def _get_one( time.sleep(request_delay or 1) continue + attempts = 0 results.update(page_results) if len(results.data) == 0 or number_of_requests > stopping_criteria: diff --git a/tests/server/test_client.py b/tests/server/test_client.py index 484c3e114..64283b247 100644 --- a/tests/server/test_client.py +++ b/tests/server/test_client.py @@ -654,3 +654,70 @@ def test_guardrail_sync(http_client): ) assert len(result[TEST_URL].errors) == 1 assert "infinite" in result[TEST_URL].errors[0] + + +def _always_too_many_requests(calls: list[str], monkeypatch): + """Returns sync and async test HTTP clients that always respond with + 429 Too Many Requests, recording each requested URL in `calls`, and + disables the client's wait between retries.""" + from .utils import AsyncHttpxTestClient, RequestsTestClient + + def too_many_requests(url): + calls.append(url) + # Fail loudly rather than hang if the retry limit is never reached + if len(calls) > 50: + raise ValueError("Retry limit was not respected") + return httpx.Response(429, content=b"Too Many Requests") + + class SyncClient(RequestsTestClient): + def request(self, method, url, *args, **kwargs): + return too_many_requests(url) + + class AsyncClient(AsyncHttpxTestClient): + async def request(self, method, url, **kwargs): + return too_many_requests(url) + + async def no_sleep(*_): + pass + + monkeypatch.setattr("optimade.client.client.time.sleep", lambda *_: None) + monkeypatch.setattr("optimade.client.client.asyncio.sleep", no_sleep) + + return SyncClient, AsyncClient + + +@pytest.mark.asyncio +async def test_retry_limit_async(monkeypatch): + """Test that a provider that always returns 429 is retried at most + `max_attempts` times, rather than forever.""" + calls: list[str] = [] + _, async_client = _always_too_many_requests(calls, monkeypatch) + cli = OptimadeClient( + base_urls=[TEST_URL], + http_client=async_client, + use_async=True, + max_attempts=2, + ) + result = await cli.get_one_async( + endpoint="structures", base_url=TEST_URL, filter="" + ) + assert len(calls) == cli.max_attempts + 1 + assert len(result[TEST_URL].errors) == 1 + assert "Exceeded maximum number of retries" in result[TEST_URL].errors[0] + + +def test_retry_limit_sync(monkeypatch): + """Test that a provider that always returns 429 is retried at most + `max_attempts` times, rather than forever.""" + calls: list[str] = [] + sync_client, _ = _always_too_many_requests(calls, monkeypatch) + cli = OptimadeClient( + base_urls=[TEST_URL], + http_client=sync_client, + use_async=False, + max_attempts=2, + ) + result = cli.get_one(endpoint="structures", base_url=TEST_URL, filter="") + assert len(calls) == cli.max_attempts + 1 + assert len(result[TEST_URL].errors) == 1 + assert "Exceeded maximum number of retries" in result[TEST_URL].errors[0]