Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions optimade/client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -965,12 +965,12 @@ async def _get_one_async(
results = QueryResults()
number_of_requests = 0
total_data_available: int | None = None
attempts = 0
try:
async with self._http_client(headers=self.headers) as client: # type: ignore[union-attr,call-arg,misc]
while next_url:
number_of_requests += 1

attempts = 0
try:
if self.verbosity:
self._progress.print(
Expand Down Expand Up @@ -1011,6 +1011,7 @@ async def _get_one_async(
await asyncio.sleep(request_delay or 1)
continue

attempts = 0
results.update(page_results)

if not paginate:
Expand Down Expand Up @@ -1070,6 +1071,7 @@ def _get_one(
results = QueryResults()
number_of_requests: int = 0
total_data_available: int | None = None
attempts = 0
try:
with self._http_client() as client: # type: ignore[misc]
client.headers.update(self.headers)
Expand All @@ -1080,7 +1082,6 @@ def _get_one(

while next_url:
number_of_requests += 1
attempts = 0
try:
if self.verbosity:
self._progress.print(
Expand Down Expand Up @@ -1120,6 +1121,7 @@ def _get_one(
time.sleep(request_delay or 1)
continue

attempts = 0
results.update(page_results)

if len(results.data) == 0 or number_of_requests > stopping_criteria:
Expand Down
67 changes: 67 additions & 0 deletions tests/server/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -654,3 +654,70 @@ def test_guardrail_sync(http_client):
)
assert len(result[TEST_URL].errors) == 1
assert "infinite" in result[TEST_URL].errors[0]


def _always_too_many_requests(calls: list[str], monkeypatch):
"""Returns sync and async test HTTP clients that always respond with
429 Too Many Requests, recording each requested URL in `calls`, and
disables the client's wait between retries."""
from .utils import AsyncHttpxTestClient, RequestsTestClient

def too_many_requests(url):
calls.append(url)
# Fail loudly rather than hang if the retry limit is never reached
if len(calls) > 50:
raise ValueError("Retry limit was not respected")
return httpx.Response(429, content=b"Too Many Requests")

class SyncClient(RequestsTestClient):
def request(self, method, url, *args, **kwargs):
return too_many_requests(url)

class AsyncClient(AsyncHttpxTestClient):
async def request(self, method, url, **kwargs):
return too_many_requests(url)

async def no_sleep(*_):
pass

monkeypatch.setattr("optimade.client.client.time.sleep", lambda *_: None)
monkeypatch.setattr("optimade.client.client.asyncio.sleep", no_sleep)

return SyncClient, AsyncClient


@pytest.mark.asyncio
async def test_retry_limit_async(monkeypatch):
"""Test that a provider that always returns 429 is retried at most
`max_attempts` times, rather than forever."""
calls: list[str] = []
_, async_client = _always_too_many_requests(calls, monkeypatch)
cli = OptimadeClient(
base_urls=[TEST_URL],
http_client=async_client,
use_async=True,
max_attempts=2,
)
result = await cli.get_one_async(
endpoint="structures", base_url=TEST_URL, filter=""
)
assert len(calls) == cli.max_attempts + 1
assert len(result[TEST_URL].errors) == 1
assert "Exceeded maximum number of retries" in result[TEST_URL].errors[0]


def test_retry_limit_sync(monkeypatch):
"""Test that a provider that always returns 429 is retried at most
`max_attempts` times, rather than forever."""
calls: list[str] = []
sync_client, _ = _always_too_many_requests(calls, monkeypatch)
cli = OptimadeClient(
base_urls=[TEST_URL],
http_client=sync_client,
use_async=False,
max_attempts=2,
)
result = cli.get_one(endpoint="structures", base_url=TEST_URL, filter="")
assert len(calls) == cli.max_attempts + 1
assert len(result[TEST_URL].errors) == 1
assert "Exceeded maximum number of retries" in result[TEST_URL].errors[0]
Loading