diff --git a/launch/core/platforms/base.py b/launch/core/platforms/base.py index da4991e..0865a7d 100644 --- a/launch/core/platforms/base.py +++ b/launch/core/platforms/base.py @@ -5,6 +5,7 @@ import os import re import tarfile +import time from dataclasses import dataclass from pathlib import Path from typing import Any, Literal @@ -260,27 +261,76 @@ def commit(self, image_name: str, tag: str = "latest", push: bool = False) -> st @staticmethod def pull_image(image_name: str) -> bool: + """Ensure an image is local, retrying transient registry/daemon failures. + + Docker Desktop and registries can return transient ``unexpected EOF``, + ``502 Bad Gateway`` or connection-reset errors while downloading a layer. + A single failed pull must not be treated as a missing benchmark image: + retry with bounded backoff, then report the terminal error with its + original detail. The image is re-checked after every successful pull so + a partial daemon response cannot be mistaken for a usable image. """ - Pull Docker image from registry. - - Args: - image_name (str): Name of the Docker image to pull - - Returns: - bool: True if successful, False if image not found - """ - client = docker.from_env(timeout=3600) # pull should finish in 1 hour try: - # Check if image exists locally - client.images.get(image_name) - return True - except docker.errors.ImageNotFound: - # Image doesn't exist locally, try to pull it + pull_timeout = int(os.getenv("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS", "3600")) + except ValueError as exc: + raise ValueError("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS must be a positive integer") from exc + if pull_timeout <= 0: + raise ValueError("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS must be a positive integer") + client = docker.from_env(timeout=pull_timeout) + + try: + max_attempts = int(os.getenv("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "3")) + except ValueError as exc: + raise ValueError("REPOLAUNCH_IMAGE_PULL_ATTEMPTS must be a positive integer") from exc + if max_attempts <= 0: + raise ValueError("REPOLAUNCH_IMAGE_PULL_ATTEMPTS must be a positive integer") + try: + backoff = float(os.getenv("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "2")) + except ValueError as exc: + raise ValueError("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS must be non-negative") from exc + if backoff < 0: + raise ValueError("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS must be non-negative") + + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): try: - client.images.pull(image_name) + # A transient daemon error while checking the local cache must + # be retried just like a transient registry pull error. + try: + client.images.get(image_name) + return True + except docker.errors.ImageNotFound: + pass + + try: + client.images.pull(image_name) + except docker.errors.ImageNotFound as exc: + # A registry-level missing image is definitive. + raise ValueError(f"Image {image_name} not found in registry") from exc + + try: + client.images.get(image_name) + except docker.errors.ImageNotFound as exc: + # A successful pull response followed by a missing local + # image indicates an incomplete daemon operation; retry it. + raise docker.errors.DockerException( + f"Docker pull completed but image {image_name} was not found locally" + ) from exc return True - except docker.errors.ImageNotFound: - raise ValueError(f"Image {image_name} not found in registry") + except ValueError: + raise + except docker.errors.DockerException as exc: + last_error = exc + if attempt == max_attempts: + break + delay = backoff * (2 ** (attempt - 1)) + if delay: + time.sleep(delay) + + detail = str(last_error).strip() if last_error else "unknown Docker pull failure" + raise RuntimeError( + f"Docker image pull for {image_name} failed after {max_attempts} attempts: {detail}" + ) from last_error def cleanup(self, prune_dangling: bool = True) -> None: if self.stopped: diff --git a/launch/core/platforms/linux.py b/launch/core/platforms/linux.py index bd87786..0a407f3 100644 --- a/launch/core/platforms/linux.py +++ b/launch/core/platforms/linux.py @@ -17,6 +17,7 @@ ) import os, json +import posixpath from typing import Any, Optional import queue import threading @@ -205,13 +206,32 @@ def apply_patch(self, patch: str, verbose: bool = False) -> bool: filename = f"{uuid.uuid4()}.diff" hostpath = os.path.join(self.mnt_host, filename) - with open(hostpath, "w") as f: + # Preserve the model patch's bytes across Windows host staging. Text mode + # with the default newline policy rewrites LF to CRLF on Windows, which + # can make otherwise valid unified-diff context fail to apply in Linux. + with open(hostpath, "w", encoding="utf-8", newline="\n") as f: f.write(patch) - containerpath = os.path.join(self.mnt_container, filename) + # `mnt_container` is a POSIX path even when RepoLaunch runs on Windows. + # Using os.path.join here creates `/mnt_tmp\\name.diff`, which the Linux + # shell interprets as `/mnt_tmpname.diff` and cannot open. + containerpath = posixpath.join(self.mnt_container, filename) cmd = f"""git apply --reject --whitespace=nowarn {containerpath} """ - res = self.send_command(cmd) - self.send_command(f"rm {containerpath}") + try: + res = self.send_command(cmd) + finally: + # The container-side file is temporary, and the bind-mounted host + # file must be removed as well. Otherwise repeated evaluations leave + # untracked patch files in the evaluator checkout and make a later + # clean-source check fail closed for the wrong reason. + try: + self.send_command(f"rm {containerpath}") + except Exception: + pass + try: + os.remove(hostpath) + except FileNotFoundError: + pass if int(res.metadata.exit_code) == 0: print(f"{cmd} ---- Patch applied Successfully!", flush=True) return True diff --git a/tests/image_pull_retry_test.py b/tests/image_pull_retry_test.py new file mode 100644 index 0000000..e7b7f9e --- /dev/null +++ b/tests/image_pull_retry_test.py @@ -0,0 +1,117 @@ +import os +from unittest.mock import Mock, patch + +import docker +import pytest + +from launch.core.platforms.base import BaseRuntime + + +class TestPullImageRetry: + def test_retries_transient_api_error_then_verifies_image(self): + client = Mock() + client.images.get.side_effect = [ + docker.errors.ImageNotFound("missing before first pull"), + docker.errors.ImageNotFound("missing before retry"), + object(), + ] + client.images.pull.side_effect = [ + docker.errors.APIError("unexpected EOF"), + object(), + ] + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch("launch.core.platforms.base.time.sleep") as sleep, \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "3", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "2"}, clear=False): + assert BaseRuntime.pull_image("example/image:latest") is True + + assert client.images.pull.call_count == 2 + assert client.images.get.call_count == 3 + assert sleep.call_count == 1 + sleep.assert_called_once_with(2) + + def test_retries_docker_exception_and_reports_terminal_detail(self): + client = Mock() + client.images.get.side_effect = docker.errors.ImageNotFound("missing") + client.images.pull.side_effect = docker.errors.DockerException("502 Bad Gateway") + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch("launch.core.platforms.base.time.sleep"), \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "2", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "0"}, clear=False): + with pytest.raises(RuntimeError, match=r"failed after 2 attempts: 502 Bad Gateway"): + BaseRuntime.pull_image("example/image:latest") + + assert client.images.pull.call_count == 2 + + def test_retries_when_pull_returns_before_image_is_locally_visible(self): + client = Mock() + client.images.get.side_effect = [ + docker.errors.ImageNotFound("missing before first pull"), + docker.errors.ImageNotFound("missing after first pull"), + docker.errors.ImageNotFound("missing before retry pull"), + object(), + ] + client.images.pull.side_effect = [object(), object()] + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch("launch.core.platforms.base.time.sleep") as sleep, \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "3", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "0"}, clear=False): + assert BaseRuntime.pull_image("example/image:latest") is True + + assert client.images.pull.call_count == 2 + assert client.images.get.call_count == 4 + sleep.assert_not_called() + + def test_retries_transient_cache_lookup_error(self): + client = Mock() + client.images.get.side_effect = [ + docker.errors.DockerException("502 Bad Gateway"), + docker.errors.ImageNotFound("missing after retry"), + object(), + ] + client.images.pull.return_value = object() + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch("launch.core.platforms.base.time.sleep") as sleep, \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "2", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "2"}, clear=False): + assert BaseRuntime.pull_image("example/image:latest") is True + + client.images.pull.assert_called_once_with("example/image:latest") + assert client.images.get.call_count == 3 + sleep.assert_called_once_with(2) + + def test_uses_long_default_timeout_for_large_images(self): + client = Mock() + client.images.get.return_value = object() + + with patch("launch.core.platforms.base.docker.from_env", return_value=client) as from_env: + assert BaseRuntime.pull_image("example/image:latest") is True + + from_env.assert_called_once_with(timeout=3600) + + def test_does_not_retry_definitive_missing_image(self): + client = Mock() + client.images.get.side_effect = docker.errors.ImageNotFound("missing") + client.images.pull.side_effect = docker.errors.ImageNotFound("not in registry") + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "5"}, clear=False): + with pytest.raises(ValueError, match="not found in registry"): + BaseRuntime.pull_image("example/image:latest") + + client.images.pull.assert_called_once_with("example/image:latest") + + @pytest.mark.parametrize("name,value", [ + ("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "0"), + ("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "bad"), + ("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "-1"), + ("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "bad"), + ]) + def test_rejects_invalid_retry_configuration(self, name, value): + client = Mock() + client.images.get.side_effect = docker.errors.ImageNotFound("missing") + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch.dict(os.environ, {name: value}, clear=False): + with pytest.raises(ValueError): + BaseRuntime.pull_image("example/image:latest") + client.images.pull.assert_not_called() \ No newline at end of file diff --git a/tests/runtime_test.py b/tests/runtime_test.py index 809ec3d..1f84ae8 100644 --- a/tests/runtime_test.py +++ b/tests/runtime_test.py @@ -42,6 +42,7 @@ import os import platform as host_platform import warnings +from types import SimpleNamespace import docker import pytest @@ -170,6 +171,37 @@ def test_runtime_constructor_attributes(runtime_cls, expected_attrs, patch_runti +def test_linux_apply_patch_uses_posix_container_path(tmp_path): + runtime = LinuxRuntime.__new__(LinuxRuntime) + runtime.mnt_host = str(tmp_path) + runtime.mnt_container = "/mnt_tmp" + commands = [] + staged_bytes = [] + + def fake_send_command(command): + commands.append(command) + if command.startswith("git apply"): + staged_files = list(tmp_path.glob("*.diff")) + assert len(staged_files) == 1 + staged_bytes.append(staged_files[0].read_bytes()) + return SimpleNamespace( + output="", + metadata=SimpleNamespace(exit_code=0), + ) + + patch = "diff --git a/a b/a\n@@ -1 +1 @@\n-old\n+new\n" + runtime.send_command = fake_send_command + assert runtime.apply_patch(patch, verbose=False) is True + assert commands[0].startswith("git apply") + assert "/mnt_tmp/" in commands[0] + assert "\\" not in commands[0] + assert staged_bytes == [patch.encode("utf-8")] + # apply_patch must remove the host-side staging file as well as the + # container-side file, so a later evaluator provenance check is not dirtied. + assert not list(tmp_path.glob("*.diff")) + runtime.stopped = True + + def supported_integration_platforms() -> set[str]: system = host_platform.system().lower() if system == "windows":