From 49895d2d5bba54c1f1cd81e01e9b299ebc21afaa Mon Sep 17 00:00:00 2001 From: "Slai.life" Date: Fri, 30 Oct 2026 00:44:41 +0800 Subject: [PATCH 1/7] Retry transient Docker image pulls --- launch/core/platforms/base.py | 56 ++++++++++++++++++++------- tests/image_pull_retry_test.py | 70 ++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 tests/image_pull_retry_test.py diff --git a/launch/core/platforms/base.py b/launch/core/platforms/base.py index da4991e..7f94497 100644 --- a/launch/core/platforms/base.py +++ b/launch/core/platforms/base.py @@ -5,6 +5,7 @@ import os import re import tarfile +import time from dataclasses import dataclass from pathlib import Path from typing import Any, Literal @@ -260,27 +261,56 @@ def commit(self, image_name: str, tag: str = "latest", push: bool = False) -> st @staticmethod def pull_image(image_name: str) -> bool: + """Ensure an image is local, retrying transient registry/daemon failures. + + Docker Desktop and registries can return transient ``unexpected EOF``, + ``502 Bad Gateway`` or connection-reset errors while downloading a layer. + A single failed pull must not be treated as a missing benchmark image: + retry with bounded backoff, then report the terminal error with its + original detail. The image is re-checked after every successful pull so + a partial daemon response cannot be mistaken for a usable image. """ - Pull Docker image from registry. - - Args: - image_name (str): Name of the Docker image to pull - - Returns: - bool: True if successful, False if image not found - """ - client = docker.from_env(timeout=3600) # pull should finish in 1 hour + client = docker.from_env(timeout=30) try: - # Check if image exists locally client.images.get(image_name) return True except docker.errors.ImageNotFound: - # Image doesn't exist locally, try to pull it + pass + + try: + max_attempts = int(os.getenv("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "3")) + except ValueError as exc: + raise ValueError("REPOLAUNCH_IMAGE_PULL_ATTEMPTS must be a positive integer") from exc + if max_attempts <= 0: + raise ValueError("REPOLAUNCH_IMAGE_PULL_ATTEMPTS must be a positive integer") + try: + backoff = float(os.getenv("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "2")) + except ValueError as exc: + raise ValueError("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS must be non-negative") from exc + if backoff < 0: + raise ValueError("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS must be non-negative") + + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): try: client.images.pull(image_name) + client.images.get(image_name) return True - except docker.errors.ImageNotFound: - raise ValueError(f"Image {image_name} not found in registry") + except docker.errors.ImageNotFound as exc: + # A definitive registry miss is not transient. + raise ValueError(f"Image {image_name} not found in registry") from exc + except (docker.errors.APIError, docker.errors.DockerException) as exc: + last_error = exc + if attempt == max_attempts: + break + delay = backoff * (2 ** (attempt - 1)) + if delay: + time.sleep(delay) + + detail = str(last_error).strip() if last_error else "unknown Docker pull failure" + raise RuntimeError( + f"Docker image pull for {image_name} failed after {max_attempts} attempts: {detail}" + ) from last_error def cleanup(self, prune_dangling: bool = True) -> None: if self.stopped: diff --git a/tests/image_pull_retry_test.py b/tests/image_pull_retry_test.py new file mode 100644 index 0000000..fd468ff --- /dev/null +++ b/tests/image_pull_retry_test.py @@ -0,0 +1,70 @@ +import os +from unittest.mock import Mock, patch + +import docker +import pytest + +from launch.core.platforms.base import BaseRuntime + + +class TestPullImageRetry: + def test_retries_transient_api_error_then_verifies_image(self): + client = Mock() + client.images.get.side_effect = [ + docker.errors.ImageNotFound("missing"), + object(), + ] + client.images.pull.side_effect = [ + docker.errors.APIError("unexpected EOF"), + object(), + ] + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch("launch.core.platforms.base.time.sleep") as sleep, \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "3", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "2"}, clear=False): + assert BaseRuntime.pull_image("example/image:latest") is True + + assert client.images.pull.call_count == 2 + assert client.images.get.call_count == 2 + assert sleep.call_count == 1 + sleep.assert_called_once_with(2) + + def test_retries_docker_exception_and_reports_terminal_detail(self): + client = Mock() + client.images.get.side_effect = docker.errors.ImageNotFound("missing") + client.images.pull.side_effect = docker.errors.DockerException("502 Bad Gateway") + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch("launch.core.platforms.base.time.sleep"), \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "2", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "0"}, clear=False): + with pytest.raises(RuntimeError, match=r"failed after 2 attempts: 502 Bad Gateway"): + BaseRuntime.pull_image("example/image:latest") + + assert client.images.pull.call_count == 2 + + def test_does_not_retry_definitive_missing_image(self): + client = Mock() + client.images.get.side_effect = docker.errors.ImageNotFound("missing") + client.images.pull.side_effect = docker.errors.ImageNotFound("not in registry") + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "5"}, clear=False): + with pytest.raises(ValueError, match="not found in registry"): + BaseRuntime.pull_image("example/image:latest") + + client.images.pull.assert_called_once_with("example/image:latest") + + @pytest.mark.parametrize("name,value", [ + ("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "0"), + ("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "bad"), + ("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "-1"), + ("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "bad"), + ]) + def test_rejects_invalid_retry_configuration(self, name, value): + client = Mock() + client.images.get.side_effect = docker.errors.ImageNotFound("missing") + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch.dict(os.environ, {name: value}, clear=False): + with pytest.raises(ValueError): + BaseRuntime.pull_image("example/image:latest") + client.images.pull.assert_not_called() \ No newline at end of file From 4672b8bbc13a0c282ffa69e4c128c3b3903ecd8f Mon Sep 17 00:00:00 2001 From: "Slai.life" Date: Fri, 30 Oct 2026 02:03:28 +0800 Subject: [PATCH 2/7] Preserve long Docker pull timeout and retry cache checks --- launch/core/platforms/base.py | 42 +++++++++++++++++++++++++--------- tests/image_pull_retry_test.py | 27 ++++++++++++++++++++++ 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/launch/core/platforms/base.py b/launch/core/platforms/base.py index 7f94497..0865a7d 100644 --- a/launch/core/platforms/base.py +++ b/launch/core/platforms/base.py @@ -270,12 +270,13 @@ def pull_image(image_name: str) -> bool: original detail. The image is re-checked after every successful pull so a partial daemon response cannot be mistaken for a usable image. """ - client = docker.from_env(timeout=30) try: - client.images.get(image_name) - return True - except docker.errors.ImageNotFound: - pass + pull_timeout = int(os.getenv("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS", "3600")) + except ValueError as exc: + raise ValueError("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS must be a positive integer") from exc + if pull_timeout <= 0: + raise ValueError("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS must be a positive integer") + client = docker.from_env(timeout=pull_timeout) try: max_attempts = int(os.getenv("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "3")) @@ -293,13 +294,32 @@ def pull_image(image_name: str) -> bool: last_error: Exception | None = None for attempt in range(1, max_attempts + 1): try: - client.images.pull(image_name) - client.images.get(image_name) + # A transient daemon error while checking the local cache must + # be retried just like a transient registry pull error. + try: + client.images.get(image_name) + return True + except docker.errors.ImageNotFound: + pass + + try: + client.images.pull(image_name) + except docker.errors.ImageNotFound as exc: + # A registry-level missing image is definitive. + raise ValueError(f"Image {image_name} not found in registry") from exc + + try: + client.images.get(image_name) + except docker.errors.ImageNotFound as exc: + # A successful pull response followed by a missing local + # image indicates an incomplete daemon operation; retry it. + raise docker.errors.DockerException( + f"Docker pull completed but image {image_name} was not found locally" + ) from exc return True - except docker.errors.ImageNotFound as exc: - # A definitive registry miss is not transient. - raise ValueError(f"Image {image_name} not found in registry") from exc - except (docker.errors.APIError, docker.errors.DockerException) as exc: + except ValueError: + raise + except docker.errors.DockerException as exc: last_error = exc if attempt == max_attempts: break diff --git a/tests/image_pull_retry_test.py b/tests/image_pull_retry_test.py index fd468ff..389ba4a 100644 --- a/tests/image_pull_retry_test.py +++ b/tests/image_pull_retry_test.py @@ -42,6 +42,33 @@ def test_retries_docker_exception_and_reports_terminal_detail(self): assert client.images.pull.call_count == 2 + def test_retries_when_pull_returns_before_image_is_locally_visible(self): + client = Mock() + client.images.get.side_effect = [ + docker.errors.ImageNotFound("missing before pull"), + docker.errors.ImageNotFound("missing after first pull"), + object(), + ] + client.images.pull.side_effect = [object(), object()] + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch("launch.core.platforms.base.time.sleep") as sleep, \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "3", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "0"}, clear=False): + assert BaseRuntime.pull_image("example/image:latest") is True + + assert client.images.pull.call_count == 2 + assert client.images.get.call_count == 3 + sleep.assert_not_called() + + def test_uses_long_default_timeout_for_large_images(self): + client = Mock() + client.images.get.return_value = object() + + with patch("launch.core.platforms.base.docker.from_env", return_value=client) as from_env: + assert BaseRuntime.pull_image("example/image:latest") is True + + from_env.assert_called_once_with(timeout=3600) + def test_does_not_retry_definitive_missing_image(self): client = Mock() client.images.get.side_effect = docker.errors.ImageNotFound("missing") From 1622bba88c3ded78893844b47e88c41cdd07fb16 Mon Sep 17 00:00:00 2001 From: "Slai.life" Date: Fri, 30 Oct 2026 02:14:41 +0800 Subject: [PATCH 3/7] Fix retry test sequencing and cache lookup coverage --- tests/image_pull_retry_test.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/image_pull_retry_test.py b/tests/image_pull_retry_test.py index 389ba4a..e7b7f9e 100644 --- a/tests/image_pull_retry_test.py +++ b/tests/image_pull_retry_test.py @@ -11,7 +11,8 @@ class TestPullImageRetry: def test_retries_transient_api_error_then_verifies_image(self): client = Mock() client.images.get.side_effect = [ - docker.errors.ImageNotFound("missing"), + docker.errors.ImageNotFound("missing before first pull"), + docker.errors.ImageNotFound("missing before retry"), object(), ] client.images.pull.side_effect = [ @@ -25,7 +26,7 @@ def test_retries_transient_api_error_then_verifies_image(self): assert BaseRuntime.pull_image("example/image:latest") is True assert client.images.pull.call_count == 2 - assert client.images.get.call_count == 2 + assert client.images.get.call_count == 3 assert sleep.call_count == 1 sleep.assert_called_once_with(2) @@ -45,8 +46,9 @@ def test_retries_docker_exception_and_reports_terminal_detail(self): def test_retries_when_pull_returns_before_image_is_locally_visible(self): client = Mock() client.images.get.side_effect = [ - docker.errors.ImageNotFound("missing before pull"), + docker.errors.ImageNotFound("missing before first pull"), docker.errors.ImageNotFound("missing after first pull"), + docker.errors.ImageNotFound("missing before retry pull"), object(), ] client.images.pull.side_effect = [object(), object()] @@ -57,9 +59,27 @@ def test_retries_when_pull_returns_before_image_is_locally_visible(self): assert BaseRuntime.pull_image("example/image:latest") is True assert client.images.pull.call_count == 2 - assert client.images.get.call_count == 3 + assert client.images.get.call_count == 4 sleep.assert_not_called() + def test_retries_transient_cache_lookup_error(self): + client = Mock() + client.images.get.side_effect = [ + docker.errors.DockerException("502 Bad Gateway"), + docker.errors.ImageNotFound("missing after retry"), + object(), + ] + client.images.pull.return_value = object() + + with patch("launch.core.platforms.base.docker.from_env", return_value=client), \ + patch("launch.core.platforms.base.time.sleep") as sleep, \ + patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "2", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "2"}, clear=False): + assert BaseRuntime.pull_image("example/image:latest") is True + + client.images.pull.assert_called_once_with("example/image:latest") + assert client.images.get.call_count == 3 + sleep.assert_called_once_with(2) + def test_uses_long_default_timeout_for_large_images(self): client = Mock() client.images.get.return_value = object() From 740a82746dc11a2cfd1f744d913eee440321fcc0 Mon Sep 17 00:00:00 2001 From: "Slai.life" Date: Fri, 30 Oct 2026 02:44:57 +0800 Subject: [PATCH 4/7] Use POSIX paths for Linux container patch mounts --- launch/core/platforms/linux.py | 6 +++++- tests/runtime_test.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/launch/core/platforms/linux.py b/launch/core/platforms/linux.py index bd87786..f747d2e 100644 --- a/launch/core/platforms/linux.py +++ b/launch/core/platforms/linux.py @@ -17,6 +17,7 @@ ) import os, json +import posixpath from typing import Any, Optional import queue import threading @@ -207,7 +208,10 @@ def apply_patch(self, patch: str, verbose: bool = False) -> bool: hostpath = os.path.join(self.mnt_host, filename) with open(hostpath, "w") as f: f.write(patch) - containerpath = os.path.join(self.mnt_container, filename) + # `mnt_container` is a POSIX path even when RepoLaunch runs on Windows. + # Using os.path.join here creates `/mnt_tmp\\name.diff`, which the Linux + # shell interprets as `/mnt_tmpname.diff` and cannot open. + containerpath = posixpath.join(self.mnt_container, filename) cmd = f"""git apply --reject --whitespace=nowarn {containerpath} """ res = self.send_command(cmd) diff --git a/tests/runtime_test.py b/tests/runtime_test.py index 809ec3d..ca79629 100644 --- a/tests/runtime_test.py +++ b/tests/runtime_test.py @@ -42,6 +42,7 @@ import os import platform as host_platform import warnings +from types import SimpleNamespace import docker import pytest @@ -170,6 +171,26 @@ def test_runtime_constructor_attributes(runtime_cls, expected_attrs, patch_runti +def test_linux_apply_patch_uses_posix_container_path(tmp_path): + runtime = LinuxRuntime.__new__(LinuxRuntime) + runtime.mnt_host = str(tmp_path) + runtime.mnt_container = "/mnt_tmp" + commands = [] + + def fake_send_command(command): + commands.append(command) + return SimpleNamespace( + output="", + metadata=SimpleNamespace(exit_code=0), + ) + + runtime.send_command = fake_send_command + assert runtime.apply_patch("diff --git a/a b/a\n", verbose=False) is True + assert commands[0].startswith("git apply") + assert "/mnt_tmp/" in commands[0] + assert "\\" not in commands[0] + + def supported_integration_platforms() -> set[str]: system = host_platform.system().lower() if system == "windows": From e18ad26890a5fe8c288ee82de516e6bc1334a871 Mon Sep 17 00:00:00 2001 From: "Slai.life" Date: Fri, 30 Oct 2026 02:51:22 +0800 Subject: [PATCH 5/7] Avoid destructor warning in container path regression test --- tests/runtime_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/runtime_test.py b/tests/runtime_test.py index ca79629..53e42b8 100644 --- a/tests/runtime_test.py +++ b/tests/runtime_test.py @@ -189,6 +189,7 @@ def fake_send_command(command): assert commands[0].startswith("git apply") assert "/mnt_tmp/" in commands[0] assert "\\" not in commands[0] + runtime.stopped = True def supported_integration_platforms() -> set[str]: From 2222ae6d8a3d2968e6c152930a74e5d999997680 Mon Sep 17 00:00:00 2001 From: "Slai.life" Date: Fri, 30 Oct 2026 02:56:13 +0800 Subject: [PATCH 6/7] Preserve LF patch bytes in Windows staging --- launch/core/platforms/linux.py | 5 ++++- tests/runtime_test.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/launch/core/platforms/linux.py b/launch/core/platforms/linux.py index f747d2e..afff27c 100644 --- a/launch/core/platforms/linux.py +++ b/launch/core/platforms/linux.py @@ -206,7 +206,10 @@ def apply_patch(self, patch: str, verbose: bool = False) -> bool: filename = f"{uuid.uuid4()}.diff" hostpath = os.path.join(self.mnt_host, filename) - with open(hostpath, "w") as f: + # Preserve the model patch's bytes across Windows host staging. Text mode + # with the default newline policy rewrites LF to CRLF on Windows, which + # can make otherwise valid unified-diff context fail to apply in Linux. + with open(hostpath, "w", encoding="utf-8", newline="\n") as f: f.write(patch) # `mnt_container` is a POSIX path even when RepoLaunch runs on Windows. # Using os.path.join here creates `/mnt_tmp\\name.diff`, which the Linux diff --git a/tests/runtime_test.py b/tests/runtime_test.py index 53e42b8..506b1a1 100644 --- a/tests/runtime_test.py +++ b/tests/runtime_test.py @@ -184,11 +184,15 @@ def fake_send_command(command): metadata=SimpleNamespace(exit_code=0), ) + patch = "diff --git a/a b/a\n@@ -1 +1 @@\n-old\n+new\n" runtime.send_command = fake_send_command - assert runtime.apply_patch("diff --git a/a b/a\n", verbose=False) is True + assert runtime.apply_patch(patch, verbose=False) is True assert commands[0].startswith("git apply") assert "/mnt_tmp/" in commands[0] assert "\\" not in commands[0] + staged_files = list(tmp_path.glob("*.diff")) + assert len(staged_files) == 1 + assert staged_files[0].read_bytes() == patch.encode("utf-8") runtime.stopped = True From 12703f5c573fe4d2be6ccf4493f91fabe744cf11 Mon Sep 17 00:00:00 2001 From: "Slai.life" Date: Fri, 30 Oct 2026 03:05:43 +0800 Subject: [PATCH 7/7] Clean up host patch staging files --- launch/core/platforms/linux.py | 17 +++++++++++++++-- tests/runtime_test.py | 12 +++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/launch/core/platforms/linux.py b/launch/core/platforms/linux.py index afff27c..0a407f3 100644 --- a/launch/core/platforms/linux.py +++ b/launch/core/platforms/linux.py @@ -217,8 +217,21 @@ def apply_patch(self, patch: str, verbose: bool = False) -> bool: containerpath = posixpath.join(self.mnt_container, filename) cmd = f"""git apply --reject --whitespace=nowarn {containerpath} """ - res = self.send_command(cmd) - self.send_command(f"rm {containerpath}") + try: + res = self.send_command(cmd) + finally: + # The container-side file is temporary, and the bind-mounted host + # file must be removed as well. Otherwise repeated evaluations leave + # untracked patch files in the evaluator checkout and make a later + # clean-source check fail closed for the wrong reason. + try: + self.send_command(f"rm {containerpath}") + except Exception: + pass + try: + os.remove(hostpath) + except FileNotFoundError: + pass if int(res.metadata.exit_code) == 0: print(f"{cmd} ---- Patch applied Successfully!", flush=True) return True diff --git a/tests/runtime_test.py b/tests/runtime_test.py index 506b1a1..1f84ae8 100644 --- a/tests/runtime_test.py +++ b/tests/runtime_test.py @@ -176,9 +176,14 @@ def test_linux_apply_patch_uses_posix_container_path(tmp_path): runtime.mnt_host = str(tmp_path) runtime.mnt_container = "/mnt_tmp" commands = [] + staged_bytes = [] def fake_send_command(command): commands.append(command) + if command.startswith("git apply"): + staged_files = list(tmp_path.glob("*.diff")) + assert len(staged_files) == 1 + staged_bytes.append(staged_files[0].read_bytes()) return SimpleNamespace( output="", metadata=SimpleNamespace(exit_code=0), @@ -190,9 +195,10 @@ def fake_send_command(command): assert commands[0].startswith("git apply") assert "/mnt_tmp/" in commands[0] assert "\\" not in commands[0] - staged_files = list(tmp_path.glob("*.diff")) - assert len(staged_files) == 1 - assert staged_files[0].read_bytes() == patch.encode("utf-8") + assert staged_bytes == [patch.encode("utf-8")] + # apply_patch must remove the host-side staging file as well as the + # container-side file, so a later evaluator provenance check is not dirtied. + assert not list(tmp_path.glob("*.diff")) runtime.stopped = True