Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 67 additions & 17 deletions launch/core/platforms/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import os
import re
import tarfile
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal
Expand Down Expand Up @@ -260,27 +261,76 @@ def commit(self, image_name: str, tag: str = "latest", push: bool = False) -> st

@staticmethod
def pull_image(image_name: str) -> bool:
"""Ensure an image is local, retrying transient registry/daemon failures.

Docker Desktop and registries can return transient ``unexpected EOF``,
``502 Bad Gateway`` or connection-reset errors while downloading a layer.
A single failed pull must not be treated as a missing benchmark image:
retry with bounded backoff, then report the terminal error with its
original detail. The image is re-checked after every successful pull so
a partial daemon response cannot be mistaken for a usable image.
"""
Pull Docker image from registry.

Args:
image_name (str): Name of the Docker image to pull

Returns:
bool: True if successful, False if image not found
"""
client = docker.from_env(timeout=3600) # pull should finish in 1 hour
try:
# Check if image exists locally
client.images.get(image_name)
return True
except docker.errors.ImageNotFound:
# Image doesn't exist locally, try to pull it
pull_timeout = int(os.getenv("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS", "3600"))
except ValueError as exc:
raise ValueError("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS must be a positive integer") from exc
if pull_timeout <= 0:
raise ValueError("REPOLAUNCH_IMAGE_PULL_TIMEOUT_SECONDS must be a positive integer")
client = docker.from_env(timeout=pull_timeout)

try:
max_attempts = int(os.getenv("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "3"))
except ValueError as exc:
raise ValueError("REPOLAUNCH_IMAGE_PULL_ATTEMPTS must be a positive integer") from exc
if max_attempts <= 0:
raise ValueError("REPOLAUNCH_IMAGE_PULL_ATTEMPTS must be a positive integer")
try:
backoff = float(os.getenv("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "2"))
except ValueError as exc:
raise ValueError("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS must be non-negative") from exc
if backoff < 0:
raise ValueError("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS must be non-negative")

last_error: Exception | None = None
for attempt in range(1, max_attempts + 1):
try:
client.images.pull(image_name)
# A transient daemon error while checking the local cache must
# be retried just like a transient registry pull error.
try:
client.images.get(image_name)
return True
except docker.errors.ImageNotFound:
pass

try:
client.images.pull(image_name)
except docker.errors.ImageNotFound as exc:
# A registry-level missing image is definitive.
raise ValueError(f"Image {image_name} not found in registry") from exc

try:
client.images.get(image_name)
except docker.errors.ImageNotFound as exc:
# A successful pull response followed by a missing local
# image indicates an incomplete daemon operation; retry it.
raise docker.errors.DockerException(
f"Docker pull completed but image {image_name} was not found locally"
) from exc
return True
except docker.errors.ImageNotFound:
raise ValueError(f"Image {image_name} not found in registry")
except ValueError:
raise
except docker.errors.DockerException as exc:
last_error = exc
if attempt == max_attempts:
break
delay = backoff * (2 ** (attempt - 1))
if delay:
time.sleep(delay)

detail = str(last_error).strip() if last_error else "unknown Docker pull failure"
raise RuntimeError(
f"Docker image pull for {image_name} failed after {max_attempts} attempts: {detail}"
) from last_error

def cleanup(self, prune_dangling: bool = True) -> None:
if self.stopped:
Expand Down
28 changes: 24 additions & 4 deletions launch/core/platforms/linux.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
)

import os, json
import posixpath
from typing import Any, Optional
import queue
import threading
Expand Down Expand Up @@ -205,13 +206,32 @@ def apply_patch(self, patch: str, verbose: bool = False) -> bool:

filename = f"{uuid.uuid4()}.diff"
hostpath = os.path.join(self.mnt_host, filename)
with open(hostpath, "w") as f:
# Preserve the model patch's bytes across Windows host staging. Text mode
# with the default newline policy rewrites LF to CRLF on Windows, which
# can make otherwise valid unified-diff context fail to apply in Linux.
with open(hostpath, "w", encoding="utf-8", newline="\n") as f:
f.write(patch)
containerpath = os.path.join(self.mnt_container, filename)
# `mnt_container` is a POSIX path even when RepoLaunch runs on Windows.
# Using os.path.join here creates `/mnt_tmp\\name.diff`, which the Linux
# shell interprets as `/mnt_tmpname.diff` and cannot open.
containerpath = posixpath.join(self.mnt_container, filename)

cmd = f"""git apply --reject --whitespace=nowarn {containerpath} """
res = self.send_command(cmd)
self.send_command(f"rm {containerpath}")
try:
res = self.send_command(cmd)
finally:
# The container-side file is temporary, and the bind-mounted host
# file must be removed as well. Otherwise repeated evaluations leave
# untracked patch files in the evaluator checkout and make a later
# clean-source check fail closed for the wrong reason.
try:
self.send_command(f"rm {containerpath}")
except Exception:
pass
try:
os.remove(hostpath)
except FileNotFoundError:
pass
if int(res.metadata.exit_code) == 0:
print(f"{cmd} ---- Patch applied Successfully!", flush=True)
return True
Expand Down
117 changes: 117 additions & 0 deletions tests/image_pull_retry_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
import os
from unittest.mock import Mock, patch

import docker
import pytest

from launch.core.platforms.base import BaseRuntime


class TestPullImageRetry:
def test_retries_transient_api_error_then_verifies_image(self):
client = Mock()
client.images.get.side_effect = [
docker.errors.ImageNotFound("missing before first pull"),
docker.errors.ImageNotFound("missing before retry"),
object(),
]
client.images.pull.side_effect = [
docker.errors.APIError("unexpected EOF"),
object(),
]

with patch("launch.core.platforms.base.docker.from_env", return_value=client), \
patch("launch.core.platforms.base.time.sleep") as sleep, \
patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "3", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "2"}, clear=False):
assert BaseRuntime.pull_image("example/image:latest") is True

assert client.images.pull.call_count == 2
assert client.images.get.call_count == 3
assert sleep.call_count == 1
sleep.assert_called_once_with(2)

def test_retries_docker_exception_and_reports_terminal_detail(self):
client = Mock()
client.images.get.side_effect = docker.errors.ImageNotFound("missing")
client.images.pull.side_effect = docker.errors.DockerException("502 Bad Gateway")

with patch("launch.core.platforms.base.docker.from_env", return_value=client), \
patch("launch.core.platforms.base.time.sleep"), \
patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "2", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "0"}, clear=False):
with pytest.raises(RuntimeError, match=r"failed after 2 attempts: 502 Bad Gateway"):
BaseRuntime.pull_image("example/image:latest")

assert client.images.pull.call_count == 2

def test_retries_when_pull_returns_before_image_is_locally_visible(self):
client = Mock()
client.images.get.side_effect = [
docker.errors.ImageNotFound("missing before first pull"),
docker.errors.ImageNotFound("missing after first pull"),
docker.errors.ImageNotFound("missing before retry pull"),
object(),
]
client.images.pull.side_effect = [object(), object()]

with patch("launch.core.platforms.base.docker.from_env", return_value=client), \
patch("launch.core.platforms.base.time.sleep") as sleep, \
patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "3", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "0"}, clear=False):
assert BaseRuntime.pull_image("example/image:latest") is True

assert client.images.pull.call_count == 2
assert client.images.get.call_count == 4
sleep.assert_not_called()

def test_retries_transient_cache_lookup_error(self):
client = Mock()
client.images.get.side_effect = [
docker.errors.DockerException("502 Bad Gateway"),
docker.errors.ImageNotFound("missing after retry"),
object(),
]
client.images.pull.return_value = object()

with patch("launch.core.platforms.base.docker.from_env", return_value=client), \
patch("launch.core.platforms.base.time.sleep") as sleep, \
patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "2", "REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS": "2"}, clear=False):
assert BaseRuntime.pull_image("example/image:latest") is True

client.images.pull.assert_called_once_with("example/image:latest")
assert client.images.get.call_count == 3
sleep.assert_called_once_with(2)

def test_uses_long_default_timeout_for_large_images(self):
client = Mock()
client.images.get.return_value = object()

with patch("launch.core.platforms.base.docker.from_env", return_value=client) as from_env:
assert BaseRuntime.pull_image("example/image:latest") is True

from_env.assert_called_once_with(timeout=3600)

def test_does_not_retry_definitive_missing_image(self):
client = Mock()
client.images.get.side_effect = docker.errors.ImageNotFound("missing")
client.images.pull.side_effect = docker.errors.ImageNotFound("not in registry")

with patch("launch.core.platforms.base.docker.from_env", return_value=client), \
patch.dict(os.environ, {"REPOLAUNCH_IMAGE_PULL_ATTEMPTS": "5"}, clear=False):
with pytest.raises(ValueError, match="not found in registry"):
BaseRuntime.pull_image("example/image:latest")

client.images.pull.assert_called_once_with("example/image:latest")

@pytest.mark.parametrize("name,value", [
("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "0"),
("REPOLAUNCH_IMAGE_PULL_ATTEMPTS", "bad"),
("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "-1"),
("REPOLAUNCH_IMAGE_PULL_BACKOFF_SECONDS", "bad"),
])
def test_rejects_invalid_retry_configuration(self, name, value):
client = Mock()
client.images.get.side_effect = docker.errors.ImageNotFound("missing")
with patch("launch.core.platforms.base.docker.from_env", return_value=client), \
patch.dict(os.environ, {name: value}, clear=False):
with pytest.raises(ValueError):
BaseRuntime.pull_image("example/image:latest")
client.images.pull.assert_not_called()
32 changes: 32 additions & 0 deletions tests/runtime_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import os
import platform as host_platform
import warnings
from types import SimpleNamespace

import docker
import pytest
Expand Down Expand Up @@ -170,6 +171,37 @@ def test_runtime_constructor_attributes(runtime_cls, expected_attrs, patch_runti



def test_linux_apply_patch_uses_posix_container_path(tmp_path):
runtime = LinuxRuntime.__new__(LinuxRuntime)
runtime.mnt_host = str(tmp_path)
runtime.mnt_container = "/mnt_tmp"
commands = []
staged_bytes = []

def fake_send_command(command):
commands.append(command)
if command.startswith("git apply"):
staged_files = list(tmp_path.glob("*.diff"))
assert len(staged_files) == 1
staged_bytes.append(staged_files[0].read_bytes())
return SimpleNamespace(
output="",
metadata=SimpleNamespace(exit_code=0),
)

patch = "diff --git a/a b/a\n@@ -1 +1 @@\n-old\n+new\n"
runtime.send_command = fake_send_command
assert runtime.apply_patch(patch, verbose=False) is True
assert commands[0].startswith("git apply")
assert "/mnt_tmp/" in commands[0]
assert "\\" not in commands[0]
assert staged_bytes == [patch.encode("utf-8")]
# apply_patch must remove the host-side staging file as well as the
# container-side file, so a later evaluator provenance check is not dirtied.
assert not list(tmp_path.glob("*.diff"))
runtime.stopped = True


def supported_integration_platforms() -> set[str]:
system = host_platform.system().lower()
if system == "windows":
Expand Down