diff --git a/benchmarks/swebench/Dockerfile.grading-entrypoint b/benchmarks/swebench/Dockerfile.grading-entrypoint new file mode 100644 index 000000000..90863c5b4 --- /dev/null +++ b/benchmarks/swebench/Dockerfile.grading-entrypoint @@ -0,0 +1,24 @@ +ARG SDK_IMAGE +FROM ${SDK_IMAGE} + +# SWE-Bench's grading harness (swebench.harness.docker_build.build_container) +# creates the instance container with `command="tail -f /dev/null"` and does +# NOT override `entrypoint`. Docker only replaces a container's command with +# the caller-supplied one when the image itself has no ENTRYPOINT; if an +# ENTRYPOINT is set, the supplied command is appended as arguments to it +# instead. +# +# The eval-agent-server image's ENTRYPOINT starts the OpenHands agent server +# (`tini -- python -m openhands.agent_server`), so when that image is reused +# as a SWE-Bench grading image, the harness's `tail -f /dev/null` ends up +# appended as bogus argv to the agent server rather than replacing it. The +# agent server then runs as PID 1 with unexpected arguments outside its +# normal SDK-orchestrated lifecycle, which can exit or crash, taking the +# whole container down mid-grading (typically right after the test script +# finishes, when the harness issues a follow-up `docker exec` for a +# post-test `git diff` that then fails with "container is not running"). +# +# Clearing ENTRYPOINT here lets the harness's plain `tail -f /dev/null` +# command run standalone as the container's real PID 1, exactly as a +# regular SWE-Bench grading image expects. +ENTRYPOINT [] diff --git a/benchmarks/swebench/README.md b/benchmarks/swebench/README.md index 05517ad72..43a0176c1 100644 --- a/benchmarks/swebench/README.md +++ b/benchmarks/swebench/README.md @@ -254,6 +254,21 @@ uv run swebench-eval output.jsonl \ uv run swebench-eval output.jsonl --skip-evaluation ``` +**Reusing local agent-server images for `--no-modal` grading:** + +```bash +uv run swebench-eval output.jsonl --no-modal --reuse-local-agent-images +``` + +If you already built local eval-agent-server images for inference +(`--workspace docker`), pass `--reuse-local-agent-images` to alias them +under the official SWE-Bench grading image tags before evaluation starts, +instead of pulling/building the (functionally equivalent) official images +from Docker Hub. Instances without a matching local eval-agent-server image +fall back to SWE-Bench's normal pull/build behavior, so this is safe to +leave on. Has no effect with `--modal` or `--apptainer`, since neither goes +through `swebench.harness.docker_build`. + **Local Apptainer evaluation:** ```bash diff --git a/benchmarks/swebench/build_images.py b/benchmarks/swebench/build_images.py index 2b04b2cdd..93b5442e7 100644 --- a/benchmarks/swebench/build_images.py +++ b/benchmarks/swebench/build_images.py @@ -14,7 +14,9 @@ """ import argparse +import json import os +import subprocess import sys from pathlib import Path @@ -26,12 +28,16 @@ ) from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE from benchmarks.utils.dataset import get_dataset -from benchmarks.utils.image_utils import remote_image_exists +from benchmarks.utils.image_utils import local_image_exists, remote_image_exists +from benchmarks.utils.version import get_phased_image_tag_prefix from openhands.sdk import get_logger logger = get_logger(__name__) WRAPPER_DOCKERFILE = Path(__file__).with_name("Dockerfile.swebench-deps") +GRADING_ENTRYPOINT_DOCKERFILE = Path(__file__).with_name( + "Dockerfile.grading-entrypoint" +) def get_official_docker_image( @@ -138,6 +144,141 @@ def wrap_image(agent_image: str, push: bool = False) -> BuildOutput: ) +def prepare_local_grading_image(agent_image: str, grading_image: str) -> BuildOutput: + """ + Alias a locally-built agent-server image under the official SWE-Bench + grading image tag, with its ENTRYPOINT cleared. + + This lets `swebench-eval --no-modal` reuse an already-built local + eval-agent-server image as the SWE-Bench grading image for that + instance, instead of pulling/building the (functionally equivalent) + official image from Docker Hub. See Dockerfile.grading-entrypoint for + why the ENTRYPOINT needs clearing. + + Requires `agent_image` to already exist locally; this function never + builds or pulls the agent-server image itself. + + Deliberately uses plain `docker build` rather than + run_docker_build_layer()'s `docker buildx build`: this is a local-only + alias (never pushed), and buildx builds route through whichever buildx + builder instance is currently active. A `docker-container` driver + instance (e.g. this repo's "openhands-builder", used for the phased + image pipeline) runs BuildKit in its own container with its own image + store, isolated from the host Docker daemon, so it can't resolve a + locally-built-only base image as FROM. Plain `docker build` always uses + the host daemon's own image store and has no such isolation. + """ + if not local_image_exists(agent_image): + return BuildOutput( + base_image=agent_image, + tags=[], + error=f"Agent-server image {agent_image} not found locally.", + status="failed", + ) + + if not GRADING_ENTRYPOINT_DOCKERFILE.exists(): + return BuildOutput( + base_image=agent_image, + tags=[], + error=( + "Grading entrypoint Dockerfile not found at " + f"{GRADING_ENTRYPOINT_DOCKERFILE}" + ), + ) + + logger.info("Preparing local grading image %s from %s", grading_image, agent_image) + + cmd = [ + "docker", + "build", + "--file", + str(GRADING_ENTRYPOINT_DOCKERFILE), + "--build-arg", + f"SDK_IMAGE={agent_image}", + "--tag", + grading_image, + str(GRADING_ENTRYPOINT_DOCKERFILE.parent), + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + return BuildOutput( + base_image=agent_image, + tags=[], + error=( + f"docker build failed (exit {result.returncode}): " + f"{result.stderr.strip()}" + ), + status="failed", + ) + + return BuildOutput(base_image=agent_image, tags=[grading_image], error=None) + + +def prepare_local_grading_images_for_predictions( + predictions_file: Path, +) -> dict[str, bool]: + """ + Prepare local SWE-Bench grading images for every instance in a + predictions file, reusing already-built local eval-agent-server images + where available. + + For each instance: + - If the official SWE-Bench grading image already exists locally, + nothing to do (skipped, counted as not-prepared). + - Else, if a matching local eval-agent-server image exists (same tag + prefix / build target run_infer.py would have built), alias it + under the official grading image tag via + prepare_local_grading_image(). + - Else, nothing to prepare; the instance is left for SWE-Bench's + normal pull/build behavior during grading. + + This is purely a local speed/bandwidth optimization for `swebench-eval + --no-modal`; it has no effect on correctness, since the grading + environment itself (everything but ENTRYPOINT) is identical to the + official image the eval-agent-server image was built from. + + Returns a dict mapping instance_id -> whether an image was prepared. + """ + prefix = get_phased_image_tag_prefix() + target = constants.DEFAULT_BUILD_TARGET + suffix = f"-{target}" if target != constants.BUILD_TARGET_BINARY else "" + + instance_ids: list[str] = [] + with open(predictions_file) as f: + for line in f: + line = line.strip() + if not line: + continue + data = json.loads(line) + instance_id = data.get("instance_id") + if instance_id: + instance_ids.append(instance_id) + + results: dict[str, bool] = {} + for instance_id in instance_ids: + grading_image = get_official_docker_image(instance_id) + if local_image_exists(grading_image): + results[instance_id] = False + continue + + custom_tag = extract_custom_tag(grading_image) + agent_image = f"{EVAL_AGENT_SERVER_IMAGE}:{prefix}-{custom_tag}{suffix}" + if not local_image_exists(agent_image): + results[instance_id] = False + continue + + output = prepare_local_grading_image(agent_image, grading_image) + results[instance_id] = output.error is None + if output.error: + logger.warning( + "Failed to prepare local grading image for %s: %s", + instance_id, + output.error, + ) + + return results + + def get_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Build agent-server images using the three-phase pipeline." diff --git a/benchmarks/swebench/eval_infer.py b/benchmarks/swebench/eval_infer.py index d1ad104ee..e8603fc32 100644 --- a/benchmarks/swebench/eval_infer.py +++ b/benchmarks/swebench/eval_infer.py @@ -261,6 +261,21 @@ def main() -> None: help="Timeout in seconds for evaluation", ) + parser.add_argument( + "--reuse-local-agent-images", + action="store_true", + help=( + "Before --no-modal evaluation, alias any already-built local " + "eval-agent-server images (with ENTRYPOINT cleared) under the " + "official SWE-Bench grading image tags, so grading reuses them " + "instead of pulling the official images from Docker Hub. " + "Purely a local speed/bandwidth optimization; instances " + "without a local eval-agent-server image fall back to " + "SWE-Bench's normal pull/build behavior. No effect on --modal " + "or --apptainer runs." + ), + ) + parser.add_argument( "--apptainer", action="store_true", @@ -338,6 +353,26 @@ def main() -> None: apptainer_cache=args.apptainer_cache, ) else: + if args.reuse_local_agent_images and not args.modal: + from benchmarks.swebench.build_images import ( + prepare_local_grading_images_for_predictions, + ) + + logger.info( + "Preparing local grading images from already-built " + "eval-agent-server images..." + ) + prep_results = prepare_local_grading_images_for_predictions( + output_file + ) + prepared = sum(1 for v in prep_results.values() if v) + logger.info( + f"Prepared {prepared}/{len(prep_results)} local " + "grading images (instances without a local " + "eval-agent-server image fall back to the normal " + "SWE-Bench pull/build)." + ) + # Run evaluation run_swebench_evaluation( str(output_file), diff --git a/tests/test_swebench_eval_infer.py b/tests/test_swebench_eval_infer.py index 29a1df3c3..e0d5dc539 100644 --- a/tests/test_swebench_eval_infer.py +++ b/tests/test_swebench_eval_infer.py @@ -1,6 +1,7 @@ """Tests for SWE-Bench eval_infer functionality.""" import json +import sys import tempfile from types import SimpleNamespace @@ -12,7 +13,7 @@ TESTS_TIMEOUT, ) -from benchmarks.swebench import apptainer_eval +from benchmarks.swebench import apptainer_eval, eval_infer from benchmarks.swebench.eval_infer import convert_to_swebench_format from benchmarks.utils.constants import MODEL_NAME_OR_PATH @@ -233,3 +234,142 @@ def fake_score_instance( assert report["resolved"] == 1 assert report["resolved_ids"] == ["django__django-1"] assert report["unresolved_ids"] == ["django__django-2"] + + +class TestReuseLocalAgentImagesFlag: + """Tests for the --reuse-local-agent-images CLI flag wiring in main(). + + The flag should trigger prepare_local_grading_images_for_predictions() + before the SWE-Bench harness subprocess runs (--no-modal only), and + should be a no-op when omitted. + """ + + def _write_input_file(self, tmp_path): + input_file = tmp_path / "output.jsonl" + input_file.write_text( + json.dumps( + { + "instance_id": "django__django-1", + "test_result": {"git_patch": "diff"}, + } + ) + + "\n" + ) + return input_file + + def _patch_common(self, monkeypatch, tmp_path, calls): + def fake_run_eval(predictions_file, run_id, *args, **kwargs): + calls["run_eval"] = True + report_filename = f"{MODEL_NAME_OR_PATH}.{run_id}.json" + (tmp_path / report_filename).write_text("{}") + + monkeypatch.setattr(eval_infer, "run_swebench_evaluation", fake_run_eval) + monkeypatch.setattr(eval_infer, "generate_cost_report", lambda *a, **k: None) + monkeypatch.setattr( + eval_infer.LaminarService, + "get", + classmethod( + lambda cls: SimpleNamespace( + update_evaluation_scores=lambda *a, **k: None + ) + ), + ) + + def test_reuse_flag_triggers_local_image_prep(self, tmp_path, monkeypatch): + input_file = self._write_input_file(tmp_path) + calls: dict = {} + self._patch_common(monkeypatch, tmp_path, calls) + + def fake_prepare(predictions_file): + calls["prepared"] = predictions_file + return {"django__django-1": True} + + monkeypatch.setattr( + "benchmarks.swebench.build_images." + "prepare_local_grading_images_for_predictions", + fake_prepare, + ) + monkeypatch.setattr( + sys, + "argv", + [ + "swebench-eval", + str(input_file), + "--run-id", + "test-run", + "--no-modal", + "--reuse-local-agent-images", + ], + ) + + eval_infer.main() + + assert "prepared" in calls + assert calls["run_eval"] is True + + def test_no_reuse_flag_skips_local_image_prep(self, tmp_path, monkeypatch): + input_file = self._write_input_file(tmp_path) + calls: dict = {} + self._patch_common(monkeypatch, tmp_path, calls) + + def fake_prepare(predictions_file): + calls["prepared"] = predictions_file + return {} + + monkeypatch.setattr( + "benchmarks.swebench.build_images." + "prepare_local_grading_images_for_predictions", + fake_prepare, + ) + monkeypatch.setattr( + sys, + "argv", + [ + "swebench-eval", + str(input_file), + "--run-id", + "test-run", + "--no-modal", + ], + ) + + eval_infer.main() + + assert "prepared" not in calls + assert calls["run_eval"] is True + + def test_reuse_flag_has_no_effect_with_modal(self, tmp_path, monkeypatch): + """--reuse-local-agent-images is meaningless for --modal runs (Modal + sandboxes don't go through swebench.harness.docker_build at all), so + it should not trigger local image prep in that mode. + """ + input_file = self._write_input_file(tmp_path) + calls: dict = {} + self._patch_common(monkeypatch, tmp_path, calls) + + def fake_prepare(predictions_file): + calls["prepared"] = predictions_file + return {} + + monkeypatch.setattr( + "benchmarks.swebench.build_images." + "prepare_local_grading_images_for_predictions", + fake_prepare, + ) + monkeypatch.setattr( + sys, + "argv", + [ + "swebench-eval", + str(input_file), + "--run-id", + "test-run", + "--modal", + "--reuse-local-agent-images", + ], + ) + + eval_infer.main() + + assert "prepared" not in calls + assert calls["run_eval"] is True diff --git a/tests/test_swebench_grading_images.py b/tests/test_swebench_grading_images.py new file mode 100644 index 000000000..5ef659d4f --- /dev/null +++ b/tests/test_swebench_grading_images.py @@ -0,0 +1,204 @@ +"""Tests for benchmarks.swebench.build_images local grading image helpers. + +Covers prepare_local_grading_image() and +prepare_local_grading_images_for_predictions(), which let `swebench-eval +--no-modal` reuse already-built local eval-agent-server images as SWE-Bench +grading images (with ENTRYPOINT cleared) instead of pulling/building the +official images from Docker Hub. +""" + +import json +import tempfile +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from benchmarks.utils.build_utils import BuildOutput + + +class TestPrepareLocalGradingImage: + @patch("benchmarks.swebench.build_images.local_image_exists", return_value=False) + def test_errors_when_agent_image_not_local(self, _mock_exists): + from benchmarks.swebench.build_images import prepare_local_grading_image + + result = prepare_local_grading_image( + agent_image="ghcr.io/openhands/eval-agent-server:abc-sweb.eval.x86_64.foo_1776_bar-source-minimal", + grading_image="swebench/sweb.eval.x86_64.foo_1776_bar:latest", + ) + assert result.error is not None + assert "not found locally" in result.error + assert result.tags == [] + + @patch("benchmarks.swebench.build_images.subprocess.run") + @patch("benchmarks.swebench.build_images.local_image_exists", return_value=True) + def test_builds_with_expected_args(self, _mock_exists, mock_run): + """Uses plain `docker build` (not `docker buildx build`): buildx + routes through whichever builder is currently active, and a + docker-container-driver builder can't see images that only exist in + the host daemon's local store. See prepare_local_grading_image()'s + docstring. + """ + from benchmarks.swebench.build_images import prepare_local_grading_image + + agent_image = ( + "ghcr.io/openhands/eval-agent-server:" + "abc-sweb.eval.x86_64.foo_1776_bar-source-minimal" + ) + grading_image = "swebench/sweb.eval.x86_64.foo_1776_bar:latest" + mock_run.return_value = SimpleNamespace(returncode=0, stdout="", stderr="") + + result = prepare_local_grading_image(agent_image, grading_image) + + assert result.error is None + assert result.tags == [grading_image] + (cmd,), kwargs = mock_run.call_args + assert cmd[:2] == ["docker", "build"] + assert "buildx" not in cmd + assert f"SDK_IMAGE={agent_image}" in cmd + assert grading_image in cmd + assert kwargs.get("capture_output") is True + + @patch("benchmarks.swebench.build_images.subprocess.run") + @patch("benchmarks.swebench.build_images.local_image_exists", return_value=True) + def test_records_docker_build_failure(self, _mock_exists, mock_run): + from benchmarks.swebench.build_images import prepare_local_grading_image + + agent_image = ( + "ghcr.io/openhands/eval-agent-server:" + "abc-sweb.eval.x86_64.foo_1776_bar-source-minimal" + ) + grading_image = "swebench/sweb.eval.x86_64.foo_1776_bar:latest" + mock_run.return_value = SimpleNamespace( + returncode=1, stdout="", stderr="something went wrong" + ) + + result = prepare_local_grading_image(agent_image, grading_image) + + assert result.error is not None + assert "something went wrong" in result.error + assert result.tags == [] + + +class TestPrepareLocalGradingImagesForPredictions: + def _write_predictions(self, instance_ids: list[str]) -> Path: + f = tempfile.NamedTemporaryFile( + mode="w", suffix=".jsonl", delete=False, encoding="utf-8" + ) + for iid in instance_ids: + f.write(json.dumps({"instance_id": iid, "model_patch": "diff"}) + "\n") + f.close() + return Path(f.name) + + @patch("benchmarks.swebench.build_images.get_phased_image_tag_prefix") + @patch("benchmarks.swebench.build_images.prepare_local_grading_image") + @patch("benchmarks.swebench.build_images.local_image_exists") + def test_skips_instance_with_official_image_already_local( + self, mock_exists, mock_prepare, mock_prefix + ): + from benchmarks.swebench.build_images import ( + prepare_local_grading_images_for_predictions, + ) + + mock_prefix.return_value = "abc123-def456" + # Official grading image already exists locally -> nothing to do. + mock_exists.return_value = True + + predictions = self._write_predictions(["django__django-11333"]) + try: + results = prepare_local_grading_images_for_predictions(predictions) + finally: + predictions.unlink() + + assert results == {"django__django-11333": False} + mock_prepare.assert_not_called() + + @patch("benchmarks.swebench.build_images.get_phased_image_tag_prefix") + @patch("benchmarks.swebench.build_images.prepare_local_grading_image") + @patch("benchmarks.swebench.build_images.local_image_exists") + def test_skips_instance_with_no_local_agent_image( + self, mock_exists, mock_prepare, mock_prefix + ): + from benchmarks.swebench.build_images import ( + prepare_local_grading_images_for_predictions, + ) + + mock_prefix.return_value = "abc123-def456" + # Neither the official grading image nor the agent-server image + # exist locally -> nothing to prepare, fall back to SWE-Bench. + mock_exists.return_value = False + + predictions = self._write_predictions(["django__django-11333"]) + try: + results = prepare_local_grading_images_for_predictions(predictions) + finally: + predictions.unlink() + + assert results == {"django__django-11333": False} + mock_prepare.assert_not_called() + + @patch("benchmarks.swebench.build_images.get_phased_image_tag_prefix") + @patch("benchmarks.swebench.build_images.prepare_local_grading_image") + @patch("benchmarks.swebench.build_images.local_image_exists") + def test_prepares_instance_with_local_agent_image( + self, mock_exists, mock_prepare, mock_prefix + ): + from benchmarks.swebench.build_images import ( + prepare_local_grading_images_for_predictions, + ) + + mock_prefix.return_value = "abc123-def456" + grading_image = ( + "docker.io/swebench/sweb.eval.x86_64.django_1776_django-11333:latest" + ) + agent_image = ( + "ghcr.io/openhands/eval-agent-server:" + "abc123-def456-sweb.eval.x86_64.django_1776_django-11333-source-minimal" + ) + # Official image missing locally, but the agent-server image is + # present -> should prepare. + mock_exists.side_effect = lambda image: image == agent_image + mock_prepare.return_value = BuildOutput( + base_image=agent_image, + tags=[grading_image], + error=None, + ) + + predictions = self._write_predictions(["django__django-11333"]) + try: + results = prepare_local_grading_images_for_predictions(predictions) + finally: + predictions.unlink() + + assert results == {"django__django-11333": True} + mock_prepare.assert_called_once_with(agent_image, grading_image) + + @patch("benchmarks.swebench.build_images.get_phased_image_tag_prefix") + @patch("benchmarks.swebench.build_images.prepare_local_grading_image") + @patch("benchmarks.swebench.build_images.local_image_exists") + def test_records_failure_without_raising( + self, mock_exists, mock_prepare, mock_prefix + ): + from benchmarks.swebench.build_images import ( + prepare_local_grading_images_for_predictions, + ) + + mock_prefix.return_value = "abc123-def456" + agent_image = ( + "ghcr.io/openhands/eval-agent-server:" + "abc123-def456-sweb.eval.x86_64.django_1776_django-11333-source-minimal" + ) + mock_exists.side_effect = lambda image: image == agent_image + mock_prepare.return_value = BuildOutput( + base_image=agent_image, + tags=[], + error="docker build failed", + ) + + predictions = self._write_predictions(["django__django-11333"]) + try: + results = prepare_local_grading_images_for_predictions(predictions) + finally: + predictions.unlink() + + # Failure to prepare shouldn't raise; caller falls back to SWE-Bench. + assert results == {"django__django-11333": False}