feat: reuse local eval-agent-server images for --no-modal SWE-Bench grading

benchmarks/swebench/Dockerfile.grading-entrypoint

@@ -0,0 +1,24 @@
+ARG SDK_IMAGE
+FROM ${SDK_IMAGE}
+
+# SWE-Bench's grading harness (swebench.harness.docker_build.build_container)
+# creates the instance container with `command="tail -f /dev/null"` and does
+# NOT override `entrypoint`. Docker only replaces a container's command with
+# the caller-supplied one when the image itself has no ENTRYPOINT; if an
+# ENTRYPOINT is set, the supplied command is appended as arguments to it
+# instead.
+#
+# The eval-agent-server image's ENTRYPOINT starts the OpenHands agent server
+# (`tini -- python -m openhands.agent_server`), so when that image is reused
+# as a SWE-Bench grading image, the harness's `tail -f /dev/null` ends up
+# appended as bogus argv to the agent server rather than replacing it. The
+# agent server then runs as PID 1 with unexpected arguments outside its
+# normal SDK-orchestrated lifecycle, which can exit or crash, taking the
+# whole container down mid-grading (typically right after the test script
+# finishes, when the harness issues a follow-up `docker exec` for a
+# post-test `git diff` that then fails with "container is not running").
+#
+# Clearing ENTRYPOINT here lets the harness's plain `tail -f /dev/null`
+# command run standalone as the container's real PID 1, exactly as a
+# regular SWE-Bench grading image expects.
+ENTRYPOINT []

benchmarks/swebench/README.md

@@ -254,6 +254,21 @@ uv run swebench-eval output.jsonl \
 uv run swebench-eval output.jsonl --skip-evaluation
 ```
 
+**Reusing local agent-server images for `--no-modal` grading:**
+
+```bash
+uv run swebench-eval output.jsonl --no-modal --reuse-local-agent-images
+```
+
+If you already built local eval-agent-server images for inference
+(`--workspace docker`), pass `--reuse-local-agent-images` to alias them
+under the official SWE-Bench grading image tags before evaluation starts,
+instead of pulling/building the (functionally equivalent) official images
+from Docker Hub. Instances without a matching local eval-agent-server image
+fall back to SWE-Bench's normal pull/build behavior, so this is safe to
+leave on. Has no effect with `--modal` or `--apptainer`, since neither goes
+through `swebench.harness.docker_build`.
+
 **Local Apptainer evaluation:**
 
 ```bash

benchmarks/swebench/build_images.py

@@ -14,7 +14,9 @@
 """
 
 import argparse
+import json
 import os
+import subprocess
 import sys
 from pathlib import Path
 
@@ -26,12 +28,16 @@
 )
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.dataset import get_dataset
-from benchmarks.utils.image_utils import remote_image_exists
+from benchmarks.utils.image_utils import local_image_exists, remote_image_exists
+from benchmarks.utils.version import get_phased_image_tag_prefix
 from openhands.sdk import get_logger
 
 
 logger = get_logger(__name__)
 WRAPPER_DOCKERFILE = Path(__file__).with_name("Dockerfile.swebench-deps")
+GRADING_ENTRYPOINT_DOCKERFILE = Path(__file__).with_name(
+    "Dockerfile.grading-entrypoint"
+)
 
 
 def get_official_docker_image(
@@ -138,6 +144,141 @@ def wrap_image(agent_image: str, push: bool = False) -> BuildOutput:
     )
 
 
+def prepare_local_grading_image(agent_image: str, grading_image: str) -> BuildOutput:
+    """
+    Alias a locally-built agent-server image under the official SWE-Bench
+    grading image tag, with its ENTRYPOINT cleared.
+
+    This lets `swebench-eval --no-modal` reuse an already-built local
+    eval-agent-server image as the SWE-Bench grading image for that
+    instance, instead of pulling/building the (functionally equivalent)
+    official image from Docker Hub. See Dockerfile.grading-entrypoint for
+    why the ENTRYPOINT needs clearing.
+
+    Requires `agent_image` to already exist locally; this function never
+    builds or pulls the agent-server image itself.
+
+    Deliberately uses plain `docker build` rather than
+    run_docker_build_layer()'s `docker buildx build`: this is a local-only
+    alias (never pushed), and buildx builds route through whichever buildx
+    builder instance is currently active. A `docker-container` driver
+    instance (e.g. this repo's "openhands-builder", used for the phased
+    image pipeline) runs BuildKit in its own container with its own image
+    store, isolated from the host Docker daemon, so it can't resolve a
+    locally-built-only base image as FROM. Plain `docker build` always uses
+    the host daemon's own image store and has no such isolation.
+    """
+    if not local_image_exists(agent_image):
+        return BuildOutput(
+            base_image=agent_image,
+            tags=[],
+            error=f"Agent-server image {agent_image} not found locally.",
+            status="failed",
+        )
+
+    if not GRADING_ENTRYPOINT_DOCKERFILE.exists():
+        return BuildOutput(
+            base_image=agent_image,
+            tags=[],
+            error=(
+                "Grading entrypoint Dockerfile not found at "
+                f"{GRADING_ENTRYPOINT_DOCKERFILE}"
+            ),
+        )
+
+    logger.info("Preparing local grading image %s from %s", grading_image, agent_image)
+
+    cmd = [
+        "docker",
+        "build",
+        "--file",
+        str(GRADING_ENTRYPOINT_DOCKERFILE),
+        "--build-arg",
+        f"SDK_IMAGE={agent_image}",
+        "--tag",
+        grading_image,
+        str(GRADING_ENTRYPOINT_DOCKERFILE.parent),
+    ]
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        return BuildOutput(
+            base_image=agent_image,
+            tags=[],
+            error=(
+                f"docker build failed (exit {result.returncode}): "
+                f"{result.stderr.strip()}"
+            ),
+            status="failed",
+        )
+
+    return BuildOutput(base_image=agent_image, tags=[grading_image], error=None)
+
+
+def prepare_local_grading_images_for_predictions(
+    predictions_file: Path,
+) -> dict[str, bool]:
+    """
+    Prepare local SWE-Bench grading images for every instance in a
+    predictions file, reusing already-built local eval-agent-server images
+    where available.
+
+    For each instance:
+      - If the official SWE-Bench grading image already exists locally,
+        nothing to do (skipped, counted as not-prepared).
+      - Else, if a matching local eval-agent-server image exists (same tag
+        prefix / build target run_infer.py would have built), alias it
+        under the official grading image tag via
+        prepare_local_grading_image().
+      - Else, nothing to prepare; the instance is left for SWE-Bench's
+        normal pull/build behavior during grading.
+
+    This is purely a local speed/bandwidth optimization for `swebench-eval
+    --no-modal`; it has no effect on correctness, since the grading
+    environment itself (everything but ENTRYPOINT) is identical to the
+    official image the eval-agent-server image was built from.
+
+    Returns a dict mapping instance_id -> whether an image was prepared.
+    """
+    prefix = get_phased_image_tag_prefix()
+    target = constants.DEFAULT_BUILD_TARGET
+    suffix = f"-{target}" if target != constants.BUILD_TARGET_BINARY else ""
+
+    instance_ids: list[str] = []
+    with open(predictions_file) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            data = json.loads(line)
+            instance_id = data.get("instance_id")
+            if instance_id:
+                instance_ids.append(instance_id)
+
+    results: dict[str, bool] = {}
+    for instance_id in instance_ids:
+        grading_image = get_official_docker_image(instance_id)
+        if local_image_exists(grading_image):
+            results[instance_id] = False
+            continue
+
+        custom_tag = extract_custom_tag(grading_image)
+        agent_image = f"{EVAL_AGENT_SERVER_IMAGE}:{prefix}-{custom_tag}{suffix}"
+        if not local_image_exists(agent_image):
+            results[instance_id] = False
+            continue
+
+        output = prepare_local_grading_image(agent_image, grading_image)
+        results[instance_id] = output.error is None
+        if output.error:
+            logger.warning(
+                "Failed to prepare local grading image for %s: %s",
+                instance_id,
+                output.error,
+            )
+
+    return results
+
+
 def get_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Build agent-server images using the three-phase pipeline."

benchmarks/swebench/eval_infer.py

@@ -261,6 +261,21 @@ def main() -> None:
         help="Timeout in seconds for evaluation",
     )
 
+    parser.add_argument(
+        "--reuse-local-agent-images",
+        action="store_true",
+        help=(
+            "Before --no-modal evaluation, alias any already-built local "
+            "eval-agent-server images (with ENTRYPOINT cleared) under the "
+            "official SWE-Bench grading image tags, so grading reuses them "
+            "instead of pulling the official images from Docker Hub. "
+            "Purely a local speed/bandwidth optimization; instances "
+            "without a local eval-agent-server image fall back to "
+            "SWE-Bench's normal pull/build behavior. No effect on --modal "
+            "or --apptainer runs."
+        ),
+    )
+
     parser.add_argument(
         "--apptainer",
         action="store_true",
@@ -338,6 +353,26 @@ def main() -> None:
                     apptainer_cache=args.apptainer_cache,
                 )
             else:
+                if args.reuse_local_agent_images and not args.modal:
+                    from benchmarks.swebench.build_images import (
+                        prepare_local_grading_images_for_predictions,
+                    )
+
+                    logger.info(
+                        "Preparing local grading images from already-built "
+                        "eval-agent-server images..."
+                    )
+                    prep_results = prepare_local_grading_images_for_predictions(
+                        output_file
+                    )
+                    prepared = sum(1 for v in prep_results.values() if v)
+                    logger.info(
+                        f"Prepared {prepared}/{len(prep_results)} local "
+                        "grading images (instances without a local "
+                        "eval-agent-server image fall back to the normal "
+                        "SWE-Bench pull/build)."
+                    )
+
                 # Run evaluation
                 run_swebench_evaluation(
                     str(output_file),