From 73862e1ae789c9cde08ac4d4d496d5285ca69670 Mon Sep 17 00:00:00 2001 From: akhatua2 Date: Fri, 14 Aug 2026 22:25:23 -0700 Subject: [PATCH 1/2] coop: stop a pair dying with one of its agents A pair is graded only when both sides return, so one agent's death loses the pair. Three things had to line up wrong for that, and all three did. Sandbox lifetimes were 3600s while a heavy reasoner needs ~4x the wall-clock per step, so agents were still working when Modal reclaimed them. Four sites pinned the value and the explicit one in adapter.py quietly beat the dataclass default, so raising that default alone did nothing. The git daemon expired on the same clock, and it hosts the bare repo both agents push and fetch through -- one expiry breaks every git operation at once and leaves agents chasing commits the remote no longer has (`fatal: invalid object name`). All now 10800. Departure was self-reported and only on a clean submit, so a crash or step-limit exit left the peer's has_exited() False forever. Now announced from a finally, on every path. That still cannot cover the case that actually happens: the sandbox is reclaimed, the process is killed outright, and no finally runs. Liveness is therefore proven rather than death announced -- agents refresh an `:alive` key each step and a lapsed heartbeat reads as gone. An agent that had not started yet is never mistaken for one that died, and the survivor is told which of the two occurred instead of being informed that a killed peer "completed their work". Measured on the run that prompted this: one survivor issued 42 sleeps totalling 91 minutes, received zero exit notices, and outlived its own sandbox waiting. --- CHANGELOG.md | 43 +++++++++++++ .../agents/mini_swe_agent_v2/adapter.py | 2 +- .../mini_swe_agent_v2/agents/default.py | 63 ++++++++++++------- .../connectors/git_servers/__init__.py | 2 +- .../connectors/git_servers/docker.py | 2 +- .../connectors/git_servers/gcp.py | 2 +- .../connectors/git_servers/modal.py | 4 +- .../mini_swe_agent_v2/connectors/messaging.py | 44 ++++++++++++- .../mini_swe_agent_v2/environments/modal.py | 4 +- 9 files changed, 134 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea707b48..3b9a82f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.29] - 2026-08-14 + +### Fixed + +- **All 199 features are now gradeable.** The property the benchmark rests on — a feature's tests + must FAIL on the base commit and PASS with its gold patch — did not hold for 24 of them, and + none were findable by reading the specs: 12 had a `runner.sh` that never invoked the feature's + own tests, 8 broke on dependency drift, 5 carried feature 1's expectations in every test patch, + 2 had tests that did not discriminate, 1 had overlapping hunks. `pallets_jinja/1621` f5 passed + 6/6 on an untouched tree, so it scored for any submission at all. No `feature.patch` was + modified: where a fix had a choice it went to the spec or the tests, never the reference. + `scripts/check_gradeable.py` reproduces the sweep. Reasoning per feature is in + `dataset/SPEC_AUDIT.md`. + +- **Sandbox setup no longer deletes the build output the images pre-compile.** `test_merged` ran + `git clean -fdx` before each graded feature, and `-x` removes gitignored paths — which is + exactly where the images keep the artifacts they built at the base commit. typst's Dockerfile + runs `cargo build --package typst-tests --tests` for this purpose and its `runner.sh` already + said `git clean -fd # No -x to preserve target/`; the harness overrode both, costing 335 crate + compiles on every graded run. + +- **Agent and git-daemon sandboxes live 3 hours instead of 1.** A heavy reasoner generates ~4x the + tokens per step, so agents were still working when Modal reclaimed their sandbox at 3600s. Four + separate sites pinned the old value and the explicit one in `adapter.py` silently beat the + dataclass default. The git daemon expiring is the worse half: it hosts the bare repo both agents + push and fetch through, so one expiry breaks every git operation in the pair at once and leaves + agents chasing commits the remote no longer has (`fatal: invalid object name`). + +- **An agent announces its departure on every exit path.** `mark_exited()` was only called after a + clean submit, so a crash or a step-limit exit left the peer's `has_exited()` False forever. The + peer then waited on someone who was never coming back: one agent issued 42 `sleep` commands + totalling 91 minutes, received zero exit notices, and outlived its own sandbox doing it. Because + a pair is only graded when both sides return, a single silent death loses the pair. + +- **A peer that is killed is now detected, not just one that leaves politely.** Departure was + entirely self-reported, which cannot work for the case that actually happens — the sandbox is + reclaimed, the process is killed outright, and no `finally` runs. Agents now refresh an `:alive` + key every step and `has_exited()` treats a lapsed heartbeat as gone, so silence is the signal + and death needs no cooperation from the dead. `is_unreachable()` separates the two, and the + message injected into the survivor's history says which happened rather than claiming a killed + peer "completed their work" — an agent told something untrue about the remote acts on it. A peer + that has not started yet is never mistaken for one that has died. + ## [0.0.28] - 2026-08-10 ### Fixed diff --git a/src/cooperbench/agents/mini_swe_agent_v2/adapter.py b/src/cooperbench/agents/mini_swe_agent_v2/adapter.py index 3b8dc1f1..f41ca2d6 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/adapter.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/adapter.py @@ -148,7 +148,7 @@ def run( env_kwargs = { "image": image, "cwd": "/workspace/repo", - "timeout": 3600, + "timeout": 10800, } container_env = dict(env_cfg.get("env") or {}) # In team mode, propagate the CB_TEAM_* env vars into every diff --git a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py index 09a1ef6a..4bb28348 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py @@ -190,25 +190,33 @@ def run(self, task: str = "", **kwargs) -> dict: self.model.format_message(role="system", content=self._render_template(self.config.system_template)), self.model.format_message(role="user", content=self._render_template(self.config.instance_template)), ) - while True: - try: - self.step() - except InterruptAgentFlow as e: - self.add_messages(*e.messages) - except Exception as e: - self.handle_uncaught_exception(e) - raise - finally: - self.save(self.config.output_path) - if self.messages[-1].get("role") == "exit": - if self._nudge_unsubmitted(): - continue - if self.comm: - # `published` means "the peer can see my work on the remote". That is now - # true exactly when the agent opened a PR, which it does itself -- there - # is no separate publish step to perform on its behalf. - self.comm.mark_exited(published=self._opened_pr()) - break + try: + while True: + try: + self.step() + except InterruptAgentFlow as e: + self.add_messages(*e.messages) + except Exception as e: + self.handle_uncaught_exception(e) + raise + finally: + self.save(self.config.output_path) + if self.messages[-1].get("role") == "exit": + if self._nudge_unsubmitted(): + continue + break + finally: + # Announce departure on EVERY exit path, not just a clean submit. A crash or a + # step-limit exit used to leave this unset, so the peer's has_exited() stayed False + # and it waited on someone who was never coming back -- one agent burned 91 minutes + # of sleep that way and outlived its own sandbox. + if self.comm: + # `published` means "the peer can see my work on the remote", i.e. a PR is open. + try: + published = self._opened_pr() + except Exception: + published = False + self.comm.mark_exited(published=published) return self.messages[-1].get("extra", {}) MAX_SUBMIT_NUDGES = 2 @@ -278,6 +286,8 @@ def step(self) -> list[dict]: and (in team mode) the shared task list before querying.""" # Check for inter-agent messages before querying LLM if self.comm: + if hasattr(self.comm, "heartbeat"): + self.comm.heartbeat() messages = self.comm.receive() for msg in messages: ts = msg.get("timestamp", "")[:19].replace("T", " ") @@ -472,14 +482,19 @@ def _announce_departed_peers(self) -> None: if not self.comm.has_exited(peer): continue announced.add(peer) - self.log(f"PEER EXITED: {peer}") + gone = getattr(self.comm, "is_unreachable", None) and self.comm.is_unreachable(peer) + self.log(f"PEER {'UNREACHABLE' if gone else 'EXITED'}: {peer}") + headline = ( + f"[{peer} is no longer running] They stopped without submitting, so nothing " + f"further will arrive from them." + if gone else + f"[{peer} has completed their work and exited] They will not read or " + f"answer further messages." + ) self.add_messages( self.model.format_message( role="user", - content=( - f"[{peer} has completed their work and exited] They will not read or " - f"answer further messages.\n\n{self._peer_work_pointer(peer)}" - ), + content=f"{headline}\n\n{self._peer_work_pointer(peer)}", ) ) diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/__init__.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/__init__.py index 6f4b765d..4d325e6c 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/__init__.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/__init__.py @@ -26,7 +26,7 @@ def create_git_server( run_id: str, *, app: modal.App | None = None, - timeout: int = 3600, + timeout: int = 10800, # GCP-specific options project_id: str | None = None, zone: str = "us-central1-a", diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/docker.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/docker.py index 9d5c8ecd..d50780d8 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/docker.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/docker.py @@ -124,7 +124,7 @@ def __init__(self, *, run_id: str, hostname: str, port: int, network_name: str): self._logger = logging.getLogger("cooperbench.agents.mini_swe_agent_v2.git_server.docker") @classmethod - def create(cls, run_id: str, timeout: int = 3600) -> DockerGitServer: + def create(cls, run_id: str, timeout: int = 10800) -> DockerGitServer: """Ensure shared infra is up, then init a per-run bare repo on it. Args: diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/gcp.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/gcp.py index a3e41f52..67cd6a43 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/gcp.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/gcp.py @@ -58,7 +58,7 @@ def create( zone: str = "us-central1-a", machine_type: str = "e2-micro", network: str | None = None, - timeout: int = 3600, + timeout: int = 10800, ) -> GCPGitServer: """Create and start a git server VM. diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/modal.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/modal.py index fc8fc748..b4a6f7d2 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/modal.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/modal.py @@ -28,7 +28,9 @@ def create( cls, app: modal.App, run_id: str, - timeout: int = 3600, + # Must outlive the agents: this is the shared remote they push and fetch through, so + # when it expires mid-run every git operation in the pair fails at once. + timeout: int = 10800, ) -> ModalGitServer: """Create and start a git server sandbox. diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/messaging.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/messaging.py index 11017a12..198f7992 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/messaging.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/messaging.py @@ -27,6 +27,10 @@ import redis +# Long enough to outlast a slow step -- a single agent turn can run minutes on a long prompt +# plus a test run, and a heartbeat that expires mid-step would declare a working agent dead. +ALIVE_TTL = 600 + class MessagingConnector: """Redis-based mailbox messaging between agents.""" @@ -51,14 +55,32 @@ def __init__(self, agent_id: str, agents: list[str], url: str = "redis://localho self._client = redis.from_url(url) self._inbox_key = f"{self._prefix}{agent_id}:inbox" + self._seen_alive: set[str] = set() # Clear stale messages from previous runs self._client.delete(self._inbox_key) self._client.delete(self._exited_key(agent_id)) + self.heartbeat() def _exited_key(self, agent_id: str) -> str: return f"{self._prefix}{agent_id}:exited" + def _alive_key(self, agent_id: str) -> str: + return f"{self._prefix}{agent_id}:alive" + + def heartbeat(self) -> None: + """Refresh this agent's liveness key. + + `mark_exited` is self-reported, so it cannot fire when the sandbox is reclaimed -- + the process is killed outright and no `finally` runs. The peer then waits forever on + someone who is already gone. A key that must be refreshed inverts that: silence is + the signal, so death needs no cooperation from the dead. + """ + try: + self._client.setex(self._alive_key(self.agent_id), ALIVE_TTL, "1") + except redis.RedisError: # never let bookkeeping take down a run + pass + def mark_exited(self, published: bool = False) -> None: """Record that this agent has finished, so peers stop waiting on it. @@ -76,9 +98,27 @@ def mark_exited(self, published: bool = False) -> None: pass def has_exited(self, agent_id: str) -> bool: - """True when ``agent_id`` has finished its work and left.""" + """True when ``agent_id`` is gone -- whether it said so or simply stopped. + + Only report a lapsed heartbeat for an agent we have actually seen alive, so a peer + that has not started yet is never mistaken for one that has died. + """ + try: + if self._client.exists(self._exited_key(agent_id)): + return True + if self._client.exists(self._alive_key(agent_id)): + self._seen_alive.add(agent_id) + return False + return agent_id in self._seen_alive + except redis.RedisError: + return False + + def is_unreachable(self, agent_id: str) -> bool: + """Gone WITHOUT announcing it, i.e. killed rather than finished.""" try: - return bool(self._client.exists(self._exited_key(agent_id))) + if self._client.exists(self._exited_key(agent_id)): + return False + return agent_id in self._seen_alive and not self._client.exists(self._alive_key(agent_id)) except redis.RedisError: return False diff --git a/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py b/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py index 78a848f0..b9eaa4c7 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py @@ -91,7 +91,9 @@ def _invalidate_image(image_name: str) -> None: class ModalEnvironmentConfig(BaseModel): image: str cwd: str = "/" - timeout: int = 3600 # sandbox lifetime + # Sandbox lifetime. At 3600 a heavy reasoner blew past it mid-run and its partner then hung + # waiting on an agent that no longer existed. + timeout: int = 10800 command_timeout: int = 300 # per command; longest real one observed is ~104s (npm test) env: dict[str, str] = {} max_retries: int = 5 From 5bc8aabdec51d6ac74e9659642d74a8fa33bb95e Mon Sep 17 00:00:00 2001 From: akhatua2 Date: Fri, 14 Aug 2026 22:27:13 -0700 Subject: [PATCH 2/2] style: ruff format --- src/cooperbench/agents/mini_swe_agent_v2/agents/default.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py index 4bb28348..e9f74c2c 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py @@ -487,9 +487,8 @@ def _announce_departed_peers(self) -> None: headline = ( f"[{peer} is no longer running] They stopped without submitting, so nothing " f"further will arrive from them." - if gone else - f"[{peer} has completed their work and exited] They will not read or " - f"answer further messages." + if gone + else f"[{peer} has completed their work and exited] They will not read or answer further messages." ) self.add_messages( self.model.format_message(