diff --git a/CHANGELOG.md b/CHANGELOG.md index ea707b48..3b9a82f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.29] - 2026-08-14 + +### Fixed + +- **All 199 features are now gradeable.** The property the benchmark rests on — a feature's tests + must FAIL on the base commit and PASS with its gold patch — did not hold for 24 of them, and + none were findable by reading the specs: 12 had a `runner.sh` that never invoked the feature's + own tests, 8 broke on dependency drift, 5 carried feature 1's expectations in every test patch, + 2 had tests that did not discriminate, 1 had overlapping hunks. `pallets_jinja/1621` f5 passed + 6/6 on an untouched tree, so it scored for any submission at all. No `feature.patch` was + modified: where a fix had a choice it went to the spec or the tests, never the reference. + `scripts/check_gradeable.py` reproduces the sweep. Reasoning per feature is in + `dataset/SPEC_AUDIT.md`. + +- **Sandbox setup no longer deletes the build output the images pre-compile.** `test_merged` ran + `git clean -fdx` before each graded feature, and `-x` removes gitignored paths — which is + exactly where the images keep the artifacts they built at the base commit. typst's Dockerfile + runs `cargo build --package typst-tests --tests` for this purpose and its `runner.sh` already + said `git clean -fd # No -x to preserve target/`; the harness overrode both, costing 335 crate + compiles on every graded run. + +- **Agent and git-daemon sandboxes live 3 hours instead of 1.** A heavy reasoner generates ~4x the + tokens per step, so agents were still working when Modal reclaimed their sandbox at 3600s. Four + separate sites pinned the old value and the explicit one in `adapter.py` silently beat the + dataclass default. The git daemon expiring is the worse half: it hosts the bare repo both agents + push and fetch through, so one expiry breaks every git operation in the pair at once and leaves + agents chasing commits the remote no longer has (`fatal: invalid object name`). + +- **An agent announces its departure on every exit path.** `mark_exited()` was only called after a + clean submit, so a crash or a step-limit exit left the peer's `has_exited()` False forever. The + peer then waited on someone who was never coming back: one agent issued 42 `sleep` commands + totalling 91 minutes, received zero exit notices, and outlived its own sandbox doing it. Because + a pair is only graded when both sides return, a single silent death loses the pair. + +- **A peer that is killed is now detected, not just one that leaves politely.** Departure was + entirely self-reported, which cannot work for the case that actually happens — the sandbox is + reclaimed, the process is killed outright, and no `finally` runs. Agents now refresh an `:alive` + key every step and `has_exited()` treats a lapsed heartbeat as gone, so silence is the signal + and death needs no cooperation from the dead. `is_unreachable()` separates the two, and the + message injected into the survivor's history says which happened rather than claiming a killed + peer "completed their work" — an agent told something untrue about the remote acts on it. A peer + that has not started yet is never mistaken for one that has died. + ## [0.0.28] - 2026-08-10 ### Fixed diff --git a/src/cooperbench/agents/mini_swe_agent_v2/adapter.py b/src/cooperbench/agents/mini_swe_agent_v2/adapter.py index 3b8dc1f1..f41ca2d6 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/adapter.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/adapter.py @@ -148,7 +148,7 @@ def run( env_kwargs = { "image": image, "cwd": "/workspace/repo", - "timeout": 3600, + "timeout": 10800, } container_env = dict(env_cfg.get("env") or {}) # In team mode, propagate the CB_TEAM_* env vars into every diff --git a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py index 09a1ef6a..e9f74c2c 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py @@ -190,25 +190,33 @@ def run(self, task: str = "", **kwargs) -> dict: self.model.format_message(role="system", content=self._render_template(self.config.system_template)), self.model.format_message(role="user", content=self._render_template(self.config.instance_template)), ) - while True: - try: - self.step() - except InterruptAgentFlow as e: - self.add_messages(*e.messages) - except Exception as e: - self.handle_uncaught_exception(e) - raise - finally: - self.save(self.config.output_path) - if self.messages[-1].get("role") == "exit": - if self._nudge_unsubmitted(): - continue - if self.comm: - # `published` means "the peer can see my work on the remote". That is now - # true exactly when the agent opened a PR, which it does itself -- there - # is no separate publish step to perform on its behalf. - self.comm.mark_exited(published=self._opened_pr()) - break + try: + while True: + try: + self.step() + except InterruptAgentFlow as e: + self.add_messages(*e.messages) + except Exception as e: + self.handle_uncaught_exception(e) + raise + finally: + self.save(self.config.output_path) + if self.messages[-1].get("role") == "exit": + if self._nudge_unsubmitted(): + continue + break + finally: + # Announce departure on EVERY exit path, not just a clean submit. A crash or a + # step-limit exit used to leave this unset, so the peer's has_exited() stayed False + # and it waited on someone who was never coming back -- one agent burned 91 minutes + # of sleep that way and outlived its own sandbox. + if self.comm: + # `published` means "the peer can see my work on the remote", i.e. a PR is open. + try: + published = self._opened_pr() + except Exception: + published = False + self.comm.mark_exited(published=published) return self.messages[-1].get("extra", {}) MAX_SUBMIT_NUDGES = 2 @@ -278,6 +286,8 @@ def step(self) -> list[dict]: and (in team mode) the shared task list before querying.""" # Check for inter-agent messages before querying LLM if self.comm: + if hasattr(self.comm, "heartbeat"): + self.comm.heartbeat() messages = self.comm.receive() for msg in messages: ts = msg.get("timestamp", "")[:19].replace("T", " ") @@ -472,14 +482,18 @@ def _announce_departed_peers(self) -> None: if not self.comm.has_exited(peer): continue announced.add(peer) - self.log(f"PEER EXITED: {peer}") + gone = getattr(self.comm, "is_unreachable", None) and self.comm.is_unreachable(peer) + self.log(f"PEER {'UNREACHABLE' if gone else 'EXITED'}: {peer}") + headline = ( + f"[{peer} is no longer running] They stopped without submitting, so nothing " + f"further will arrive from them." + if gone + else f"[{peer} has completed their work and exited] They will not read or answer further messages." + ) self.add_messages( self.model.format_message( role="user", - content=( - f"[{peer} has completed their work and exited] They will not read or " - f"answer further messages.\n\n{self._peer_work_pointer(peer)}" - ), + content=f"{headline}\n\n{self._peer_work_pointer(peer)}", ) ) diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/__init__.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/__init__.py index 6f4b765d..4d325e6c 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/__init__.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/__init__.py @@ -26,7 +26,7 @@ def create_git_server( run_id: str, *, app: modal.App | None = None, - timeout: int = 3600, + timeout: int = 10800, # GCP-specific options project_id: str | None = None, zone: str = "us-central1-a", diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/docker.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/docker.py index 9d5c8ecd..d50780d8 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/docker.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/docker.py @@ -124,7 +124,7 @@ def __init__(self, *, run_id: str, hostname: str, port: int, network_name: str): self._logger = logging.getLogger("cooperbench.agents.mini_swe_agent_v2.git_server.docker") @classmethod - def create(cls, run_id: str, timeout: int = 3600) -> DockerGitServer: + def create(cls, run_id: str, timeout: int = 10800) -> DockerGitServer: """Ensure shared infra is up, then init a per-run bare repo on it. Args: diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/gcp.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/gcp.py index a3e41f52..67cd6a43 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/gcp.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/gcp.py @@ -58,7 +58,7 @@ def create( zone: str = "us-central1-a", machine_type: str = "e2-micro", network: str | None = None, - timeout: int = 3600, + timeout: int = 10800, ) -> GCPGitServer: """Create and start a git server VM. diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/modal.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/modal.py index fc8fc748..b4a6f7d2 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/modal.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git_servers/modal.py @@ -28,7 +28,9 @@ def create( cls, app: modal.App, run_id: str, - timeout: int = 3600, + # Must outlive the agents: this is the shared remote they push and fetch through, so + # when it expires mid-run every git operation in the pair fails at once. + timeout: int = 10800, ) -> ModalGitServer: """Create and start a git server sandbox. diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/messaging.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/messaging.py index 11017a12..198f7992 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/messaging.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/messaging.py @@ -27,6 +27,10 @@ import redis +# Long enough to outlast a slow step -- a single agent turn can run minutes on a long prompt +# plus a test run, and a heartbeat that expires mid-step would declare a working agent dead. +ALIVE_TTL = 600 + class MessagingConnector: """Redis-based mailbox messaging between agents.""" @@ -51,14 +55,32 @@ def __init__(self, agent_id: str, agents: list[str], url: str = "redis://localho self._client = redis.from_url(url) self._inbox_key = f"{self._prefix}{agent_id}:inbox" + self._seen_alive: set[str] = set() # Clear stale messages from previous runs self._client.delete(self._inbox_key) self._client.delete(self._exited_key(agent_id)) + self.heartbeat() def _exited_key(self, agent_id: str) -> str: return f"{self._prefix}{agent_id}:exited" + def _alive_key(self, agent_id: str) -> str: + return f"{self._prefix}{agent_id}:alive" + + def heartbeat(self) -> None: + """Refresh this agent's liveness key. + + `mark_exited` is self-reported, so it cannot fire when the sandbox is reclaimed -- + the process is killed outright and no `finally` runs. The peer then waits forever on + someone who is already gone. A key that must be refreshed inverts that: silence is + the signal, so death needs no cooperation from the dead. + """ + try: + self._client.setex(self._alive_key(self.agent_id), ALIVE_TTL, "1") + except redis.RedisError: # never let bookkeeping take down a run + pass + def mark_exited(self, published: bool = False) -> None: """Record that this agent has finished, so peers stop waiting on it. @@ -76,9 +98,27 @@ def mark_exited(self, published: bool = False) -> None: pass def has_exited(self, agent_id: str) -> bool: - """True when ``agent_id`` has finished its work and left.""" + """True when ``agent_id`` is gone -- whether it said so or simply stopped. + + Only report a lapsed heartbeat for an agent we have actually seen alive, so a peer + that has not started yet is never mistaken for one that has died. + """ + try: + if self._client.exists(self._exited_key(agent_id)): + return True + if self._client.exists(self._alive_key(agent_id)): + self._seen_alive.add(agent_id) + return False + return agent_id in self._seen_alive + except redis.RedisError: + return False + + def is_unreachable(self, agent_id: str) -> bool: + """Gone WITHOUT announcing it, i.e. killed rather than finished.""" try: - return bool(self._client.exists(self._exited_key(agent_id))) + if self._client.exists(self._exited_key(agent_id)): + return False + return agent_id in self._seen_alive and not self._client.exists(self._alive_key(agent_id)) except redis.RedisError: return False diff --git a/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py b/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py index 78a848f0..b9eaa4c7 100644 --- a/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py +++ b/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py @@ -91,7 +91,9 @@ def _invalidate_image(image_name: str) -> None: class ModalEnvironmentConfig(BaseModel): image: str cwd: str = "/" - timeout: int = 3600 # sandbox lifetime + # Sandbox lifetime. At 3600 a heavy reasoner blew past it mid-run and its partner then hung + # waiting on an agent that no longer existed. + timeout: int = 10800 command_timeout: int = 300 # per command; longest real one observed is ~104s (npm test) env: dict[str, str] = {} max_retries: int = 5