diff --git a/.gitignore b/.gitignore
index 343f9bc1..0f12bdda 100644
--- a/.gitignore
+++ b/.gitignore
@@ -68,3 +68,6 @@ dataset/**/requirements_validated.txt
 **/agent_workspace/
 .claude/
 workspace/
+
+# Regenerated by scripts/check_gradeable.py on every run
+dataset/gradeable_report*.json
diff --git a/dataset/.spec_audit_verdicts.json b/dataset/.spec_audit_verdicts.json
new file mode 100644
index 00000000..56d6ce6a
--- /dev/null
+++ b/dataset/.spec_audit_verdicts.json
@@ -0,0 +1,798 @@
+{
+ "dottxt_ai_outlines_task/task1371|feature1": [
+  "OK",
+  "custom filters API named"
+ ],
+ "dottxt_ai_outlines_task/task1371|feature2": [
+  "OK",
+  "bare `Exception` in the test is a catch-all any error satisfies"
+ ],
+ "dottxt_ai_outlines_task/task1371|feature3": [
+  "OK",
+  "conditional filter loading described"
+ ],
+ "dottxt_ai_outlines_task/task1371|feature4": [
+  "OK",
+  "built-in filter extensions named"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature1": [
+  "OK",
+  "regex type: format described and the type name given"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature10": [
+  "OK",
+  "over-complete \u2014 spec contains the answer verbatim"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature2": [
+  "OK",
+  "regex type: format described and the type name given"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature3": [
+  "OK",
+  "regex type: format described and the type name given"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature4": [
+  "OK",
+  "regex type: format described and the type name given"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature5": [
+  "OK",
+  "regex type: format described and the type name given"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature6": [
+  "OK",
+  "regex type: format described and the type name given"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature7": [
+  "OK",
+  "every assertion derivable; \"xor\" is exact"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature8": [
+  "OK",
+  "format-only is explicit ('format' x3, XXX-XX-XXXX given) \u2014 note 000-00-0000 and 999-99-9999 are valid, so real SSN area rules must NOT be applied"
+ ],
+ "dottxt_ai_outlines_task/task1655|feature9": [
+  "OK",
+  "regex type: format described and the type name given"
+ ],
+ "dottxt_ai_outlines_task/task1706|feature1": [
+  "OK",
+  "'\u2026is not available' is pre-existing outlines text (0 occurrences in gold)"
+ ],
+ "dottxt_ai_outlines_task/task1706|feature2": [
+  "OK",
+  "cache_compiled_grammars and the processor API both named"
+ ],
+ "dottxt_ai_outlines_task/task1706|feature3": [
+  "OK",
+  "`temperature` named; only pre-existing messages asserted"
+ ],
+ "dottxt_ai_outlines_task/task1706|feature4": [
+  "SPEC",
+  "exact batch-size ValueError text"
+ ],
+ "dottxt_ai_outlines_task/task1706|feature5": [
+  "SPEC",
+  "exact custom_adapter ValueError texts \u2014 conditions stated, wording was not"
+ ],
+ "dottxt_ai_outlines_task/task1706|feature6": [
+  "OK",
+  "the matched messages come from the test's own validator, not from gold"
+ ],
+ "dottxt_ai_outlines_task/task1706|feature7": [
+  "OK",
+  "'Grammar error' is raised by the test's own stub; fallback API named"
+ ],
+ "dottxt_ai_outlines_task/task1706|feature8": [
+  "OK",
+  "all four memory attributes named in the spec"
+ ],
+ "dspy_task/task8394|feature1": [
+  "OK",
+  "bypass() context manager named and described"
+ ],
+ "dspy_task/task8394|feature2": [
+  "OK",
+  "over-complete \u2014 gives the exact ns_hash formula and key format"
+ ],
+ "dspy_task/task8394|feature3": [
+  "OK",
+  "every assertion maps to a stated sentence"
+ ],
+ "dspy_task/task8394|feature4": [
+  "OK",
+  "both candidate gaps derivable; amendment reverted"
+ ],
+ "dspy_task/task8394|feature5": [
+  "OK",
+  "compression + magic-header format stated in the spec"
+ ],
+ "dspy_task/task8563|feature1": [
+  "OK",
+  "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation"
+ ],
+ "dspy_task/task8563|feature2": [
+  "OK",
+  "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation"
+ ],
+ "dspy_task/task8563|feature3": [
+  "OK",
+  "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation"
+ ],
+ "dspy_task/task8563|feature4": [
+  "OK",
+  "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation"
+ ],
+ "dspy_task/task8563|feature5": [
+  "OK",
+  "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation"
+ ],
+ "dspy_task/task8563|feature6": [
+  "OK",
+  "ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation"
+ ],
+ "dspy_task/task8587|feature1": [
+  "OK",
+  "nothing flagged \u2014 streaming API fully named in each spec"
+ ],
+ "dspy_task/task8587|feature2": [
+  "OK",
+  "nothing flagged \u2014 streaming API fully named in each spec"
+ ],
+ "dspy_task/task8587|feature3": [
+  "OK",
+  "nothing flagged \u2014 streaming API fully named in each spec"
+ ],
+ "dspy_task/task8587|feature4": [
+  "OK",
+  "nothing flagged \u2014 streaming API fully named in each spec"
+ ],
+ "dspy_task/task8587|feature5": [
+  "OK",
+  "nothing flagged \u2014 streaming API fully named in each spec"
+ ],
+ "dspy_task/task8587|feature6": [
+  "OK",
+  "nothing flagged \u2014 streaming API fully named in each spec"
+ ],
+ "dspy_task/task8635|feature1": [
+  "OK",
+  "nothing flagged \u2014 proposer params fully named in each spec"
+ ],
+ "dspy_task/task8635|feature2": [
+  "OK",
+  "nothing flagged \u2014 proposer params fully named in each spec"
+ ],
+ "dspy_task/task8635|feature3": [
+  "OK",
+  "nothing flagged \u2014 proposer params fully named in each spec"
+ ],
+ "dspy_task/task8635|feature4": [
+  "OK",
+  "nothing flagged \u2014 proposer params fully named in each spec"
+ ],
+ "dspy_task/task8635|feature5": [
+  "OK",
+  "nothing flagged \u2014 proposer params fully named in each spec"
+ ],
+ "dspy_task/task8635|feature6": [
+  "OK",
+  "nothing flagged \u2014 proposer params fully named in each spec"
+ ],
+ "go_chi_task/task26|feature1": [
+  "OK",
+  "\"alias for chi.URLParam\" carries the ordering requirement"
+ ],
+ "go_chi_task/task26|feature2": [
+  "SPEC",
+  "added RouteMetric/SimpleMetricsCollector field names"
+ ],
+ "go_chi_task/task26|feature3": [
+  "SPEC",
+  "added RouteSelector API \u2014 spec named no identifiers at all"
+ ],
+ "go_chi_task/task26|feature4": [
+  "SPEC",
+  "added WithPriority + Priority* constants"
+ ],
+ "go_chi_task/task27|feature1": [
+  "OK",
+  "HTTP method parsing in patterns described"
+ ],
+ "go_chi_task/task27|feature2": [
+  "OK",
+  "method aliases + case insensitivity stated"
+ ],
+ "go_chi_task/task27|feature3": [
+  "SPEC",
+  "added EnableDebugLogging() \u2014 gold introduces it, spec named nothing"
+ ],
+ "go_chi_task/task27|feature4": [
+  "OK",
+  "path prefix support described"
+ ],
+ "go_chi_task/task56|feature1": [
+  "OK",
+  "Allow header on 405 described"
+ ],
+ "go_chi_task/task56|feature2": [
+  "OK",
+  "501 default + configurability stated"
+ ],
+ "go_chi_task/task56|feature3": [
+  "OK",
+  "request context tracing described"
+ ],
+ "go_chi_task/task56|feature4": [
+  "OK",
+  "method validation middleware described"
+ ],
+ "go_chi_task/task56|feature5": [
+  "OK",
+  "CORS options described"
+ ],
+ "huggingface_datasets_task/task3997|feature1": [
+  "OK",
+  "Features sync semantics stated"
+ ],
+ "huggingface_datasets_task/task3997|feature2": [
+  "SPEC",
+  "added set_custom_decoding_criteria \u2014 gold introduces it, spec named nothing"
+ ],
+ "huggingface_datasets_task/task3997|feature3": [
+  "SPEC",
+  "same hook name, needed for the caching feature too"
+ ],
+ "huggingface_datasets_task/task3997|feature4": [
+  "OK",
+  "extended update() semantics stated"
+ ],
+ "huggingface_datasets_task/task3997|feature5": [
+  "OK",
+  "deep-merge behaviour stated"
+ ],
+ "huggingface_datasets_task/task6252|feature1": [
+  "OK",
+  "EXIF orientation correction described; decode-time behaviour stated"
+ ],
+ "huggingface_datasets_task/task6252|feature2": [
+  "OK",
+  "center-crop-to-square parameter named"
+ ],
+ "huggingface_datasets_task/task6252|feature3": [
+  "OK",
+  "max-resolution clamping parameter named"
+ ],
+ "huggingface_datasets_task/task6252|feature4": [
+  "OK",
+  "2x arithmetic follows from \"all four sides\""
+ ],
+ "huggingface_datasets_task/task6252|feature5": [
+  "OK",
+  "`decode_example` is the repo's existing decode entry point"
+ ],
+ "huggingface_datasets_task/task6252|feature6": [
+  "OK",
+  "single assertion, stated verbatim"
+ ],
+ "huggingface_datasets_task/task7309|feature1": [
+  "OK",
+  "predicate pushdown + filters described"
+ ],
+ "huggingface_datasets_task/task7309|feature2": [
+  "OK",
+  "streaming sort described"
+ ],
+ "llama_index_task/task17070|feature1": [
+  "OK",
+  "NDCG metric change fully described incl. flag semantics"
+ ],
+ "llama_index_task/task17070|feature2": [
+  "OK",
+  "NDCG metric change fully described incl. flag semantics"
+ ],
+ "llama_index_task/task17070|feature3": [
+  "OK",
+  "NDCG metric change fully described incl. flag semantics"
+ ],
+ "llama_index_task/task17244|feature1": [
+  "OK",
+  "ImageBlock parameter named with its semantics stated"
+ ],
+ "llama_index_task/task17244|feature2": [
+  "OK",
+  "ImageBlock parameter named with its semantics stated"
+ ],
+ "llama_index_task/task17244|feature3": [
+  "OK",
+  "ImageBlock parameter named with its semantics stated"
+ ],
+ "llama_index_task/task17244|feature4": [
+  "OK",
+  "ImageBlock parameter named with its semantics stated"
+ ],
+ "llama_index_task/task17244|feature5": [
+  "OK",
+  "ImageBlock parameter named with its semantics stated"
+ ],
+ "llama_index_task/task17244|feature6": [
+  "OK",
+  "ImageBlock parameter named with its semantics stated"
+ ],
+ "llama_index_task/task17244|feature7": [
+  "OK",
+  "ImageBlock parameter named with its semantics stated"
+ ],
+ "llama_index_task/task18813|feature1": [
+  "OK",
+  "DocumentBlock is pre-existing; empty-bytes error described"
+ ],
+ "llama_index_task/task18813|feature2": [
+  "SPEC",
+  "exact 'exceeds maximum allowed size' message \u2014 spec gave only the condition"
+ ],
+ "llama_index_task/task18813|feature3": [
+  "OK",
+  "resolve_* variant named with its return shape stated"
+ ],
+ "llama_index_task/task18813|feature4": [
+  "OK",
+  "resolve_* variant named with its return shape stated"
+ ],
+ "llama_index_task/task18813|feature5": [
+  "OK",
+  "resolve_* variant named with its return shape stated"
+ ],
+ "llama_index_task/task18813|feature6": [
+  "OK",
+  "resolve_* variant named with its return shape stated"
+ ],
+ "openai_tiktoken_task/task0|feature1": [
+  "OK",
+  "encode() parameter named with its post-processing semantics stated"
+ ],
+ "openai_tiktoken_task/task0|feature10": [
+  "OK",
+  "encode() parameter named with its post-processing semantics stated"
+ ],
+ "openai_tiktoken_task/task0|feature2": [
+  "OK",
+  "encode() parameter named with its post-processing semantics stated"
+ ],
+ "openai_tiktoken_task/task0|feature3": [
+  "OK",
+  "encode() parameter named with its post-processing semantics stated"
+ ],
+ "openai_tiktoken_task/task0|feature4": [
+  "OK",
+  "encode() parameter named with its post-processing semantics stated"
+ ],
+ "openai_tiktoken_task/task0|feature5": [
+  "OK",
+  "encode() parameter named with its post-processing semantics stated"
+ ],
+ "openai_tiktoken_task/task0|feature6": [
+  "OK",
+  "composition order stated with an example"
+ ],
+ "openai_tiktoken_task/task0|feature7": [
+  "OK",
+  "encode() parameter named with its post-processing semantics stated"
+ ],
+ "openai_tiktoken_task/task0|feature8": [
+  "OK",
+  "recomputed the dict \u2014 exactly \"top 3 adjacent pairs\""
+ ],
+ "openai_tiktoken_task/task0|feature9": [
+  "OK",
+  "encode() parameter named with its post-processing semantics stated"
+ ],
+ "pallets_click_task/task2068|feature1": [
+  "OK",
+  "multi-file edit; `filename` sequence stated, assertions are content round-trips"
+ ],
+ "pallets_click_task/task2068|feature10": [
+  "OK",
+  "`working_dir` named; \"Editing failed\" is pre-existing"
+ ],
+ "pallets_click_task/task2068|feature11": [
+  "OK",
+  "spec gives the full callback signature the test asserts, incl. timestamp"
+ ],
+ "pallets_click_task/task2068|feature12": [
+  "OK",
+  "`handle_exit_codes`/`exit_code` named; \"Editing failed\" pre-existing"
+ ],
+ "pallets_click_task/task2068|feature2": [
+  "SPEC",
+  "exact timeout message \u2014 and gold does not pluralise, so \"after 1 seconds\""
+ ],
+ "pallets_click_task/task2068|feature3": [
+  "SPEC",
+  "exact `Failed to create backup file: {error}` text"
+ ],
+ "pallets_click_task/task2068|feature4": [
+  "OK",
+  "`editor_args` named; \"Editing failed\" is pre-existing click text"
+ ],
+ "pallets_click_task/task2068|feature5": [
+  "OK",
+  "`lock_files` named; reuses the pre-existing \"Editing failed\" message"
+ ],
+ "pallets_click_task/task2068|feature6": [
+  "OK",
+  "`process_priority` named; test asserts the attribute round-trips"
+ ],
+ "pallets_click_task/task2068|feature7": [
+  "OK",
+  "`escape_shell` named; assertions compare escaped vs unescaped commands"
+ ],
+ "pallets_click_task/task2068|feature8": [
+  "SPEC",
+  "test inspects Popen kwargs for preexec_fn=os.setpgrp; start_new_session is equally correct, so 50/50"
+ ],
+ "pallets_click_task/task2068|feature9": [
+  "OK",
+  "`isolate_env`/`env` named; assertions are on the captured environment"
+ ],
+ "pallets_click_task/task2800|feature1": [
+  "OK",
+  "over-complete \u2014 names function, call sites, construct"
+ ],
+ "pallets_click_task/task2800|feature2": [
+  "OK",
+  "exit-hook API named in the spec"
+ ],
+ "pallets_click_task/task2800|feature3": [
+  "OK",
+  "resource-limit parameter named and described"
+ ],
+ "pallets_click_task/task2800|feature4": [
+  "OK",
+  "'Test error' comes from the test's own stub"
+ ],
+ "pallets_click_task/task2800|feature5": [
+  "OK",
+  "retry messages are test-local; default 0 = no retries is stated, so propagation follows"
+ ],
+ "pallets_click_task/task2800|feature6": [
+  "OK",
+  "snapshot/restore API named in the spec"
+ ],
+ "pallets_click_task/task2800|feature7": [
+  "OK",
+  "explicit API Specification section"
+ ],
+ "pallets_click_task/task2956|feature1": [
+  "OK",
+  "`Argument` is pre-existing; `shell_complete` is click's standard completion hook and the spec's subject"
+ ],
+ "pallets_click_task/task2956|feature2": [
+  "OK",
+  "'Custom error' is raised by the test's own validator, not gold"
+ ],
+ "pallets_click_task/task2956|feature3": [
+  "OK",
+  "`to_info_dict` is pre-existing click API (context line)"
+ ],
+ "pallets_click_task/task2956|feature4": [
+  "OK",
+  "`invoke` is pre-existing click API (context line)"
+ ],
+ "pallets_click_task/task2956|feature5": [
+  "OK",
+  "nothing flagged"
+ ],
+ "pallets_click_task/task2956|feature6": [
+  "OK",
+  "nothing flagged"
+ ],
+ "pallets_click_task/task2956|feature7": [
+  "OK",
+  "nothing flagged"
+ ],
+ "pallets_click_task/task2956|feature8": [
+  "OK",
+  "nothing flagged"
+ ],
+ "pallets_jinja_task/task1465|feature1": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature10": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature2": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature3": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature4": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature5": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature6": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature7": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature8": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1465|feature9": [
+  "OK",
+  "groupby param named with semantics, default and backward-compat all stated"
+ ],
+ "pallets_jinja_task/task1559|feature1": [
+  "OK",
+  "i18n extension API and tag syntax fully named in the spec"
+ ],
+ "pallets_jinja_task/task1559|feature10": [
+  "OK",
+  "i18n extension API and tag syntax fully named in the spec"
+ ],
+ "pallets_jinja_task/task1559|feature2": [
+  "OK",
+  "i18n extension API and tag syntax fully named in the spec"
+ ],
+ "pallets_jinja_task/task1559|feature3": [
+  "OK",
+  "i18n extension API and tag syntax fully named in the spec"
+ ],
+ "pallets_jinja_task/task1559|feature4": [
+  "OK",
+  "over-complete \u2014 error strings quoted verbatim"
+ ],
+ "pallets_jinja_task/task1559|feature5": [
+  "OK",
+  "i18n extension API and tag syntax fully named in the spec"
+ ],
+ "pallets_jinja_task/task1559|feature6": [
+  "OK",
+  "i18n extension API and tag syntax fully named in the spec"
+ ],
+ "pallets_jinja_task/task1559|feature7": [
+  "OK",
+  "i18n extension API and tag syntax fully named in the spec"
+ ],
+ "pallets_jinja_task/task1559|feature8": [
+  "SPEC",
+  "comma separators + trimmed covering fallback"
+ ],
+ "pallets_jinja_task/task1559|feature9": [
+  "OK",
+  "i18n extension API and tag syntax fully named in the spec"
+ ],
+ "pallets_jinja_task/task1621|feature1": [
+  "OK",
+  "fallback search paths named and described"
+ ],
+ "pallets_jinja_task/task1621|feature10": [
+  "OK",
+  "aliasing map named and described"
+ ],
+ "pallets_jinja_task/task1621|feature2": [
+  "OK",
+  "normalisation rules stated"
+ ],
+ "pallets_jinja_task/task1621|feature3": [
+  "OK",
+  "validation rules stated"
+ ],
+ "pallets_jinja_task/task1621|feature4": [
+  "OK",
+  "invalid-tuple handling already stated"
+ ],
+ "pallets_jinja_task/task1621|feature5": [
+  "TEST",
+  "6/6 passed on base; 3 tests rewritten to observe _path_cache"
+ ],
+ "pallets_jinja_task/task1621|feature6": [
+  "OK",
+  "spec names `path_transform`; the flagged `transform_path` is the test's own local function"
+ ],
+ "pallets_jinja_task/task1621|feature7": [
+  "OK",
+  "expansion of env vars / user paths stated"
+ ],
+ "pallets_jinja_task/task1621|feature8": [
+  "OK",
+  "filtering predicate named"
+ ],
+ "pallets_jinja_task/task1621|feature9": [
+  "OK",
+  "`TemplateNotFound` is jinja's standard exception; monitoring API named"
+ ],
+ "pillow_task/task25|feature1": [
+  "OK",
+  "readonly-on-matching-filename rule stated"
+ ],
+ "pillow_task/task25|feature2": [
+  "OK",
+  "preserve-readonly option named"
+ ],
+ "pillow_task/task25|feature3": [
+  "OK",
+  "`OSError` is Pillow's standard save error; spec says dry_run still surfaces format errors"
+ ],
+ "pillow_task/task25|feature4": [
+  "OK",
+  "backup-on-overwrite behaviour stated"
+ ],
+ "pillow_task/task25|feature5": [
+  "OK",
+  "corner-pixel watermark option named"
+ ],
+ "pillow_task/task290|feature1": [
+  "OK",
+  "palette-limit semantics stated"
+ ],
+ "pillow_task/task290|feature2": [
+  "OK",
+  "maximum colour limit stated"
+ ],
+ "pillow_task/task290|feature3": [
+  "OK",
+  "custom palette sort callable named"
+ ],
+ "pillow_task/task290|feature4": [
+  "TEST",
+  "MSE assertions wrapped in uint8 and the reference does not honour the absolute threshold; rewritten to assert the ordering. Verified on Modal: passes with gold, fails without."
+ ],
+ "pillow_task/task290|feature5": [
+  "SPEC",
+  "lightness_factor <= 0 raises ValueError"
+ ],
+ "pillow_task/task68|feature1": [
+  "OK",
+  "tuple XMP form stated"
+ ],
+ "pillow_task/task68|feature2": [
+  "OK",
+  "metadata-removal option named"
+ ],
+ "pillow_task/task68|feature3": [
+  "OK",
+  "orientation transformer callback named"
+ ],
+ "pillow_task/task68|feature4": [
+  "OK",
+  "`TypeError` is the pre-existing exif_transpose error the handler replaces"
+ ],
+ "pillow_task/task68|feature5": [
+  "OK",
+  "logging hooks named"
+ ],
+ "react_hook_form_task/task153|feature1": [
+  "OK",
+  "over-complete \u2014 callback signature, ordering and target file all stated"
+ ],
+ "react_hook_form_task/task153|feature2": [
+  "OK",
+  "over-complete \u2014 callback signature, ordering and target file all stated"
+ ],
+ "react_hook_form_task/task153|feature3": [
+  "OK",
+  "over-complete \u2014 callback signature, ordering and target file all stated"
+ ],
+ "react_hook_form_task/task153|feature4": [
+  "OK",
+  "over-complete \u2014 callback signature, ordering and target file all stated"
+ ],
+ "react_hook_form_task/task153|feature5": [
+  "OK",
+  "over-complete \u2014 callback signature, ordering and target file all stated"
+ ],
+ "react_hook_form_task/task153|feature6": [
+  "OK",
+  "over-complete \u2014 callback signature, ordering and target file all stated"
+ ],
+ "react_hook_form_task/task85|feature1": [
+  "OK",
+  "over-complete \u2014 names the prop, its type, the file and often the implementation shape"
+ ],
+ "react_hook_form_task/task85|feature2": [
+  "OK",
+  "over-complete \u2014 names the prop, its type, the file and often the implementation shape"
+ ],
+ "react_hook_form_task/task85|feature3": [
+  "OK",
+  "over-complete \u2014 names the prop, its type, the file and often the implementation shape"
+ ],
+ "react_hook_form_task/task85|feature4": [
+  "OK",
+  "over-complete \u2014 names the prop, its type, the file and often the implementation shape"
+ ],
+ "react_hook_form_task/task85|feature5": [
+  "OK",
+  "over-complete \u2014 names the prop, its type, the file and often the implementation shape"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature1": [
+  "OK",
+  "IsIP: version/netmask/ipaddress module all named"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature2": [
+  "OK",
+  "IsMac: 'hyphen'/'dot' map 1:1 onto the four named formats"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature3": [
+  "OK",
+  "IsEmail: dotted-domain requirement is the universal convention"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature4": [
+  "OK",
+  "IsURL: (protocol, domain) order follows the title"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature5": [
+  "SPEC",
+  "country codes are case-insensitive \u2014 gold does .upper(), spec silent"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature6": [
+  "SPEC",
+  "issuer tokens are exactly 'Visa'/'Mastercard'/'AmericanExpress' (one word)"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature7": [
+  "OK",
+  "IsHash: algorithms and case-preserving repr both stated"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature8": [
+  "OK",
+  "IsRegex: repr format given for both flag cases"
+ ],
+ "samuelcolvin_dirty_equals_task/task43|feature9": [
+  "OK",
+  "IsColor: hex/rgb/hsl named in the title"
+ ],
+ "typst_task/task6554|feature1": [
+  "OK",
+  "default param stated with examples"
+ ],
+ "typst_task/task6554|feature10": [
+  "SPEC",
+  "exact no-words + unknown-unit diagnostics"
+ ],
+ "typst_task/task6554|feature2": [
+  "SPEC",
+  "exact out-of-bounds diagnostic"
+ ],
+ "typst_task/task6554|feature3": [
+  "OK",
+  "safe param stated with examples"
+ ],
+ "typst_task/task6554|feature4": [
+  "SPEC",
+  "exact not-enough-characters + repeat diagnostics"
+ ],
+ "typst_task/task6554|feature5": [
+  "SPEC",
+  "exact not-enough-characters + count diagnostics"
+ ],
+ "typst_task/task6554|feature6": [
+  "SPEC",
+  "exact \"string contains only whitespace\" (new, not pre-existing)"
+ ],
+ "typst_task/task6554|feature7": [
+  "SPEC",
+  "exact no-character-matches diagnostic + check ordering"
+ ],
+ "typst_task/task6554|feature8": [
+  "SPEC",
+  "exact unknown-case-option diagnostic"
+ ],
+ "typst_task/task6554|feature9": [
+  "OK",
+  "strip param stated with examples"
+ ]
+}
\ No newline at end of file
diff --git a/dataset/SPEC_AUDIT.md b/dataset/SPEC_AUDIT.md
new file mode 100644
index 00000000..5112d4d7
--- /dev/null
+++ b/dataset/SPEC_AUDIT.md
@@ -0,0 +1,880 @@
+# Spec sufficiency audit
+
+One question per feature:
+
+> Could a supremely intelligent LLM, given **only** `feature.md` and the repository — no
+> `tests.patch`, no `feature.patch` — arrive at every behaviour the hidden tests assert?
+
+If yes, the spec is done; a model that fails it failed on its own merits. If no, the spec is
+underspecified and gets the **minimal** statement that closes the gap.
+
+## The line between a fix and a leak
+
+Each missed assertion is one of two things:
+
+| | test | action |
+|---|---|---|
+| **arbitrary** | a perfect reasoner could not have derived it — an identifier, a literal message, a grammar choice, a design decision with several defensible answers | state it in `feature.md`, as a contract |
+| **derivable** | a perfect reasoner gets there from what is already written plus ordinary engineering judgement | change nothing |
+
+A statement leaks if it could not have been written **without having seen the reference
+implementation**. Contracts describe observable behaviour at a public boundary — names, messages,
+accepted inputs, ordering guarantees. Leaks name internal mechanisms, point at which call will
+raise, or say where the bug will be.
+
+## Which file to edit
+
+Either `feature.md` or `tests.patch` is fair game. **The gold `feature.patch` must never need to
+change** — it is the reference the whole task is built around, and a fix that invalidates it is a
+rewrite, not a repair. So:
+
+- **Amend `feature.md`** when the assertion is a legitimate requirement that the spec simply
+  failed to state. Keeps test coverage intact. Gold is unaffected by construction.
+- **Amend `tests.patch`** when the assertion itself is the problem — it tests nothing (passes on
+  the unmodified base), it is computed wrongly, or it demands something no spec should have to
+  promise. Relaxing or removing an assertion leaves gold passing; tightening one may not, so
+  re-check gold after any test edit.
+
+Prefer amending `feature.md` where both are available: deleting an assertion shrinks what the
+benchmark measures, whereas stating the requirement keeps the coverage and makes it winnable.
+
+Worked example of the distinction, from `pallets_jinja/1621` f4. The spec already says *"Invalid
+tuple formats should be handled gracefully (treat as regular paths)"*. The graded failure is
+`TypeError: expected str, bytes or os.PathLike object, not tuple`. Writing *"do not pass it to
+`os.fspath()`, which raises TypeError on a tuple"* would be a leak — it names the call and the
+mechanism. It is also unnecessary: "handled gracefully" already forbids raising, so a strong
+reasoner covers it. **Verdict: derivable, no change.**
+
+## Status — 199/199 features audited
+
+`OK` 174 · `SPEC` 23 · `TEST` 2
+
+`OK` sufficient, no change · `SPEC` feature.md amended · `TEST` tests.patch amended · `RUN?` defect identified, fix needs a container run · `-` not yet audited
+
+| task | feature | verdict | note |
+|---|---|---|---|
+| dottxt_ai_outlines_task/task1371 | f1 | `OK  ` | custom filters API named |
+|  | f2 | `OK  ` | bare `Exception` in the test is a catch-all any error satisfies |
+|  | f3 | `OK  ` | conditional filter loading described |
+|  | f4 | `OK  ` | built-in filter extensions named |
+| dottxt_ai_outlines_task/task1655 | f1 | `OK  ` | regex type: format described and the type name given |
+|  | f10 | `OK  ` | over-complete — spec contains the answer verbatim |
+|  | f2 | `OK  ` | regex type: format described and the type name given |
+|  | f3 | `OK  ` | regex type: format described and the type name given |
+|  | f4 | `OK  ` | regex type: format described and the type name given |
+|  | f5 | `OK  ` | regex type: format described and the type name given |
+|  | f6 | `OK  ` | regex type: format described and the type name given |
+|  | f7 | `OK  ` | every assertion derivable; "xor" is exact |
+|  | f8 | `OK  ` | format-only is explicit ('format' x3, XXX-XX-XXXX given) — note 000-00-0000 and 999-99-9999 are valid, so real SSN area rules must NOT be applied |
+|  | f9 | `OK  ` | regex type: format described and the type name given |
+| dottxt_ai_outlines_task/task1706 | f1 | `OK  ` | '…is not available' is pre-existing outlines text (0 occurrences in gold) |
+|  | f2 | `OK  ` | cache_compiled_grammars and the processor API both named |
+|  | f3 | `OK  ` | `temperature` named; only pre-existing messages asserted |
+|  | f4 | `SPEC` | exact batch-size ValueError text |
+|  | f5 | `SPEC` | exact custom_adapter ValueError texts — conditions stated, wording was not |
+|  | f6 | `OK  ` | the matched messages come from the test's own validator, not from gold |
+|  | f7 | `OK  ` | 'Grammar error' is raised by the test's own stub; fallback API named |
+|  | f8 | `OK  ` | all four memory attributes named in the spec |
+| dspy_task/task8394 | f1 | `OK  ` | bypass() context manager named and described |
+|  | f2 | `OK  ` | over-complete — gives the exact ns_hash formula and key format |
+|  | f3 | `OK  ` | every assertion maps to a stated sentence |
+|  | f4 | `OK  ` | both candidate gaps derivable; amendment reverted |
+|  | f5 | `OK  ` | compression + magic-header format stated in the spec |
+| dspy_task/task8563 | f1 | `OK  ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation |
+|  | f2 | `OK  ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation |
+|  | f3 | `OK  ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation |
+|  | f4 | `OK  ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation |
+|  | f5 | `OK  ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation |
+|  | f6 | `OK  ` | ToolCall / convert_input_schema_to_tool_args are pre-existing; 'Arg X is invalid' is pre-existing dspy validation |
+| dspy_task/task8587 | f1 | `OK  ` | nothing flagged — streaming API fully named in each spec |
+|  | f2 | `OK  ` | nothing flagged — streaming API fully named in each spec |
+|  | f3 | `OK  ` | nothing flagged — streaming API fully named in each spec |
+|  | f4 | `OK  ` | nothing flagged — streaming API fully named in each spec |
+|  | f5 | `OK  ` | nothing flagged — streaming API fully named in each spec |
+|  | f6 | `OK  ` | nothing flagged — streaming API fully named in each spec |
+| dspy_task/task8635 | f1 | `OK  ` | nothing flagged — proposer params fully named in each spec |
+|  | f2 | `OK  ` | nothing flagged — proposer params fully named in each spec |
+|  | f3 | `OK  ` | nothing flagged — proposer params fully named in each spec |
+|  | f4 | `OK  ` | nothing flagged — proposer params fully named in each spec |
+|  | f5 | `OK  ` | nothing flagged — proposer params fully named in each spec |
+|  | f6 | `OK  ` | nothing flagged — proposer params fully named in each spec |
+| go_chi_task/task26 | f1 | `OK  ` | "alias for chi.URLParam" carries the ordering requirement |
+|  | f2 | `SPEC` | added RouteMetric/SimpleMetricsCollector field names |
+|  | f3 | `SPEC` | added RouteSelector API — spec named no identifiers at all |
+|  | f4 | `SPEC` | added WithPriority + Priority* constants |
+| go_chi_task/task27 | f1 | `OK  ` | HTTP method parsing in patterns described |
+|  | f2 | `OK  ` | method aliases + case insensitivity stated |
+|  | f3 | `SPEC` | added EnableDebugLogging() — gold introduces it, spec named nothing |
+|  | f4 | `OK  ` | path prefix support described |
+| go_chi_task/task56 | f1 | `OK  ` | Allow header on 405 described |
+|  | f2 | `OK  ` | 501 default + configurability stated |
+|  | f3 | `OK  ` | request context tracing described |
+|  | f4 | `OK  ` | method validation middleware described |
+|  | f5 | `OK  ` | CORS options described |
+| huggingface_datasets_task/task3997 | f1 | `OK  ` | Features sync semantics stated |
+|  | f2 | `SPEC` | added set_custom_decoding_criteria — gold introduces it, spec named nothing |
+|  | f3 | `SPEC` | same hook name, needed for the caching feature too |
+|  | f4 | `OK  ` | extended update() semantics stated |
+|  | f5 | `OK  ` | deep-merge behaviour stated |
+| huggingface_datasets_task/task6252 | f1 | `OK  ` | EXIF orientation correction described; decode-time behaviour stated |
+|  | f2 | `OK  ` | center-crop-to-square parameter named |
+|  | f3 | `OK  ` | max-resolution clamping parameter named |
+|  | f4 | `OK  ` | 2x arithmetic follows from "all four sides" |
+|  | f5 | `OK  ` | `decode_example` is the repo's existing decode entry point |
+|  | f6 | `OK  ` | single assertion, stated verbatim |
+| huggingface_datasets_task/task7309 | f1 | `OK  ` | predicate pushdown + filters described |
+|  | f2 | `OK  ` | streaming sort described |
+| llama_index_task/task17070 | f1 | `OK  ` | NDCG metric change fully described incl. flag semantics |
+|  | f2 | `OK  ` | NDCG metric change fully described incl. flag semantics |
+|  | f3 | `OK  ` | NDCG metric change fully described incl. flag semantics |
+| llama_index_task/task17244 | f1 | `OK  ` | ImageBlock parameter named with its semantics stated |
+|  | f2 | `OK  ` | ImageBlock parameter named with its semantics stated |
+|  | f3 | `OK  ` | ImageBlock parameter named with its semantics stated |
+|  | f4 | `OK  ` | ImageBlock parameter named with its semantics stated |
+|  | f5 | `OK  ` | ImageBlock parameter named with its semantics stated |
+|  | f6 | `OK  ` | ImageBlock parameter named with its semantics stated |
+|  | f7 | `OK  ` | ImageBlock parameter named with its semantics stated |
+| llama_index_task/task18813 | f1 | `OK  ` | DocumentBlock is pre-existing; empty-bytes error described |
+|  | f2 | `SPEC` | exact 'exceeds maximum allowed size' message — spec gave only the condition |
+|  | f3 | `OK  ` | resolve_* variant named with its return shape stated |
+|  | f4 | `OK  ` | resolve_* variant named with its return shape stated |
+|  | f5 | `OK  ` | resolve_* variant named with its return shape stated |
+|  | f6 | `OK  ` | resolve_* variant named with its return shape stated |
+| openai_tiktoken_task/task0 | f1 | `OK  ` | encode() parameter named with its post-processing semantics stated |
+|  | f10 | `OK  ` | encode() parameter named with its post-processing semantics stated |
+|  | f2 | `OK  ` | encode() parameter named with its post-processing semantics stated |
+|  | f3 | `OK  ` | encode() parameter named with its post-processing semantics stated |
+|  | f4 | `OK  ` | encode() parameter named with its post-processing semantics stated |
+|  | f5 | `OK  ` | encode() parameter named with its post-processing semantics stated |
+|  | f6 | `OK  ` | composition order stated with an example |
+|  | f7 | `OK  ` | encode() parameter named with its post-processing semantics stated |
+|  | f8 | `OK  ` | recomputed the dict — exactly "top 3 adjacent pairs" |
+|  | f9 | `OK  ` | encode() parameter named with its post-processing semantics stated |
+| pallets_click_task/task2068 | f1 | `OK  ` | multi-file edit; `filename` sequence stated, assertions are content round-trips |
+|  | f10 | `OK  ` | `working_dir` named; "Editing failed" is pre-existing |
+|  | f11 | `OK  ` | spec gives the full callback signature the test asserts, incl. timestamp |
+|  | f12 | `OK  ` | `handle_exit_codes`/`exit_code` named; "Editing failed" pre-existing |
+|  | f2 | `SPEC` | exact timeout message — and gold does not pluralise, so "after 1 seconds" |
+|  | f3 | `SPEC` | exact `Failed to create backup file: {error}` text |
+|  | f4 | `OK  ` | `editor_args` named; "Editing failed" is pre-existing click text |
+|  | f5 | `OK  ` | `lock_files` named; reuses the pre-existing "Editing failed" message |
+|  | f6 | `OK  ` | `process_priority` named; test asserts the attribute round-trips |
+|  | f7 | `OK  ` | `escape_shell` named; assertions compare escaped vs unescaped commands |
+|  | f8 | `SPEC` | test inspects Popen kwargs for preexec_fn=os.setpgrp; start_new_session is equally correct, so 50/50 |
+|  | f9 | `OK  ` | `isolate_env`/`env` named; assertions are on the captured environment |
+| pallets_click_task/task2800 | f1 | `OK  ` | over-complete — names function, call sites, construct |
+|  | f2 | `OK  ` | exit-hook API named in the spec |
+|  | f3 | `OK  ` | resource-limit parameter named and described |
+|  | f4 | `OK  ` | 'Test error' comes from the test's own stub |
+|  | f5 | `OK  ` | retry messages are test-local; default 0 = no retries is stated, so propagation follows |
+|  | f6 | `OK  ` | snapshot/restore API named in the spec |
+|  | f7 | `OK  ` | explicit API Specification section |
+| pallets_click_task/task2956 | f1 | `OK  ` | `Argument` is pre-existing; `shell_complete` is click's standard completion hook and the spec's subject |
+|  | f2 | `OK  ` | 'Custom error' is raised by the test's own validator, not gold |
+|  | f3 | `OK  ` | `to_info_dict` is pre-existing click API (context line) |
+|  | f4 | `OK  ` | `invoke` is pre-existing click API (context line) |
+|  | f5 | `OK  ` | nothing flagged |
+|  | f6 | `OK  ` | nothing flagged |
+|  | f7 | `OK  ` | nothing flagged |
+|  | f8 | `OK  ` | nothing flagged |
+| pallets_jinja_task/task1465 | f1 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f10 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f2 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f3 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f4 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f5 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f6 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f7 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f8 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+|  | f9 | `OK  ` | groupby param named with semantics, default and backward-compat all stated |
+| pallets_jinja_task/task1559 | f1 | `OK  ` | i18n extension API and tag syntax fully named in the spec |
+|  | f10 | `OK  ` | i18n extension API and tag syntax fully named in the spec |
+|  | f2 | `OK  ` | i18n extension API and tag syntax fully named in the spec |
+|  | f3 | `OK  ` | i18n extension API and tag syntax fully named in the spec |
+|  | f4 | `OK  ` | over-complete — error strings quoted verbatim |
+|  | f5 | `OK  ` | i18n extension API and tag syntax fully named in the spec |
+|  | f6 | `OK  ` | i18n extension API and tag syntax fully named in the spec |
+|  | f7 | `OK  ` | i18n extension API and tag syntax fully named in the spec |
+|  | f8 | `SPEC` | comma separators + trimmed covering fallback |
+|  | f9 | `OK  ` | i18n extension API and tag syntax fully named in the spec |
+| pallets_jinja_task/task1621 | f1 | `OK  ` | fallback search paths named and described |
+|  | f10 | `OK  ` | aliasing map named and described |
+|  | f2 | `OK  ` | normalisation rules stated |
+|  | f3 | `OK  ` | validation rules stated |
+|  | f4 | `OK  ` | invalid-tuple handling already stated |
+|  | f5 | `TEST` | 6/6 passed on base; 3 tests rewritten to observe _path_cache |
+|  | f6 | `OK  ` | spec names `path_transform`; the flagged `transform_path` is the test's own local function |
+|  | f7 | `OK  ` | expansion of env vars / user paths stated |
+|  | f8 | `OK  ` | filtering predicate named |
+|  | f9 | `OK  ` | `TemplateNotFound` is jinja's standard exception; monitoring API named |
+| pillow_task/task25 | f1 | `OK  ` | readonly-on-matching-filename rule stated |
+|  | f2 | `OK  ` | preserve-readonly option named |
+|  | f3 | `OK  ` | `OSError` is Pillow's standard save error; spec says dry_run still surfaces format errors |
+|  | f4 | `OK  ` | backup-on-overwrite behaviour stated |
+|  | f5 | `OK  ` | corner-pixel watermark option named |
+| pillow_task/task290 | f1 | `OK  ` | palette-limit semantics stated |
+|  | f2 | `OK  ` | maximum colour limit stated |
+|  | f3 | `OK  ` | custom palette sort callable named |
+|  | f4 | `TEST` | MSE assertions wrapped in uint8 and the reference does not honour the absolute threshold; rewritten to assert the ordering. Verified on Modal: passes with gold, fails without. |
+|  | f5 | `SPEC` | lightness_factor <= 0 raises ValueError |
+| pillow_task/task68 | f1 | `OK  ` | tuple XMP form stated |
+|  | f2 | `OK  ` | metadata-removal option named |
+|  | f3 | `OK  ` | orientation transformer callback named |
+|  | f4 | `OK  ` | `TypeError` is the pre-existing exif_transpose error the handler replaces |
+|  | f5 | `OK  ` | logging hooks named |
+| react_hook_form_task/task153 | f1 | `OK  ` | over-complete — callback signature, ordering and target file all stated |
+|  | f2 | `OK  ` | over-complete — callback signature, ordering and target file all stated |
+|  | f3 | `OK  ` | over-complete — callback signature, ordering and target file all stated |
+|  | f4 | `OK  ` | over-complete — callback signature, ordering and target file all stated |
+|  | f5 | `OK  ` | over-complete — callback signature, ordering and target file all stated |
+|  | f6 | `OK  ` | over-complete — callback signature, ordering and target file all stated |
+| react_hook_form_task/task85 | f1 | `OK  ` | over-complete — names the prop, its type, the file and often the implementation shape |
+|  | f2 | `OK  ` | over-complete — names the prop, its type, the file and often the implementation shape |
+|  | f3 | `OK  ` | over-complete — names the prop, its type, the file and often the implementation shape |
+|  | f4 | `OK  ` | over-complete — names the prop, its type, the file and often the implementation shape |
+|  | f5 | `OK  ` | over-complete — names the prop, its type, the file and often the implementation shape |
+| samuelcolvin_dirty_equals_task/task43 | f1 | `OK  ` | IsIP: version/netmask/ipaddress module all named |
+|  | f2 | `OK  ` | IsMac: 'hyphen'/'dot' map 1:1 onto the four named formats |
+|  | f3 | `OK  ` | IsEmail: dotted-domain requirement is the universal convention |
+|  | f4 | `OK  ` | IsURL: (protocol, domain) order follows the title |
+|  | f5 | `SPEC` | country codes are case-insensitive — gold does .upper(), spec silent |
+|  | f6 | `SPEC` | issuer tokens are exactly 'Visa'/'Mastercard'/'AmericanExpress' (one word) |
+|  | f7 | `OK  ` | IsHash: algorithms and case-preserving repr both stated |
+|  | f8 | `OK  ` | IsRegex: repr format given for both flag cases |
+|  | f9 | `OK  ` | IsColor: hex/rgb/hsl named in the title |
+| typst_task/task6554 | f1 | `OK  ` | default param stated with examples |
+|  | f10 | `SPEC` | exact no-words + unknown-unit diagnostics |
+|  | f2 | `SPEC` | exact out-of-bounds diagnostic |
+|  | f3 | `OK  ` | safe param stated with examples |
+|  | f4 | `SPEC` | exact not-enough-characters + repeat diagnostics |
+|  | f5 | `SPEC` | exact not-enough-characters + count diagnostics |
+|  | f6 | `SPEC` | exact "string contains only whitespace" (new, not pre-existing) |
+|  | f7 | `SPEC` | exact no-character-matches diagnostic + check ordering |
+|  | f8 | `SPEC` | exact unknown-case-option diagnostic |
+|  | f9 | `OK  ` | strip param stated with examples |
+
+## Separately: gradeability — swept across all 30 tasks
+
+Spec sufficiency is meaningless for a feature whose tests never run, so this was checked
+mechanically for every task first. Two questions: does the runner actually select each feature's
+tests, and can a run that executes nothing be scored as a pass?
+
+**Result: 5 tasks had a real defect, all now fixed. The other 25 are sound.**
+
+| task | defect | fix |
+|---|---|---|
+| `go_chi/26` | image `golang:1.21-alpine` while feature 1's test carries `//go:build go1.22`, so the file was excluded from the build and `go test -run TestPathValue` matched nothing and exited 0 | bumped to 1.22 |
+| `go_chi/26,27,56` | `go test -run <pat>` **exits 0 when the pattern matches nothing**, and the runner read exit 0 as success — a feature could score with no implementation. Filter was also unanchored, so an agent-authored `TestFooExtra` satisfied it | anchored to `^(...)$`; run under `-v` and fail if no `=== RUN` line appears |
+| `typst/6554` | runner hardcoded `-- string-first` / `-- string-last`, which only feature 7 uses. Features 2,3,4,5,6,8,9,10 declare `str-*` blocks that were **never selected** | filters derived from the test patch, full-suite fallback for patches that add cases to an existing block, zero-test run fails |
+| `dspy/8394` | a missing secondary test target printed "Skipping" and passed | exits 1 |
+| `dottxt/1706` | success condition grepped for the bare string `passed`, which also matches `0 passed, N skipped` — an all-skipped run scored as a pass | requires `[1-9][0-9]* passed` |
+
+Checked and **sound**, no change needed:
+
+- **Every Python task** — `pytest` exits 5 on "no tests collected" and 2 on a collection error, both
+  non-zero, and every runner is under `set -e` or checks the code explicitly. `dottxt/1706` reads
+  `PIPESTATUS[0]` correctly through its `tee`.
+- **Both react_hook_form tasks** — jest exits 1 when its path pattern matches no test
+  (no `--passWithNoTests`), and all 11 features' tests land in the file the runner names.
+- **File targeting across all 199 features** — every feature's `tests.patch` writes to a file the
+  task's runner actually executes. The only apparent mismatches were `openai_tiktoken/0`, whose
+  runner targets the `tests/` directory rather than named files.
+
+Separately, `--maxfail=1` was removed from `dspy/8394, 8563, 8587, 8635`. It does not mis-grade,
+but it stops at the first failure so the reported `tests_failed` is a floor rather than a count —
+in one observed run it left 2 of 14 tests unexecuted. `timeout 300` already bounds these.
+
+---
+
+# Per-feature log
+
+## dottxt_ai_outlines/1655 feature 7 — `credit_card` · ✅ no change
+
+**Spec says:** *"Valid credit card numbers will have either no separators, space xor hyphen
+separators."* Plus 16-digit, importable from `outlines.types`, and a usage example rejecting a
+15-digit string.
+
+**Hidden test asserts 21 cases.** Judged one at a time:
+
+| assertion | derivable from spec + repo? |
+|---|---|
+| 16 contiguous digits valid | yes — "16-digit", usage example |
+| `1234-5678-9012-3456` valid | yes — usage example |
+| `1234 5678 9012 3456` valid | yes — "space xor hyphen" |
+| all-zeros / all-nines valid | yes — digits are digits |
+| **mixed separators invalid** (`1234-5678 9012-3456`) | **yes** — this is what "xor" means, and the word is used correctly |
+| 15 and 17 digits invalid | yes — "16-digit"; example rejects 15 |
+| incomplete/missing group invalid | yes — follows from 4 groups of 4 |
+| dots, underscores invalid | yes — "space xor hyphen" is an exhaustive enumeration |
+| letters invalid | yes — "digit" |
+| empty string invalid | yes |
+| doubled separators invalid (`1234--5678…`) | yes — the natural grouped pattern rejects them |
+| matching is `fullmatch`, not `search` | yes — **from the repo**: the pre-existing body of `test_type_regex` in `tests/types/test_custom_types.py` reads `re.fullmatch(regex_str, test_string)`, and agents can read that file |
+
+**Verdict: sufficient, no change.** The earlier automated audit flagged "xor" as thin. I disagree:
+it is unusual phrasing but semantically exact, and it is the only word needed to forbid mixing.
+Empirically 3 of 5 archived agent regexes scored 21/21 on these cases — and the bar here is a
+supremely intelligent reasoner, not the median model, so I am deliberately **not** rewording it
+for clarity. Doing so would make the task easier rather than fix a defect.
+
+**Noted, not fixed:** `tests.patch` carries a stale header `# Credit card tests - mixed separators
+(should match)` with zero tuples beneath it, two lines above the cases asserting the exact
+opposite. Author hygiene from a flipped expectation. Agents never see this file, so it changes no
+outcome; worth deleting on any future pass through the test file.
+
+## dottxt_ai_outlines/1655 feature 10 — `hash_sha256` · ✅ no change
+
+**Sufficient — in fact over-complete.** `feature.md` contains
+`# Returns: {"type": "string", "pattern": "[a-fA-F0-9]{64}"}`, byte-identical to the gold
+implementation `hash_sha256 = Regex(r"[a-fA-F0-9]{64}")`. Every hidden assertion (63/65-char
+edges, case-insensitivity, non-hex characters, empty string, whitespace) is additionally spelled
+out in the spec's own "Validation Behavior" and "Test Coverage" sections.
+
+**Not changed, deliberately.** This is the opposite failure mode — the spec hands over the answer,
+so the feature reduces to copying one string. But an over-complete spec makes a feature *easy*,
+not *broken*: it still grades correctly and it discriminates nothing. Removing the pattern would
+make a currently-reliable feature harder, which is a change to benchmark difficulty rather than a
+repair, and it would destabilise one of the few features that passes consistently. Flagged for a
+decision rather than acted on.
+
+## dspy/8394 feature 3 — per-entry TTL · ✅ no change
+
+**Sufficient, and close to over-complete.** The spec names the concrete mechanism at every point
+the test probes: `TTLCache(maxsize=memory_max_entries, ttl=ttl)` and where to import it from,
+`LRUCache` as the fallback, `disk_cache.set(key, value, expire=ttl)`, `disk_cache.expire()` on the
+miss path, `set_ttl` in `__all__`, and `Cache(..., ttl=)` / `configure_cache(ttl=)`.
+
+Every hidden assertion maps onto a stated sentence:
+
+| assertion | where stated |
+|---|---|
+| `isinstance(memory_cache, TTLCache)` when ttl set | Solution 2 |
+| `memory_cache.ttl == ttl_seconds` | Solution 2 |
+| entry present before expiry, `None` after, both layers | Considerations: "must expire from both layers after `ttl+ε`" |
+| `isinstance(memory_cache, LRUCache)` and **not** `TTLCache` when `ttl=None` | Solution 2 "fall back to LRUCache"; `TTLCache` is not an `LRUCache` subclass in cachetools, so this follows |
+| `len(memory_cache) == 0` after expiry | TTLCache semantics |
+| `set_ttl` / `configure_cache(ttl=)` / `Cache(ttl=)` all work | Solution 1, 5, 6 and Considerations, individually |
+
+Passed in the teacher-v2 run (10/10 primary + 4 TTL). No change.
+
+## dspy/8394 feature 4 — cache statistics · ✅ no change (earlier amendment reverted)
+
+I had amended this spec, then reverted it. Recording why, since the reverted text is the clearest
+illustration of the criterion.
+
+**Amendment 1, "returns a populated dict even when the only event was a miss".** The spec already
+says *"Returns `None` when tracking is disabled"*. Read closed-world — and a supremely intelligent
+reasoner does read it that way — disabled tracking is the *only* stated `None` case, so tracking
+enabled implies non-`None`. The observed implementation invented a second case (`None` when no
+events recorded). **Derivable. Reverted.**
+
+**Amendment 2, "tracking applies to any `Cache` instance, not only `dspy.cache`".** The hidden
+tests do call `.stats()` on an ordinary `Cache` fixture rather than the singleton, and the spec
+only ever writes `dspy.cache.stats()`. But `dspy.cache` *is* a `Cache`, `stats()` is described as a
+method, and no reasonable implementation attaches a method to one instance rather than the class.
+Instrumenting `Cache.get()` covers every instance for free. **Derivable. Reverted.**
+
+**What actually failed** in the observed run: `CacheUsageTracker.record_hit` / `record_miss` were
+defined and never called — `Cache.get()` bumped its own counters and never looked the tracker up,
+so `get_stats()` returned `None` and `Prediction._cache_usage` stayed unset. The spec requires the
+producer side plainly (*"returns cache events that occurred during that specific prediction's
+execution"* — you cannot return events you never recorded). **A model failure, not a spec gap.**
+
+## go_chi/26 feature 1 — `http.Request.PathValue` · ✅ no change
+
+**Sufficient.** The one non-obvious requirement is ordering: the test reads `r.PathValue(pathKey)`
+*inside* the handler, so chi must populate the values before dispatching. The spec carries this
+with one word — *"`http.Request.PathValue` will act as an **alias** for `chi.URLParam`"*, restated
+as *"make `r.PathValue(...)` act as an alias for `chi.URLParam(r, ...)` when routing with chi"*.
+`chi.URLParam` exists to be called inside a handler; something that only becomes valid after the
+handler returns is not an alias for it. Derivable.
+
+Everything else is stated outright: conditional compilation for pre-1.22 (`path_value.go`,
+`path_value_fallback.go` are named in Files Modified), single-segment matches only, and the
+`method + pattern` registration form is pre-existing chi behaviour.
+
+This feature was **ungradeable** until the Dockerfile fix — see the gradeability section. Its spec
+was never the problem.
+
+## huggingface_datasets/6252 feature 4 — `crop_margin` · ✅ no change
+
+**Sufficient.** Three assertions, all stated:
+
+- `Image(crop_margin=10)` accepted → *"optional `crop_margin` parameter to the `Image` feature"*
+- `size == (640-20, 480-20)` → *"crop a fixed number of pixels from **each edge**"* / *"uniform
+  margin (specified in pixels) from **all four sides**"*. The 2× per dimension is arithmetic, not
+  a hidden requirement.
+- applied at decode time → *"during decoding"*
+
+`mode == "RGB"` is unchanged base behaviour. Passed 44/44 in every archived run.
+
+## huggingface_datasets/6252 feature 6 — CMYK→RGB · ✅ no change
+
+**Sufficient.** *"Detect CMYK images during feature decoding and transparently convert them to
+RGB"* covers the only real assertion (`img.mode == "RGB"`). Size preservation is implied by a
+colour-space conversion and asserted only as a sanity check. Passed 44/44 in every archived run.
+
+## openai_tiktoken/0 feature 6 — `transformers` · ✅ no change
+
+**Sufficient.** Parameter name, list form, and — the only subtle part — order of composition are
+all stated: *"applied sequentially in the order given, with each transformer receiving the output
+of the previous one"*, with a two-function example. The test exercises exactly that, including
+both orderings. Unchanged behaviour without the parameter is ordinary backward compatibility.
+
+## openai_tiktoken/0 feature 8 — `return_repeated_pattern` · ✅ no change
+
+**Sufficient.** The expected dict looked like a magic constant, so I recomputed it from the spec.
+Text is `" like love"*99 + " like hate"*20`; token ids are ` like`=1093, ` love`=3021,
+` hate`=12491. Adjacent pairs over the whole sequence:
+
+| pair | count |
+|---|---|
+| (1093, 3021) | 99 |
+| (3021, 1093) | 99 — 98 within the `love` block plus 1 at the transition |
+| (1093, 12491) | 20 |
+| (12491, 1093) | 19 — falls outside the top 3 |
+
+The asserted `{(1093,3021):99, (3021,1093):99, (1093,12491):20}` is precisely *"all adjacent token
+pairs … top 3 most frequent"*, both of which the spec says. Ties do not arise. Empty text → `{}`
+follows from "top 3" meaning at most 3. The return shape `(tokens, dict)` and the key form
+`(token1, token2)` are both in the spec's own example.
+
+**The v2 regression here was not a spec gap.** Agent 2 wrote a correct implementation, published it
+twice, then `git push -f` rewound its branch to the base commit and never opened a PR — a
+publication failure after six reversals of who owned `def encode(...)`.
+
+## pallets_click/2800 feature 1 — close contexts in completion · ✅ no change
+
+**Sufficient, over-complete.** The spec names the function (`_resolve_context`), all three call
+sites to wrap, the exact construct (`with cli.make_context(...) as ctx:`), the symptom
+(`ResourceWarning: unclosed file`), and both files. The test asserts only that no warnings are
+raised. Effectively an implementation description.
+
+## pallets_click/2800 feature 7 — `validate_nesting` · ✅ no change
+
+**Sufficient, over-complete.** The spec carries an explicit **API Specification** section, and
+every hidden assertion has a line in it:
+
+| assertion | stated as |
+|---|---|
+| `validate_nesting=False` default, normal usage unaffected | *"Default value is `False`"* |
+| attribute readable on a Context | *"must be publicly accessible on Context instances"* |
+| child inherits `True` from parent | *"all child contexts created via `_make_sub_context()` automatically inherit this setting"* |
+| grandchild inherits too | same clause, applied transitively |
+| `RuntimeError` if a child is still open at parent close | *"If a parent context is closed while any child contexts remain open, a `RuntimeError` must be raised"* |
+| closing children first is fine | *"Child contexts must be removed from parent tracking when they are closed"* |
+
+**Task-design note, not a spec defect.** f7's spec directs the implementer to set
+`validate_nesting=False` inside `_resolve_context` — the very call sites f1 is wrapping in `with`
+blocks. The two features are therefore *ordered* onto the same statements by their specs. This is
+a pairing choice, not something a spec edit can repair.
+
+## pallets_jinja/1559 feature 4 — `trans` metadata · ✅ no change
+
+**Sufficient, over-complete.** The spec states the attribute name (`trans_metadata`), the
+empty-dict rule (*"If metadata dict is empty, do **not** add the attribute"*), `hasattr`
+accessibility, the three allowed parameter names, and — unusually — the **exact error strings**:
+`"metadata parameter '{name}' must be a string literal."` and
+`"metadata parameter '{name}' requires a value assignment."`, which the tests then match on. The
+remaining assertions (empty string values, non-ASCII, long strings) follow from "string literal".
+Interaction with variables / `trimmed` / pluralization is covered by the Integration Requirements.
+Passed 61/61 in both runs.
+
+## pillow/290 feature 4 — `error_threshold` · ✅ spec fine · ⚠️ test defect, deferred
+
+**Spec sufficient.** *"maximum allowed mean square error (MSE) between the original image and the
+quantized version"*, *"stop adding colors once the error is below this threshold"*, and *"If the
+error threshold cannot be met with the specified `colors` limit, it will use the maximum allowed
+colors"* cover every behavioural assertion — including the one that reads backwards at first
+glance, `low_threshold_colors > with_threshold_colors` (threshold 20 is stricter than 30, so it
+needs *more* colours), and `impossible_colors == 10`.
+
+**But two of its assertions compute nothing.** `np.array()` of a PIL RGB image is `uint8`, so
+`(image_array - quantized_array) ** 2` wraps twice. Measured directly:
+
+```
+channels 100 vs [84, 110, 228]
+uint8 : diff [16, 246, 128] -> squares [0, 100, 0]      -> mean    33.3
+int   : squares [256, 100, 16384]                        -> mean  5580.0
+```
+
+A per-channel error of 16 contributes **0**, and 128 contributes **0**. `with_threshold_mse <=
+30.0` and `low_threshold_mse <= 20.0` are therefore comparing a wrapped, arbitrary number against
+a threshold — not merely weaker than intended, but capable of passing or failing by luck.
+
+**Fix is `(a.astype(int) - b.astype(int)) ** 2` — deliberately NOT applied yet.** Correcting the
+arithmetic raises the computed value by two orders of magnitude, and a 256-colour quantization of
+`hopper()` may well exceed the literal `30.0` / `20.0` thresholds. That would break the gold patch,
+which the rules forbid. This needs one container run to measure the true MSE and re-tune the two
+thresholds together. Queued for the execution pass.
+
+The colour-count assertions carry the real behavioural signal and are unaffected.
+
+## Correction: go_chi/26 feature 2 — "Files Modified" edit reverted
+
+I had removed `metrics_test.go` from Files Modified, on a claim that following it literally makes
+`git apply` reject the whole patch. **I verified that claim and it is false.**
+`_filter_test_files` (`src/cooperbench/eval/sandbox.py:693`) drops whole `diff --git` sections for
+test paths and re-terminates the patch, so a test-file hunk is silently discarded, never rejected.
+`pillow/290` f4 lists `Tests/test_image_quantize.py` in its own Files Modified and is a sound
+feature, so this is the dataset's normal convention.
+
+Removing it would have told the agent *not* to write tests for its own feature — the opposite of
+what we want. Reverted. The API-names half of that edit stands on its own evidence.
+
+## pallets_jinja/1621 feature 5 — path caching · ✅ spec fine · ✏️ tests rewritten
+
+**Spec sufficient.** It names the attribute `_path_cache`, its type
+`Dict[str, Optional[str]]`, the `None`-means-searched-and-missing convention, and the cache
+check / hit / miss / populate phases individually.
+
+**The tests were the defect: all six passed on the unmodified base.** Not one assertion observed
+the cache — they checked that `tmpl1.render() == tmpl2.render()`, that the first search path wins,
+and that backward compatibility holds, all of which are true with no cache at all. The
+"performance" test closed with `assert cached_time >= 0` and `assert fresh_time >= 0`, which are
+vacuous. So the feature scored 5/5 across archived runs regardless of whether it was implemented,
+including once for an agent whose patch contained no implementation.
+
+**Replaced the vacuous performance test with three that observe the contract.** Written against
+`loader.get_source()` rather than `env.get_template()`, deliberately: the `Environment` keeps its
+own template cache, which would mask the loader's behaviour.
+
+| new test | asserts | why base fails |
+|---|---|---|
+| `..._records_resolved_path` | `_path_cache` starts `{}`; after a successful lookup it holds a real file path | base has **0** occurrences of `_path_cache` in `loaders.py` → `AttributeError` |
+| `..._records_miss_as_none` | a failed lookup stores `None`; creating the file afterwards does **not** make it findable, because the cached miss answers without touching the filesystem | same |
+| `..._drops_cached_path_when_file_vanishes` | after the cached file is deleted the entry is re-resolved and re-cached as `None`, rather than the stale path being reused | same |
+
+Each traced line-by-line against `feature.patch` before being written — the second encodes gold's
+*permanent* negative caching (`if cached_filename is None: raise TemplateNotFound`) and the third
+encodes its delete-and-re-resolve path (`open_if_exists` → `None` → `del self._path_cache[...]`).
+An earlier automated audit called the permanent negative caching a regression in gold; it is in
+fact exactly what the spec prescribes, so the test encodes it rather than contradicting it.
+
+**Verification done:** cloned jinja at the task's base commit `a2920752`, confirmed the rewritten
+`tests.patch` applies cleanly to base and that `feature.patch` still applies on top, and
+regenerated the hunk header from a real `git diff` rather than hand-editing it. **Not yet done:**
+executing the suite — running the cloned repo's tests is blocked in this environment. Queued for
+the same execution pass as pillow/290 f4.
+
+## go_chi/26 feature 2 — route metrics · ✏️ amended
+
+**The spec names the types but not the fields the test reads.** It says *"Create `RouteMetric`
+struct containing pattern, method, path, duration, and URL parameters"* and *"Provide
+`SimpleMetricsCollector` with callback function"* — prose descriptions, not identifiers. The
+hidden test reads `collectedMetric.Pattern`, `.Method`, `.Duration`, `.Params["id"]` and
+constructs `&SimpleMetricsCollector{OnHit: func(metric RouteMetric) {...}}`.
+
+`Pattern`, `Method` and `Duration` follow from the prose. **`Params` and `OnHit` do not**, and
+`Params` is worse than merely unstated: chi's own vocabulary for this concept is `URLParams`
+(`RouteParams.URLParams`), which a reasoner reading the repo would naturally follow — straight
+into the wrong name. *"callback function"* gives no purchase on `OnHit` at all.
+
+**Added** a Public API block giving the two struct shapes, with a note that it is `Params`, not
+`URLParams`. Public identifiers are contract, not implementation — a real PR description states
+its API. Nothing about instrumenting `routeHTTP()`, measuring duration, or wiring the collector is
+given away.
+
+## pallets_jinja/1559 feature 8 — `ctrans` · ✏️ amended
+
+Two arbitrary grammar decisions, neither derivable:
+
+1. **Commas as separators.** The spec's only grammar line is
+   `{% ctrans condition [variables] [trimmed|notrimmed] %}` — space-separated. The test uses
+   `{% ctrans use_i18n, count %}`. `grep -ic comma feature.md` → the word never appears. Nothing
+   in the repo helps: the tag is new, so there is no existing `ctrans` to imitate.
+2. **`trimmed` covering the fallback block.** The test asserts
+   `{% ctrans False trimmed %}…{% fallback %}  Fallback  \n  Text  {% endctrans %}` renders
+   `"Fallback Text"`. The spec lists `trimmed` among the header modifiers but never says which
+   blocks it reaches, and the standard `trans` tag has no fallback block to generalise from.
+
+**Added** both as bullets under the grammar line. **Removed from my own first draft** a
+parenthetical explaining that copying `trans`'s comma handling puts the separator one slot too
+early — that is diagnostic guidance about where the bug will be, not a requirement. It was a leak
+and it is gone.
+
+## pillow/290 feature 5 — `lightness_factor` · ✏️ amended
+
+**One arbitrary requirement.** `feature.md` has zero matches for
+`valueerror|raise|invalid|must be|validat|error`; `tests.patch` contains
+`with pytest.raises(ValueError)` twice, for `lightness_factor=-0.5` and `=0`. Raising is a design
+decision, not a deduction — clamping a non-positive factor to a floor, or letting it produce a
+black palette, are both defensible readings of *"a value less than 1.0 makes it darker"*.
+
+The boundary matters and I checked it before writing: the same test requires `0.1`, `0.9`, `1.1`
+and `2.0` to **work**. So the rule is exactly `<= 0` raises, with no upper bound. A generic
+"validate your parameters" instinct that rejected `0.1` or capped at `1.0` would break passing
+assertions — which is why the sentence states the precise boundary.
+
+**Deliberately not added: the `putpalette()` mechanism.** The other half of this feature's failure
+is that assigning `im.palette.palette` never reaches the C image, so the adjustment is a silent
+no-op. But the spec already promises *"1.1 for 10% brighter"* — observable behaviour at a public
+boundary. An implementation that changes nothing violates a promise the spec already makes, so
+that is a verification failure by the implementer, not a gap in the spec.
+
+## typst/6554 features 7 and 8 — exact diagnostics · ✏️ amended
+
+typst's test harness compares diagnostic text **verbatim** (`tests/src/run.rs`,
+`message != note.message`), so the wording is load-bearing, and neither wording is derivable:
+
+| feature | required message | what the spec said |
+|---|---|---|
+| 7 | `no character matches pattern "alpha"` | *"propagate the existing … out-of-pattern errors"* |
+| 8 | `unknown case option "spongebob"` | *"Invalid values raise a descriptive error"* |
+
+Both strings are introduced *by the reference patch*, so by construction they cannot be found in
+the base repo, and the repo's own convention (`unknown variable: {}`, unquoted, colon-separated)
+points away from the quoted form.
+
+**Verified against gold before writing**, so that the spec describes what the reference actually
+emits: `eco_format!("no character matches pattern \"{}\"", pattern)` and
+`eco_format!("unknown case option \"{}\"", other)`. Both statements match exactly.
+
+Feature 7 also gained one ordering sentence — the empty-string check must precede the pattern
+check, so `"".last(pattern: "alpha")` still reports `string is empty`. Without it, fixing the
+message alone un-masks a second failure.
+
+**Deliberately not stated:** gold's third message, `unknown pattern "{}"` for an unrecognised
+pattern name. No test exercises it, and specifying untested behaviour adds obligation without
+adding measurement.
+
+The alternative to all of this — relaxing `run.rs` to substring matching — is the more principled
+fix, since grading a coding agent on exact prose is measuring the wrong thing. It is not taken
+here because `run.rs` lives in the cloned repo rather than the dataset, so changing it would alter
+the harness for all 2,354 of typst's own tests.
+
+---
+
+# Per-feature log, part 2 — the remaining 169
+
+The first 30 were judged one at a time by reading every assertion. That does not scale to 169, so
+detection was mechanised (`scripts/spec_audit_survey.py`) while the judgement stayed manual. The
+survey flags three things per feature and nothing else:
+
+- a **public symbol** the reference introduces, the test uses, and the spec never names
+- an **error message** the test matches on that the spec never gives, tagged `GOLD` if the
+  reference is where it comes from
+- an **exception type** the test expects that the spec never names
+
+Everything it flags is a candidate. **Roughly half turned out to be derivable on inspection**, and
+the recurring false positives are worth naming because they would otherwise look like defects:
+
+| false positive | why it is not a gap |
+|---|---|
+| symbol appears on a **context** line of the patch | pre-existing API — `ToolCall`, `convert_input_schema_to_tool_args`, `to_info_dict`, `invoke`, `Argument`, `DocumentBlock`, `AudioBlock`, `decode_example` |
+| message tagged `elsewhere` rather than `GOLD` | raised by the **test's own stub**, so it is not a requirement — click/2800 f4-f5, click/2956 f2, dottxt/1706 f6-f7, jinja/1621 f6 |
+| generic exception type | `AssertionError` is dirty-equals' whole idiom; bare `Exception` is a catch-all any error satisfies; `OSError`/`TypeError` were pre-existing library behaviour |
+| name collision | jinja/1621 f6 flagged `transform_path`, which is the **test's own local function** passed as `path_transform=` — the parameter the spec does name |
+
+## Amendments made in part 2
+
+| feature | what was not derivable |
+|---|---|
+| `go_chi/26` f3 | `RouteSelector` interface, `WithRouteSelector`, `NewVersionSelector`, `NewRoleBasedSelector`, `AddHandler` — the spec described dynamic routing in prose and named **no identifier at all** |
+| `go_chi/26` f4 | `WithPriority` and the `Priority*` constants — likewise |
+| `go_chi/27` f3 | `EnableDebugLogging()` — introduced by the reference, spec said only "adds logging infrastructure" |
+| `hf_datasets/3997` f2, f3 | `set_custom_decoding_criteria(custom_criteria=None)` — introduced by the reference, spec described the capability but named no method |
+| `typst/6554` f2, f4, f5, f6, f10 | exact diagnostics, verbatim-compared by the harness. f6's is the subtle one: the spec says all-whitespace strings "continue to raise the existing errors", but `string contains only whitespace` is **introduced by the patch** and is not pre-existing |
+| `click/2068` f2 | `{editor}: Editing timed out after {timeout} seconds` — and the count is not pluralised, so one second reads "after 1 seconds" |
+| `click/2068` f3 | `Failed to create backup file: {error}` |
+| `click/2068` f8 | the test inspects `Popen` kwargs for `preexec_fn=os.setpgrp`; `start_new_session=True` is equally correct on POSIX, so a reasoner picks one at 50/50 |
+| `dottxt/1706` f4, f5 | exact `ValueError` texts — the conditions were stated, the wording was not |
+| `llama_index/18813` f2 | `{method} exceeds maximum allowed size ({size} > {max_bytes})` |
+| `dirty_equals/43` f5 | country codes are matched **case-insensitively** — the reference calls `.upper()`, the spec was silent, and the test block is literally commented "Case insensitive country codes" |
+| `dirty_equals/43` f6 | issuer tokens are exactly `'Visa'`, `'Mastercard'`, `'AmericanExpress'` — the spec wrote "American Express" with a space, which is not what is accepted |
+
+## One left deliberately unchanged, and why
+
+`dottxt/1655` f8 (`ssn`) accepts `000-00-0000` and `999-99-9999` as **valid**, so it is a pure
+`XXX-XX-XXXX` format check rather than real SSN validation, which rejects the 000, 666 and 9xx
+area ranges. A strong reasoner might apply the real rules and fail. I left it: the spec says
+"format" three times, gives the `XXX-XX-XXXX` template, and never mentions area or group numbers,
+so format-only is what is asked. Flagged here because it is the closest call in the whole audit.
+
+## Shape of the corpus
+
+174 of 199 specs needed nothing, and a large fraction are **over-complete** rather than thin —
+`react_hook_form` names prop types, file paths and even `React.useRef<NodeJS.Timeout | null>`;
+`dspy/8394` f2 gives the exact `sha256(namespace.encode()).hexdigest()[:8]` key formula;
+`jinja/1559` f4 quotes its own error strings; `click/2800` f7 ships an API Specification section.
+
+The defects concentrate by **language and by author habit**, not evenly:
+
+- **Go tasks are the worst** for unnamed API. Three of the four `go_chi` features that needed
+  amendment described behaviour in prose and named no identifier, and in one case the repo's own
+  vocabulary (`URLParams`) actively misleads toward the wrong name.
+- **Rust (typst) is the worst for messages**, because its harness compares diagnostics verbatim —
+  7 of 10 features there needed the exact text.
+- **Python tasks are largely fine**, and where they are not it is error wording rather than API.
+
+---
+
+# Execution pass — both pending items resolved on Modal
+
+Two items were left needing a container run because the fix could have broken the reference. Both
+now verified against the real task images. The criterion for a gradeable feature is two runs:
+**tests + gold must pass, tests alone must fail.**
+
+## pillow/290 feature 4 — the reference does not honour its own threshold
+
+Measured with the gold patch applied, printing both the wrapped and the correct MSE:
+
+| quantization | wrapped (what was asserted) | correct | colours |
+|---|---|---|---|
+| `quantize(256)` | 18.43 | 27.16 | 256 |
+| `quantize(256, error_threshold=30.0)` | 29.80 | **59.34** | 90 |
+| `quantize(256, error_threshold=20.0)` | 19.84 | **29.85** | 203 |
+
+The old assertions were `with_threshold_mse <= 30.0` and `low_threshold_mse <= 20.0`. Under correct
+arithmetic those are **59.34** and **29.85** — so the reference implementation does not actually
+keep the error under the requested threshold. The assertions passed only because `uint8`
+subtraction and squaring wrap, and the wrapped values happened to land just under the limits.
+
+That rules out the obvious repair. Re-tuning to `<= 60` / `<= 30` would preserve gradeability while
+asserting a number with no relationship to the stated semantic. What *is* both meaningful and true
+of the reference is the ordering: a stricter threshold leaves less error and keeps more colours.
+
+**Rewritten to** `assert high_quality_mse < low_threshold_mse < with_threshold_mse`, on
+`int64`-cast arrays, keeping the colour-count assertions untouched. Verified:
+
+```
+WITH gold:     1 passed, 8 deselected
+WITHOUT gold:  1 failed — TypeError: Image.quantize() got an
+               unexpected keyword argument 'error_threshold'
+```
+
+**Not fixed, and deliberately so:** the reference's threshold semantics. Correcting it would mean
+changing `feature.patch`, which the rules forbid — a task whose reference is wrong is a task-design
+question, not a repair. Recorded here so it is not rediscovered as a model failure.
+
+## pallets_jinja/1621 feature 5 — the rewritten tests do what they claim
+
+```
+WITH gold:     48 passed
+WITHOUT gold:  3 failed, 45 passed
+               AttributeError: 'FileSystemLoader' object has no attribute '_path_cache'
+               - test_filesystem_loader_records_resolved_path
+               - test_filesystem_loader_records_miss_as_none
+               - test_filesystem_loader_drops_cached_path_when_file_vanishes
+```
+
+Exactly the three tests added, and only those. Before the rewrite all six of this feature's tests
+passed on the unmodified base, so it scored 5/5 across archived runs regardless of whether anything
+was implemented — including once for an agent whose patch contained no implementation at all.
+
+---
+
+# Gradeability sweep — running every feature's tests with and without gold
+
+The spec audit asks whether a requirement is *knowable*. It cannot tell you whether the tests
+actually measure the feature, or whether the reference even passes them. So all 199 were run twice
+on Modal through `runner.sh`: **tests alone must FAIL, tests + gold must PASS**
+(`scripts/check_gradeable.py`).
+
+**Result: 175 OK, 24 broken (12%).** None of the 24 were findable by reading specs.
+
+## The two dependency-drift failures — both bricked whole tasks
+
+**`dspy/8563`, all 6 features.** On a pristine checkout, with no patches at all:
+
+```
+4 failed, 16 passed
+dspy/clients/base_lm.py:79:  "usage": dict(response.usage)
+AttributeError: 'ModelResponse' object has no attribute 'usage'
+```
+
+`ModelResponse` is a litellm type, and dspy declares `litellm>=1.64.0` with **no upper bound**, so
+the image floated to a version that dropped `.usage`. The runner executes
+`tests/adapters/test_chat_adapter.py` in full, so those four unrelated failures fail every feature
+of the task regardless of what an agent writes.
+
+Pinned to `litellm==1.77.1` in all four dspy Dockerfiles → **20 passed**. Note the traceback
+surfaces inside `pydantic/main.py`, and pydantic 2.9.2 / 2.10.6 / 2.11.7 all reproduce it
+identically — reading the frame above the top one is what found the real culprit.
+
+**`huggingface_datasets/7309`, both features.** `test_parquet_read_geoparquet` fails
+`assert 'large_string' == 'string'` on a pristine checkout. Pinning does **not** help: identical at
+pyarrow 17.0.0, 18.1.0 and 19.0.1. The test is unrelated to either feature (parquet streaming
+filters / sorting), so the runner now deselects it → `51 passed, 1 deselected`. Deselecting shrinks
+coverage and is the weaker fix, but the alternative was leaving both features permanently
+unscoreable.
+
+Both are the same shape: **a runner that executes whole test files inherits every unrelated
+failure in them.** That is also why a feature's score currently depends on its partner not breaking
+anything in the same file — a real source of noise in the coop setting, where both patches land in
+one tree.
+
+## Deploying a runner or Dockerfile fix
+
+Neither takes effect by editing this repo. `runner.sh` is `COPY`'d into the image at build time, so
+until the image is rebuilt and pushed the old copy runs — 9 of the 24 failures were my own fixes
+looking like defects for exactly this reason. Three further traps, all hit once:
+
+1. **Architecture.** The published images are multi-arch (amd64 + arm64). A plain `docker build` on
+   Apple Silicon produces arm64 only; pushing that breaks every amd64 run. Always `--platform
+   linux/amd64,linux/arm64`.
+2. **Modal caches by tag.** `Image.from_registry(tag)` is resolved once and cached, so a freshly
+   pushed image under an unchanged tag is ignored. Run once with `MODAL_FORCE_BUILD=1` to refresh.
+   Without this, a correct fix looks like it did nothing — verified on `go_chi/26`, which only
+   went green after the force-build.
+3. **Do not full-rebuild for a runner-only change.** It re-resolves every dependency — which is how
+   these images drifted into being broken — and takes tens of minutes under emulation. Overlay one
+   `COPY` layer on the published image instead. `scripts/rebuild_task_images.sh` picks the mode
+   automatically from whether the Dockerfile changed.
+
+`go_chi/26` is the worked example of the whole chain: `golang:1.21 → 1.22` so the build-tagged test
+compiles, the runner fix so a zero-match cannot score, multi-arch rebuild, push, force-build —
+after which feature 1 is gradeable for the first time (`base_rc=1, gold_rc=0`).
+
+## Two more dependency-drift failures, found on re-verification
+
+Both were invisible until the images were rebuilt, and both broke a whole task.
+
+### `hf/6252` — scipy dragged forward by an unrelated install
+
+Measured by printing versions after each line of `runner.sh`:
+
+| after | numpy | scipy |
+|---|---|---|
+| image as shipped | 1.26.4 | 1.13.1 |
+| `uv pip install torch jax` | **2.5.2** | **1.18.0** |
+| `uv pip install "numpy<2.0"` | 1.26.4 | **1.18.0** |
+
+The runner pins numpy but not scipy, so `jax` pulls both forward and only numpy is pulled back.
+scipy 1.18 is built for numpy>=2.0 and calls `np.long`, which numpy removed in 1.24 and
+reintroduced in 2.0 — so `tests/features/test_image.py` dies with *"module 'numpy' has no
+attribute 'long'"* on base **and** gold, and all 6 features read as ungradeable.
+
+Two wrong attempts are worth recording, because both looked plausible:
+`scipy>=1.11,<2.0` in the Dockerfile resolved to 1.18 — the newest under 2.0, i.e. exactly the
+broken one — and pinning in the Dockerfile at all is futile, since `runner.sh` re-resolves at test
+time. **Fixed** by `uv pip install --system "scipy<1.14"` in `runner.sh`, after the numpy
+downgrade. Verified `OK 6`.
+
+### `dottxt/1706` — a live download, then a warning escalated to an error
+
+`tests/backends/*.py` build their fixtures from `erwanf/gpt2-mini`. When that download fails in a
+fresh sandbox, 10 tests ERROR identically on base and gold, and `runner.sh:83` fails the run on any
+`FAILED|ERROR` — so gold can never pass and all 8 features read as ungradeable. It reproduced in a
+single sandbox, so it was never a concurrency artefact; it looked like one because a sandbox that
+had already cached the model passed.
+
+Baking the model in took it from 10 errors to 1. The survivor was not a download failure at all:
+`pyproject.toml:115` sets `filterwarnings = ["error", ...]`, and huggingface_hub's Xet transfer
+calls a deprecated `hf_xet` entry point, so *any* download raises `DeprecationWarning` as an ERROR.
+
+**Fixed** with `ENV HF_HUB_DISABLE_XET=1` plus a cache of both fixture models
+(`erwanf/gpt2-mini` and the `TinyMistral-248M` GGUF). Verified `OK 8`.
+
+Both fixes live in the image, so they reach a run through Docker Hub rather than through
+`dataset/`. Note that `CooperTrain/dataset` is a **separate copy** from `CooperBench/dataset`:
+`runner.sh` and `Dockerfile` changes travel inside the image, but `feature.md` and `tests.patch`
+are read from the dataset directory at eval time, so that copy has to be kept in sync.
+
+## Still outstanding
+
+Nothing blocking. Every one of the 199 features has been measured base-fails / gold-passes.
+
+The one item deliberately left unfixed is `pillow/290` f4's reference threshold semantics
+(above) — gradeability holds, but the reference does not honour its own `error_threshold`, and
+correcting it would mean editing `feature.patch`.
diff --git a/dataset/dottxt_ai_outlines_task/task1706/Dockerfile b/dataset/dottxt_ai_outlines_task/task1706/Dockerfile
index bf85d869..f2b7d038 100644
--- a/dataset/dottxt_ai_outlines_task/task1706/Dockerfile
+++ b/dataset/dottxt_ai_outlines_task/task1706/Dockerfile
@@ -28,6 +28,20 @@ RUN uv pip install --system -e . && \
     uv pip install --system pytest pytest-xdist pytest_mock pytest-asyncio pytest-benchmark pytest-cov && \
     uv pip install --system torch "transformers<5" "tokenizers<0.21" sentencepiece xgrammar llama-cpp-python==0.3.16 psutil
 
+# pyproject.toml sets filterwarnings = ["error", ...], and huggingface_hub's Xet transfer calls a
+# now-deprecated hf_xet entry point, so any download raises DeprecationWarning as an ERROR.
+ENV HF_HUB_DISABLE_XET=1
+
+# Cache the models tests/backends/*.py builds its fixtures from, so grading never depends on a
+# live HuggingFace fetch (a failed download ERRORs 10 tests on base and gold alike).
+RUN python -c "\
+from transformers import AutoModelForCausalLM, AutoTokenizer; \
+AutoModelForCausalLM.from_pretrained('erwanf/gpt2-mini'); \
+AutoTokenizer.from_pretrained('erwanf/gpt2-mini')"
+RUN python -c "\
+from huggingface_hub import hf_hub_download; \
+hf_hub_download('M4-ai/TinyMistral-248M-v2-Instruct-GGUF', 'TinyMistral-248M-v2-Instruct.Q4_K_M.gguf')"
+
 # Copy the runner script
 COPY runner.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/runner.sh
diff --git a/dataset/dottxt_ai_outlines_task/task1706/feature4/feature.md b/dataset/dottxt_ai_outlines_task/task1706/feature4/feature.md
index 1acdd963..b8898879 100644
--- a/dataset/dottxt_ai_outlines_task/task1706/feature4/feature.md
+++ b/dataset/dottxt_ai_outlines_task/task1706/feature4/feature.md
@@ -1,5 +1,9 @@
 **Title**: Add batch size validation to XGrammarLogitsProcessor
 
+The rejection raises `ValueError` with the text
+`Batch size {batch_size} exceeds maximum allowed batch size {max_batch_size}`.
+
+
 **Pull Request Details**
 Adds a `max_batch_size` parameter to XGrammarLogitsProcessor to validate and limit batch sizes during initialization,
 preventing resource exhaustion from oversized batches.
diff --git a/dataset/dottxt_ai_outlines_task/task1706/feature5/feature.md b/dataset/dottxt_ai_outlines_task/task1706/feature5/feature.md
index 64809353..244ecab8 100644
--- a/dataset/dottxt_ai_outlines_task/task1706/feature5/feature.md
+++ b/dataset/dottxt_ai_outlines_task/task1706/feature5/feature.md
@@ -19,6 +19,9 @@ This feature enables developers to use custom tensor libraries with XGrammarLogi
    - Accept optional `custom_adapter` parameter for custom tensor implementations
    - Validate that `custom_adapter` is provided when `tensor_library_name="custom"`
    - Validate that `custom_adapter` is only provided when using "custom" library
+   - The two `ValueError` messages are
+     `custom_adapter must be provided when tensor_library_name is 'custom'` and
+     `custom_adapter should only be provided when tensor_library_name is 'custom'`
    - Override the tensor_adapter when custom adapter is provided
 
 2. Update the `XGrammarBackend` methods to:
diff --git a/dataset/dottxt_ai_outlines_task/task1706/runner.sh b/dataset/dottxt_ai_outlines_task/task1706/runner.sh
index 653367ee..a7cf5df7 100644
--- a/dataset/dottxt_ai_outlines_task/task1706/runner.sh
+++ b/dataset/dottxt_ai_outlines_task/task1706/runner.sh
@@ -77,8 +77,10 @@ timeout 300 python -m pytest tests/backends/test_xgrammar.py -v 2>&1 | tee /tmp/
 PYTEST_EXIT=${PIPESTATUS[0]}  # Get exit code of timeout/pytest, not tee
 set -e  # Re-enable exit on error
 
-# Check if tests passed in the output (even if there was a segfault)
-if grep -qE "passed.*skipped|passed" /tmp/pytest_output.log && ! grep -qE "FAILED|ERROR" /tmp/pytest_output.log; then
+# Check if tests passed in the output (even if there was a segfault).
+# The count must be non-zero: a bare "passed" also matches the "0 passed, N skipped" summary, so
+# a run whose tests were all skipped would otherwise be scored as a pass with nothing executed.
+if grep -qE "[1-9][0-9]* passed" /tmp/pytest_output.log && ! grep -qE "FAILED|ERROR" /tmp/pytest_output.log; then
     echo ""
     echo "Test execution completed!"
     # If exit code is 139 (segfault) but tests passed, exit with 0
diff --git a/dataset/dspy_task/task8394/Dockerfile b/dataset/dspy_task/task8394/Dockerfile
index 49cad9f7..d76eddea 100644
--- a/dataset/dspy_task/task8394/Dockerfile
+++ b/dataset/dspy_task/task8394/Dockerfile
@@ -19,6 +19,14 @@ WORKDIR /workspace/repo
 RUN pip install --upgrade pip && \
     pip install -e ".[dev]"
 
+# dspy declares `litellm>=1.64.0` with no upper bound, so the image floats to whatever is
+# current at build time. In newer litellm `ModelResponse` no longer carries `.usage`, and
+# `base_lm.py` does `dict(response.usage)` unconditionally — which turns four tests in
+# tests/adapters/test_chat_adapter.py red on a PRISTINE checkout. Because the runner
+# executes that file in full, every feature of the task then fails regardless of what the
+# agent wrote. Pinned to the newest version where the base suite is green (20 passed).
+RUN pip install "litellm==1.77.1"
+
 # Copy the runner script
 COPY runner.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/runner.sh
diff --git a/dataset/dspy_task/task8394/runner.sh b/dataset/dspy_task/task8394/runner.sh
index eee5bac8..547e1bc2 100644
--- a/dataset/dspy_task/task8394/runner.sh
+++ b/dataset/dspy_task/task8394/runner.sh
@@ -78,22 +78,25 @@ pip install -e ".[dev]"
 
 # Run tests with timeout and better error handling
 echo "Running tests..."
-timeout 300 python -m pytest "tests/clients/test_cache.py" -v --tb=short --maxfail=1
+# No --maxfail: stopping at the first failure left 2 of 14 tests unexecuted in observed runs, so
+# the reported tests_failed count was a floor rather than a count. `timeout 300` already bounds it.
+timeout 300 python -m pytest "tests/clients/test_cache.py" -v --tb=short
 
-# Run secondary tests if they exist (only after feature patches are applied)
-if [[ -n "$FEATURE_PATCH" ]]; then
-    SECONDARY_TEST_SPEC="tests/clients/test_cache_namespace.py tests/clients/test_cache_ttl.py"
-    if [[ -n "$SECONDARY_TEST_SPEC" ]]; then
-        IFS=' ' read -ra TEST_FILES <<< "$SECONDARY_TEST_SPEC"
-        for file in "${TEST_FILES[@]}"; do
-            if [[ -e "$file" ]]; then
-                echo "Running secondary test: $file"
-                python -m pytest "$file" -v
-            else
-                echo "Skipping missing test target: $file"
-            fi
-        done
+# Run whatever test files this feature's patch actually creates or modifies, in addition to the
+# primary suite. Two failure modes this avoids, both observed:
+#   * gating secondary tests on "$FEATURE_PATCH" means the base run never executes the feature's
+#     own tests, so a feature scores as passing on an untouched tree;
+#   * hardcoding a filename misses per-feature files (test_grounded_proposer1.py .. 5.py), so the
+#     graded tests never run at all in either pass.
+PATCH_TARGETS=$(grep -oE '^\+\+\+ b/.*' "/patches/$TEST_PATCH" 2>/dev/null | sed 's|^+++ b/||' | sort -u)
+for file in $PATCH_TARGETS; do
+    if [[ -e "$file" ]]; then
+        echo "Running tests from the test patch: $file"
+        timeout 300 python -m pytest "$file" -v || exit $?
+    else
+        echo "Error: test target named by the patch is missing: $file"
+        exit 1
     fi
-fi
+done
 
 echo "Test execution completed!"
diff --git a/dataset/dspy_task/task8563/Dockerfile b/dataset/dspy_task/task8563/Dockerfile
index 1dbd36f0..fd65682c 100644
--- a/dataset/dspy_task/task8563/Dockerfile
+++ b/dataset/dspy_task/task8563/Dockerfile
@@ -19,6 +19,14 @@ WORKDIR /workspace/repo
 RUN pip install --upgrade pip && \
     pip install -e ".[dev]"
 
+# dspy declares `litellm>=1.64.0` with no upper bound, so the image floats to whatever is
+# current at build time. In newer litellm `ModelResponse` no longer carries `.usage`, and
+# `base_lm.py` does `dict(response.usage)` unconditionally — which turns four tests in
+# tests/adapters/test_chat_adapter.py red on a PRISTINE checkout. Because the runner
+# executes that file in full, every feature of the task then fails regardless of what the
+# agent wrote. Pinned to the newest version where the base suite is green (20 passed).
+RUN pip install "litellm==1.77.1"
+
 # Copy the runner script
 COPY runner.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/runner.sh
diff --git a/dataset/dspy_task/task8563/feature2/tests.patch b/dataset/dspy_task/task8563/feature2/tests.patch
index 3f18a639..3e52cab9 100644
--- a/dataset/dspy_task/task8563/feature2/tests.patch
+++ b/dataset/dspy_task/task8563/feature2/tests.patch
@@ -1,5 +1,5 @@
 diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py
-index 18571453..8b16200b 100644
+index 185714539..cc2b3609e 100644
 --- a/tests/adapters/test_tool.py
 +++ b/tests/adapters/test_tool.py
 @@ -5,7 +5,7 @@ import pytest
@@ -11,66 +11,7 @@ index 18571453..8b16200b 100644
  
  
  # Test fixtures
-@@ -394,42 +394,33 @@ def test_async_tool_call_in_sync_mode():
- 
- 
- TOOL_CALL_TEST_CASES = [
--    ([], [{"type": "tool_calls", "tool_calls": []}]),
-+    ([], {"tool_calls": []}),
-     (
-         [{"name": "search", "args": {"query": "hello"}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
-+        },
-     ),
-     (
-         [
-             {"name": "search", "args": {"query": "hello"}},
-             {"name": "translate", "args": {"text": "world", "lang": "fr"}},
-         ],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [
--                    {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
--                    {
--                        "type": "function",
--                        "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
--                    },
--                ],
--            }
--        ],
-+        {
-+            "tool_calls": [
-+                {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
-+                {
-+                    "type": "function",
-+                    "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
-+                },
-+            ],
-+        },
-     ),
-     (
-         [{"name": "get_time", "args": {}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
-+        },
-     ),
- ]
- 
-@@ -444,6 +435,81 @@ def test_tool_calls_format_basic(tool_calls_data, expected):
+@@ -444,6 +444,81 @@ def test_tool_calls_format_basic(tool_calls_data, expected):
      assert result == expected
  
  
@@ -152,4 +93,3 @@ index 18571453..8b16200b 100644
  def test_tool_calls_format_from_dict_list():
      """Test format works with ToolCalls created from from_dict_list."""
      tool_calls_dicts = [
-
diff --git a/dataset/dspy_task/task8563/feature3/tests.patch b/dataset/dspy_task/task8563/feature3/tests.patch
index 836270e6..decbc343 100644
--- a/dataset/dspy_task/task8563/feature3/tests.patch
+++ b/dataset/dspy_task/task8563/feature3/tests.patch
@@ -1,5 +1,5 @@
 diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py
-index 18571453..26eb0028 100644
+index 185714539..cb2a8d968 100644
 --- a/tests/adapters/test_tool.py
 +++ b/tests/adapters/test_tool.py
 @@ -5,7 +5,7 @@ import pytest
@@ -11,69 +11,12 @@ index 18571453..26eb0028 100644
  
  
  # Test fixtures
-@@ -394,42 +394,33 @@ def test_async_tool_call_in_sync_mode():
- 
- 
- TOOL_CALL_TEST_CASES = [
--    ([], [{"type": "tool_calls", "tool_calls": []}]),
-+    ([], {"tool_calls": []}),
-     (
-         [{"name": "search", "args": {"query": "hello"}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
-+        },
-     ),
-     (
-         [
-             {"name": "search", "args": {"query": "hello"}},
-             {"name": "translate", "args": {"text": "world", "lang": "fr"}},
-         ],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [
--                    {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
--                    {
--                        "type": "function",
--                        "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
--                    },
--                ],
--            }
--        ],
-+        {
-+            "tool_calls": [
-+                {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
-+                {
-+                    "type": "function",
-+                    "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
-+                },
-+            ],
-+        },
-     ),
-     (
-         [{"name": "get_time", "args": {}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
-+        },
-     ),
- ]
- 
-@@ -434,6 +425,117 @@ TOOL_CALL_TEST_CASES = [
- ]
- 
- 
+@@ -457,3 +457,115 @@ def test_tool_calls_format_from_dict_list():
+     assert len(result[0]["tool_calls"]) == 2
+     assert result[0]["tool_calls"][0]["function"]["name"] == "search"
+     assert result[0]["tool_calls"][1]["function"]["name"] == "translate"
++
++
 +# Test cases for type coercion feature (PR #8563)
 +def test_tool_coercion_boolean_strings():
 +    """Test boolean string coercion: "true"/"false" → True/False (case-insensitive)."""
@@ -184,7 +127,3 @@ index 18571453..26eb0028 100644
 +    # Test with invalid duration string - should keep original and let validation fail
 +    with pytest.raises(ValueError, match="Arg duration is invalid"):
 +        tool(number="42", duration="invalid")
-+
- @pytest.mark.parametrize("tool_calls_data,expected", TOOL_CALL_TEST_CASES)
- def test_tool_calls_format_basic(tool_calls_data, expected):
-     """Test ToolCalls.format with various basic scenarios."""
diff --git a/dataset/dspy_task/task8563/feature4/tests.patch b/dataset/dspy_task/task8563/feature4/tests.patch
index d5bbef88..31d02fba 100644
--- a/dataset/dspy_task/task8563/feature4/tests.patch
+++ b/dataset/dspy_task/task8563/feature4/tests.patch
@@ -1,5 +1,5 @@
 diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py
-index 18571453..183f8725 100644
+index 185714539..d4927bfee 100644
 --- a/tests/adapters/test_tool.py
 +++ b/tests/adapters/test_tool.py
 @@ -5,7 +5,7 @@ import pytest
@@ -11,66 +11,7 @@ index 18571453..183f8725 100644
  
  
  # Test fixtures
-@@ -394,42 +394,33 @@ def test_async_tool_call_in_sync_mode():
- 
- 
- TOOL_CALL_TEST_CASES = [
--    ([], [{"type": "tool_calls", "tool_calls": []}]),
-+    ([], {"tool_calls": []}),
-     (
-         [{"name": "search", "args": {"query": "hello"}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
-+        },
-     ),
-     (
-         [
-             {"name": "search", "args": {"query": "hello"}},
-             {"name": "translate", "args": {"text": "world", "lang": "fr"}},
-         ],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [
--                    {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
--                    {
--                        "type": "function",
--                        "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
--                    },
--                ],
--            }
--        ],
-+        {
-+            "tool_calls": [
-+                {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
-+                {
-+                    "type": "function",
-+                    "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
-+                },
-+            ],
-+        },
-     ),
-     (
-         [{"name": "get_time", "args": {}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
-+        },
-     ),
- ]
- 
-@@ -170,6 +161,155 @@ def test_tool_from_function_with_pydantic():
+@@ -170,6 +170,155 @@ def test_tool_from_function_with_pydantic():
      assert tool.args["model"]["properties"]["field1"]["default"] == "hello"
  
  
diff --git a/dataset/dspy_task/task8563/feature5/tests.patch b/dataset/dspy_task/task8563/feature5/tests.patch
index b5a2d84a..e5a1d626 100644
--- a/dataset/dspy_task/task8563/feature5/tests.patch
+++ b/dataset/dspy_task/task8563/feature5/tests.patch
@@ -1,5 +1,5 @@
 diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py
-index 18571453..4a34c5e8 100644
+index 185714539..bda5fe91e 100644
 --- a/tests/adapters/test_tool.py
 +++ b/tests/adapters/test_tool.py
 @@ -5,7 +5,7 @@ import pytest
@@ -11,66 +11,7 @@ index 18571453..4a34c5e8 100644
  
  
  # Test fixtures
-@@ -394,42 +394,33 @@ def test_async_tool_call_in_sync_mode():
- 
- 
- TOOL_CALL_TEST_CASES = [
--    ([], [{"type": "tool_calls", "tool_calls": []}]),
-+    ([], {"tool_calls": []}),
-     (
-         [{"name": "search", "args": {"query": "hello"}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
-+        },
-     ),
-     (
-         [
-             {"name": "search", "args": {"query": "hello"}},
-             {"name": "translate", "args": {"text": "world", "lang": "fr"}},
-         ],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [
--                    {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
--                    {
--                        "type": "function",
--                        "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
--                    },
--                ],
--            }
--        ],
-+        {
-+            "tool_calls": [
-+                {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
-+                {
-+                    "type": "function",
-+                    "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
-+                },
-+            ],
-+        },
-     ),
-     (
-         [{"name": "get_time", "args": {}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
-+        },
-     ),
- ]
- 
-@@ -217,6 +208,262 @@ def test_parameter_desc():
+@@ -217,6 +217,262 @@ def test_parameter_desc():
      assert tool.args["x"]["description"] == "The x parameter"
  
  
diff --git a/dataset/dspy_task/task8563/feature6/tests.patch b/dataset/dspy_task/task8563/feature6/tests.patch
index c013d278..6bccd5f1 100644
--- a/dataset/dspy_task/task8563/feature6/tests.patch
+++ b/dataset/dspy_task/task8563/feature6/tests.patch
@@ -1,5 +1,5 @@
 diff --git a/tests/adapters/test_tool.py b/tests/adapters/test_tool.py
-index 18571453..f3823a50 100644
+index 185714539..4e849662c 100644
 --- a/tests/adapters/test_tool.py
 +++ b/tests/adapters/test_tool.py
 @@ -5,7 +5,7 @@ import pytest
@@ -11,66 +11,7 @@ index 18571453..f3823a50 100644
  
  
  # Test fixtures
-@@ -394,42 +394,33 @@ def test_async_tool_call_in_sync_mode():
- 
- 
- TOOL_CALL_TEST_CASES = [
--    ([], [{"type": "tool_calls", "tool_calls": []}]),
-+    ([], {"tool_calls": []}),
-     (
-         [{"name": "search", "args": {"query": "hello"}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}}],
-+        },
-     ),
-     (
-         [
-             {"name": "search", "args": {"query": "hello"}},
-             {"name": "translate", "args": {"text": "world", "lang": "fr"}},
-         ],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [
--                    {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
--                    {
--                        "type": "function",
--                        "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
--                    },
--                ],
--            }
--        ],
-+        {
-+            "tool_calls": [
-+                {"type": "function", "function": {"name": "search", "arguments": {"query": "hello"}}},
-+                {
-+                    "type": "function",
-+                    "function": {"name": "translate", "arguments": {"text": "world", "lang": "fr"}},
-+                },
-+            ],
-+        },
-     ),
-     (
-         [{"name": "get_time", "args": {}}],
--        [
--            {
--                "type": "tool_calls",
--                "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
--            }
--        ],
-+        {
-+            "tool_calls": [{"type": "function", "function": {"name": "get_time", "arguments": {}}}],
-+        },
-     ),
- ]
- 
-@@ -55,6 +46,129 @@ class Note(BaseModel):
+@@ -55,6 +55,129 @@ class Note(BaseModel):
      author: str
  
  
diff --git a/dataset/dspy_task/task8563/runner.sh b/dataset/dspy_task/task8563/runner.sh
index a091bf27..340fc721 100644
--- a/dataset/dspy_task/task8563/runner.sh
+++ b/dataset/dspy_task/task8563/runner.sh
@@ -78,7 +78,9 @@ pip install -e ".[dev]"
 
 # Run tests with timeout and better error handling
 echo "Running tests..."
-timeout 300 python -m pytest "tests/adapters/test_tool.py" "tests/adapters/test_chat_adapter.py" -v --tb=short --maxfail=1
+# No --maxfail: stopping at the first failure leaves later tests unexecuted, so the reported
+# failure count is a floor rather than a count. `timeout 300` already bounds the run.
+timeout 300 python -m pytest "tests/adapters/test_tool.py" "tests/adapters/test_chat_adapter.py" -v --tb=short
 
 # Run secondary tests if they exist (only after feature patches are applied)
 if [[ -n "$FEATURE_PATCH" ]]; then
diff --git a/dataset/dspy_task/task8587/Dockerfile b/dataset/dspy_task/task8587/Dockerfile
index 1dbd36f0..fd65682c 100644
--- a/dataset/dspy_task/task8587/Dockerfile
+++ b/dataset/dspy_task/task8587/Dockerfile
@@ -19,6 +19,14 @@ WORKDIR /workspace/repo
 RUN pip install --upgrade pip && \
     pip install -e ".[dev]"
 
+# dspy declares `litellm>=1.64.0` with no upper bound, so the image floats to whatever is
+# current at build time. In newer litellm `ModelResponse` no longer carries `.usage`, and
+# `base_lm.py` does `dict(response.usage)` unconditionally — which turns four tests in
+# tests/adapters/test_chat_adapter.py red on a PRISTINE checkout. Because the runner
+# executes that file in full, every feature of the task then fails regardless of what the
+# agent wrote. Pinned to the newest version where the base suite is green (20 passed).
+RUN pip install "litellm==1.77.1"
+
 # Copy the runner script
 COPY runner.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/runner.sh
diff --git a/dataset/dspy_task/task8587/runner.sh b/dataset/dspy_task/task8587/runner.sh
index c8ff8475..2c32ce5c 100644
--- a/dataset/dspy_task/task8587/runner.sh
+++ b/dataset/dspy_task/task8587/runner.sh
@@ -78,7 +78,9 @@ pip install -e ".[dev]"
 
 # Run tests with timeout and better error handling
 echo "Running tests..."
-timeout 300 python -m pytest "tests/streaming/test_streaming.py" -v --tb=short --maxfail=1
+# No --maxfail: stopping at the first failure leaves later tests unexecuted, so the reported
+# failure count is a floor rather than a count. `timeout 300` already bounds the run.
+timeout 300 python -m pytest "tests/streaming/test_streaming.py" -v --tb=short
 
 # Run secondary tests if they exist (only after feature patches are applied)
 if [[ -n "$FEATURE_PATCH" ]]; then
diff --git a/dataset/dspy_task/task8635/Dockerfile b/dataset/dspy_task/task8635/Dockerfile
index deeaa451..7f6c878b 100644
--- a/dataset/dspy_task/task8635/Dockerfile
+++ b/dataset/dspy_task/task8635/Dockerfile
@@ -19,6 +19,14 @@ WORKDIR /workspace/repo
 RUN pip install --upgrade pip && \
     pip install -e ".[dev]"
 
+# dspy declares `litellm>=1.64.0` with no upper bound, so the image floats to whatever is
+# current at build time. In newer litellm `ModelResponse` no longer carries `.usage`, and
+# `base_lm.py` does `dict(response.usage)` unconditionally — which turns four tests in
+# tests/adapters/test_chat_adapter.py red on a PRISTINE checkout. Because the runner
+# executes that file in full, every feature of the task then fails regardless of what the
+# agent wrote. Pinned to the newest version where the base suite is green (20 passed).
+RUN pip install "litellm==1.77.1"
+
 # Copy the runner script
 COPY runner.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/runner.sh
diff --git a/dataset/dspy_task/task8635/runner.sh b/dataset/dspy_task/task8635/runner.sh
index 829dd862..8f0e7346 100644
--- a/dataset/dspy_task/task8635/runner.sh
+++ b/dataset/dspy_task/task8635/runner.sh
@@ -78,7 +78,9 @@ pip install -e ".[dev]"
 
 # Run tests with timeout and better error handling
 echo "Running tests..."
-timeout 300 python -m pytest "tests/teleprompt/test_utils.py" "tests/propose/test_grounded_proposer.py" -v --tb=short --maxfail=1
+# No --maxfail: stopping at the first failure leaves later tests unexecuted, so the reported
+# failure count is a floor rather than a count. `timeout 300` already bounds the run.
+timeout 300 python -m pytest "tests/teleprompt/test_utils.py" "tests/propose/test_grounded_proposer.py" -v --tb=short
 
 # Run secondary tests if they exist (only after feature patches are applied)
 if [[ -n "$FEATURE_PATCH" ]]; then
@@ -96,4 +98,21 @@ if [[ -n "$FEATURE_PATCH" ]]; then
     fi
 fi
 
+# Run whatever test files this feature's patch actually creates or modifies, in addition to the
+# primary suite. Two failure modes this avoids, both observed:
+#   * gating secondary tests on "$FEATURE_PATCH" means the base run never executes the feature's
+#     own tests, so a feature scores as passing on an untouched tree;
+#   * hardcoding a filename misses per-feature files (test_grounded_proposer1.py .. 5.py), so the
+#     graded tests never run at all in either pass.
+PATCH_TARGETS=$(grep -oE '^\+\+\+ b/.*' "/patches/$TEST_PATCH" 2>/dev/null | sed 's|^+++ b/||' | sort -u)
+for file in $PATCH_TARGETS; do
+    if [[ -e "$file" ]]; then
+        echo "Running tests from the test patch: $file"
+        timeout 300 python -m pytest "$file" -v || exit $?
+    else
+        echo "Error: test target named by the patch is missing: $file"
+        exit 1
+    fi
+done
+
 echo "Test execution completed!"
diff --git a/dataset/go_chi_task/task26/Dockerfile b/dataset/go_chi_task/task26/Dockerfile
index f8470d64..33d68d7d 100644
--- a/dataset/go_chi_task/task26/Dockerfile
+++ b/dataset/go_chi_task/task26/Dockerfile
@@ -1,4 +1,8 @@
-FROM golang:1.21-alpine
+# 1.22, not 1.21: the graded test file for feature 1 carries `//go:build go1.22`, and the base
+# commit below is upstream chi's "go 1.22 ci (#898)". Under 1.21 that file is excluded from the
+# build, so `go test -run TestPathValue` matches nothing and exits 0 — which the runner scored as
+# a pass. Feature 1 was ungradeable, passing even for a patch with no implementation in it.
+FROM golang:1.22-alpine
 
 # Install system dependencies (including Python for Modal compatibility)
 RUN apk add --no-cache \
diff --git a/dataset/go_chi_task/task26/feature2/feature.md b/dataset/go_chi_task/task26/feature2/feature.md
index f6ee1b94..6fff26dc 100644
--- a/dataset/go_chi_task/task26/feature2/feature.md
+++ b/dataset/go_chi_task/task26/feature2/feature.md
@@ -15,6 +15,24 @@ Many production applications need to monitor which routes are most frequently ac
 4. **Mux integration** – Add `metricsCollector` field to `Mux` struct and `SetMetricsCollector()` method to configure the collector.  
 5. **Timing instrumentation** – In `routeHTTP()`, measure request duration using `time.Now()` and `time.Since()`, then call `RecordHit()` after handler execution if collector is configured.
 
+**Public API**
+
+Callers depend on these exact names. Note `Params`, not chi's usual `URLParams`.
+
+```go
+type RouteMetric struct {
+    Pattern  string            // matched route pattern, e.g. "/users/{id}"
+    Method   string
+    Path     string
+    Duration time.Duration
+    Params   map[string]string // URL parameters keyed by name, e.g. {"id": "123"}
+}
+
+type SimpleMetricsCollector struct {
+    OnHit func(RouteMetric)
+}
+```
+
 **Files Modified**
 - `mux.go`
 - `metrics.go`
diff --git a/dataset/go_chi_task/task26/feature3/feature.md b/dataset/go_chi_task/task26/feature3/feature.md
index f67ece60..bf22e73f 100644
--- a/dataset/go_chi_task/task26/feature3/feature.md
+++ b/dataset/go_chi_task/task26/feature3/feature.md
@@ -23,6 +23,31 @@ Modern APIs often need to route requests dynamically based on factors like API v
 
 This feature enables request-based dynamic routing by adding a context evaluation phase to the routing process. This allows applications to examine the request context (headers, auth info, etc.) and select the appropriate handler accordingly.
 
+## Public API
+
+Callers depend on these exact names.
+
+```go
+// A selector decides which handler serves a pattern, given the request.
+type RouteSelector interface {
+    SelectRoute(r *http.Request, pattern string, defaultHandler http.Handler) http.Handler
+}
+
+// Attaches a selector to a request, for use from middleware.
+func WithRouteSelector(r *http.Request, selector RouteSelector) *http.Request
+
+// Chooses by API version, read from the request.
+func NewVersionSelector(defaultVersion string) *VersionSelector
+func (vs *VersionSelector) AddHandler(pattern, version string, handler http.Handler)
+
+// Chooses by caller role, extracted by the supplied function.
+func NewRoleBasedSelector(roleExtractor func(r *http.Request) string, defaultRole string) *RoleBasedSelector
+func (rs *RoleBasedSelector) AddHandler(pattern, role string, handler http.Handler)
+```
+
+When no selector is attached, or the selector returns nothing for the pattern, the route's own
+handler serves the request as before.
+
 ## Files Modified
 
 ```
diff --git a/dataset/go_chi_task/task26/feature4/feature.md b/dataset/go_chi_task/task26/feature4/feature.md
index 461b17cd..09fe0155 100644
--- a/dataset/go_chi_task/task26/feature4/feature.md
+++ b/dataset/go_chi_task/task26/feature4/feature.md
@@ -22,6 +22,25 @@ In large applications with many routes, it's common to have situations where mul
 
 This implementation provides a way to explicitly control route matching priority, ensuring that the most appropriate handler is selected when multiple routes could match. This is particularly useful for applications with complex routing requirements, such as those with both specific and catch-all routes.
 
+## Public API
+
+Callers depend on these exact names.
+
+```go
+// Returns a router that registers subsequent routes at the given priority, so it chains:
+//   r.WithPriority(PriorityHigh).Get("/resources", handler)
+func (mx *Mux) WithPriority(priority int) *Mux
+
+// Named levels, highest value wins when several routes match.
+const (
+    PriorityHighest = 100
+    PriorityHigh    = 75
+    PriorityNormal  = 50
+    PriorityLow     = 25
+    PriorityLowest  = 0
+)
+```
+
 ## Files Modified
 
 ```
diff --git a/dataset/go_chi_task/task26/runner.sh b/dataset/go_chi_task/task26/runner.sh
index d74320ee..82e3a95d 100644
--- a/dataset/go_chi_task/task26/runner.sh
+++ b/dataset/go_chi_task/task26/runner.sh
@@ -81,19 +81,33 @@ done
 
 TEST_PATTERN=""
 if [ ${#TEST_FUNCS[@]} -gt 0 ]; then
-    TEST_PATTERN=$(IFS="|"; echo "${TEST_FUNCS[*]}" | sort -u)
+    # Anchored: `-run` takes an unanchored regex, so a bare `TestPathValue` is also satisfied by an
+    # agent-authored `TestPathValueExtra`. Anchoring makes the filter mean the functions the test
+    # patch actually declares.
+    TEST_PATTERN="^($(IFS="|"; echo "${TEST_FUNCS[*]}" | sort -u))$"
     echo "Found test functions to run: $TEST_PATTERN"
 fi
 
-# Run Go tests with timeout
+# Run Go tests with timeout. `-v` so that each executed test prints a `=== RUN` line; the summary
+# `ok <pkg> <time>` lines it also prints are unchanged, so downstream parsing is unaffected.
 echo "Running Go tests..."
 if [ -n "$TEST_PATTERN" ]; then
-    timeout 300 go test -run "$TEST_PATTERN" ./...
+    TEST_OUTPUT=$(timeout 300 go test -v -run "$TEST_PATTERN" ./... 2>&1)
     TEST_EXIT_CODE=$?
 else
-    timeout 300 go test ./...
+    TEST_OUTPUT=$(timeout 300 go test -v ./... 2>&1)
     TEST_EXIT_CODE=$?
 fi
+echo "$TEST_OUTPUT"
+
+# A filter that selects nothing is a failed run, not a passing one. `go test` exits 0 and prints
+# `ok <pkg> <time> [no tests to run]` when its -run pattern matches no test, and treating that as
+# success made this feature scoreable without any implementation at all.
+if [ "$(echo "$TEST_OUTPUT" | grep -c '^=== RUN')" -eq 0 ]; then
+    echo "Error: no test was executed. The filter '$TEST_PATTERN' matched nothing, which means the"
+    echo "       graded tests are absent, excluded by a build constraint, or failed to compile."
+    exit 1
+fi
 
 if [ $TEST_EXIT_CODE -eq 0 ]; then
     echo "All tests passed!"
diff --git a/dataset/go_chi_task/task27/feature3/feature.md b/dataset/go_chi_task/task27/feature3/feature.md
index 1fc481af..14da599f 100644
--- a/dataset/go_chi_task/task27/feature3/feature.md
+++ b/dataset/go_chi_task/task27/feature3/feature.md
@@ -8,6 +8,13 @@ Adds detailed debug-level logging for route pattern handling to make debugging e
 - Logs pattern parsing information including method detection
 - Integrates with existing logging mechanism
 
+## Public API
+
+```go
+// Turns on debug-level logging; off by default.
+func EnableDebugLogging()
+```
+
 ## Issue Context:
 
 Developers have reported difficulty debugging route handling issues, particularly with complex patterns or when methods are involved. This change provides more visibility into the pattern handling process.
diff --git a/dataset/go_chi_task/task27/runner.sh b/dataset/go_chi_task/task27/runner.sh
index ddf60625..83e5bdb3 100644
--- a/dataset/go_chi_task/task27/runner.sh
+++ b/dataset/go_chi_task/task27/runner.sh
@@ -81,19 +81,32 @@ done
 
 TEST_PATTERN=""
 if [ ${#TEST_FUNCS[@]} -gt 0 ]; then
-    TEST_PATTERN=$(IFS="|"; echo "${TEST_FUNCS[*]}" | sort -u)
+    # Anchored: `-run` takes an unanchored regex, so a bare `TestFoo` is also satisfied by an
+    # agent-authored `TestFooExtra`. Anchoring makes the filter mean the functions the test
+    # patch actually declares.
+    TEST_PATTERN="^($(IFS="|"; echo "${TEST_FUNCS[*]}" | sort -u))$"
     echo "Found test functions to run: $TEST_PATTERN"
 fi
 
 # Run Go tests with timeout
 echo "Running Go tests..."
 if [ -n "$TEST_PATTERN" ]; then
-    timeout 300 go test -run "$TEST_PATTERN" ./...
+    TEST_OUTPUT=$(timeout 300 go test -v -run "$TEST_PATTERN" ./... 2>&1)
     TEST_EXIT_CODE=$?
 else
-    timeout 300 go test ./...
+    TEST_OUTPUT=$(timeout 300 go test -v ./... 2>&1)
     TEST_EXIT_CODE=$?
 fi
+echo "$TEST_OUTPUT"
+
+# A filter that selects nothing is a failed run, not a passing one. `go test` exits 0 and prints
+# `ok <pkg> <time> [no tests to run]` when its -run pattern matches no test, which would let a
+# feature score without any implementation at all.
+if [ "$(echo "$TEST_OUTPUT" | grep -c '^=== RUN')" -eq 0 ]; then
+    echo "Error: no test was executed. The filter '$TEST_PATTERN' matched nothing, which means the"
+    echo "       graded tests are absent, excluded by a build constraint, or failed to compile."
+    exit 1
+fi
 
 if [ "$TEST_EXIT_CODE" -ne 0 ]; then
     echo "Error: Go tests failed with exit code $TEST_EXIT_CODE"
diff --git a/dataset/go_chi_task/task56/runner.sh b/dataset/go_chi_task/task56/runner.sh
index a904795b..9cab59ee 100644
--- a/dataset/go_chi_task/task56/runner.sh
+++ b/dataset/go_chi_task/task56/runner.sh
@@ -81,19 +81,32 @@ done
 
 TEST_PATTERN=""
 if [ ${#TEST_FUNCS[@]} -gt 0 ]; then
-    TEST_PATTERN=$(IFS="|"; echo "${TEST_FUNCS[*]}" | sort -u)
+    # Anchored: `-run` takes an unanchored regex, so a bare `TestFoo` is also satisfied by an
+    # agent-authored `TestFooExtra`. Anchoring makes the filter mean the functions the test
+    # patch actually declares.
+    TEST_PATTERN="^($(IFS="|"; echo "${TEST_FUNCS[*]}" | sort -u))$"
     echo "Found test functions to run: $TEST_PATTERN"
 fi
 
 # Run Go tests with timeout
 echo "Running Go tests..."
 if [ -n "$TEST_PATTERN" ]; then
-    timeout 300 go test -run "$TEST_PATTERN" ./...
+    TEST_OUTPUT=$(timeout 300 go test -v -run "$TEST_PATTERN" ./... 2>&1)
     TEST_EXIT_CODE=$?
 else
-    timeout 300 go test ./...
+    TEST_OUTPUT=$(timeout 300 go test -v ./... 2>&1)
     TEST_EXIT_CODE=$?
 fi
+echo "$TEST_OUTPUT"
+
+# A filter that selects nothing is a failed run, not a passing one. `go test` exits 0 and prints
+# `ok <pkg> <time> [no tests to run]` when its -run pattern matches no test, which would let a
+# feature score without any implementation at all.
+if [ "$(echo "$TEST_OUTPUT" | grep -c '^=== RUN')" -eq 0 ]; then
+    echo "Error: no test was executed. The filter '$TEST_PATTERN' matched nothing, which means the"
+    echo "       graded tests are absent, excluded by a build constraint, or failed to compile."
+    exit 1
+fi
 
 if [ "$TEST_EXIT_CODE" -ne 0 ]; then
     echo "Error: Go tests failed with exit code $TEST_EXIT_CODE"
diff --git a/dataset/huggingface_datasets_task/task3997/feature2/feature.md b/dataset/huggingface_datasets_task/task3997/feature2/feature.md
index 58efedef..1bbd93bf 100644
--- a/dataset/huggingface_datasets_task/task3997/feature2/feature.md
+++ b/dataset/huggingface_datasets_task/task3997/feature2/feature.md
@@ -1,5 +1,13 @@
 **Title**: Custom `require_decoding` Criteria
 
+The hook is a method on `Features`:
+
+```python
+def set_custom_decoding_criteria(self, custom_criteria: Optional[callable] = None): ...
+```
+Passing `None` restores the default behaviour.
+
+
 **Pull Request Details**
 
 **Description**:
diff --git a/dataset/huggingface_datasets_task/task3997/feature3/feature.md b/dataset/huggingface_datasets_task/task3997/feature3/feature.md
index 14e801dd..a3d7042c 100644
--- a/dataset/huggingface_datasets_task/task3997/feature3/feature.md
+++ b/dataset/huggingface_datasets_task/task3997/feature3/feature.md
@@ -1,5 +1,13 @@
 **Title**: Caching for `require_decoding`
 
+The hook is a method on `Features`:
+
+```python
+def set_custom_decoding_criteria(self, custom_criteria: Optional[callable] = None): ...
+```
+Passing `None` restores the default behaviour. The criteria set this way participate in the caching described below.
+
+
 **Pull Request Details**
 
 **Description**:
diff --git a/dataset/huggingface_datasets_task/task6252/Dockerfile b/dataset/huggingface_datasets_task/task6252/Dockerfile
index 851824fc..d06d2df9 100644
--- a/dataset/huggingface_datasets_task/task6252/Dockerfile
+++ b/dataset/huggingface_datasets_task/task6252/Dockerfile
@@ -26,6 +26,11 @@ RUN uv pip install --system "tensorflow>=2.16.0,<2.17.0" || uv pip install --sys
 RUN uv pip install --system torch jax
 # Pin numpy to < 2.0 to avoid compatibility issues with np.array(copy=False)
 RUN uv pip install --system "numpy<2.0"
+# scipy must be pinned to the numpy-1.x era alongside it. An unpinned resolve picks scipy 1.18,
+# which requires numpy>=2.0 and calls `np.long` -- an alias numpy removed in 1.24 and only
+# reintroduced in 2.0 -- so tests/features/test_image.py dies with
+# "module 'numpy' has no attribute 'long'". scipy 1.13.1 is the last release supporting numpy 1.26.
+RUN uv pip install --system "scipy<1.14"
 RUN uv pip install --system pytest pytest-xdist pytest-datadir Pillow
 
 # Copy the runner script
diff --git a/dataset/huggingface_datasets_task/task6252/runner.sh b/dataset/huggingface_datasets_task/task6252/runner.sh
index 521259f8..25108e5a 100644
--- a/dataset/huggingface_datasets_task/task6252/runner.sh
+++ b/dataset/huggingface_datasets_task/task6252/runner.sh
@@ -79,6 +79,12 @@ uv pip install --system "tensorflow==2.16.2" || uv pip install --system "tensorf
 uv pip install --system torch jax
 # Pin numpy to < 2.0 to avoid compatibility issues with np.array(copy=False)
 uv pip install --system "numpy<2.0"
+# scipy has to be pinned back with it. The `torch jax` line above resolves numpy to 2.5.2 and
+# scipy to 1.18.0; downgrading numpy alone leaves scipy 1.18, which is built for numpy>=2.0 and
+# calls `np.long` -- removed in numpy 1.24, reintroduced in 2.0 -- so tests/features/test_image.py
+# dies with "module 'numpy' has no attribute 'long'" on base AND gold. 1.13.1 is the last scipy
+# supporting numpy 1.26.
+uv pip install --system "scipy<1.14"
 # Now install the package (tensorflow already installed, so dependency resolver won't try tensorflow-macos)
 uv pip install --system -e .
 uv pip uninstall --system pyarrow
diff --git a/dataset/huggingface_datasets_task/task7309/runner.sh b/dataset/huggingface_datasets_task/task7309/runner.sh
index d77750b9..ab3ca170 100644
--- a/dataset/huggingface_datasets_task/task7309/runner.sh
+++ b/dataset/huggingface_datasets_task/task7309/runner.sh
@@ -87,7 +87,12 @@ uv pip install --system pytest pytest-xdist pytest-datadir Pillow  # Common test
 
 # Run test
 echo "RUNNING_TESTS..."
-timeout 300 python -m pytest "$TEST_PATH" -v
+# test_parquet_read_geoparquet is red on a PRISTINE checkout (`assert 'large_string' ==
+# 'string'`, unaffected by pyarrow 17/18/19) and is unrelated to either feature of this task.
+# Because the runner executes the whole file, leaving it in makes BOTH features permanently
+# unscoreable no matter what the agent writes. Deselected so the base suite is green.
+timeout 300 python -m pytest "$TEST_PATH" -v \
+    --deselect "tests/io/test_parquet.py::test_parquet_read_geoparquet"
 
 echo "TEST_EXECUTION_COMPLETED"
 
diff --git a/dataset/llama_index_task/task18813/feature2/feature.md b/dataset/llama_index_task/task18813/feature2/feature.md
index 70403da9..528bed9d 100644
--- a/dataset/llama_index_task/task18813/feature2/feature.md
+++ b/dataset/llama_index_task/task18813/feature2/feature.md
@@ -10,7 +10,9 @@ Introduces an optional `max_bytes` parameter to the `resolve_image`, `resolve_au
 Large media payloads can cause memory issues or exceed downstream model limits. There was no mechanism to enforce size constraints on resolved binary content.
 
 **Solution**:
-Add a `max_bytes` argument to each resolve method. After resolving the buffer, check its size, and raise a `ValueError` if it exceeds the limit. 
+Add a `max_bytes` argument to each resolve method. After resolving the buffer, check its size, and raise a `ValueError` if it exceeds the limit. The message is
+`{method_name} exceeds maximum allowed size ({size} > {max_bytes})`, e.g.
+`resolve_audio exceeds maximum allowed size (100 > 10)`. 
 
 **Files Modified**
 * llama-index-core/llama_index/core/base/llms/types.py
diff --git a/dataset/openai_tiktoken_task/task0/runner.sh b/dataset/openai_tiktoken_task/task0/runner.sh
index a135f2e4..c802ba5b 100644
--- a/dataset/openai_tiktoken_task/task0/runner.sh
+++ b/dataset/openai_tiktoken_task/task0/runner.sh
@@ -6,7 +6,8 @@ set -e
 cleanup() {
     if git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
         git reset --hard HEAD 2>/dev/null || true
-        git clean -fdx 2>/dev/null || true
+        # No -x: keeps the image's prebuilt native extension.
+        git clean -fd 2>/dev/null || true
     fi
 }
 
diff --git a/dataset/pallets_click_task/task2068/feature2/feature.md b/dataset/pallets_click_task/task2068/feature2/feature.md
index 435f5080..ccea013d 100644
--- a/dataset/pallets_click_task/task2068/feature2/feature.md
+++ b/dataset/pallets_click_task/task2068/feature2/feature.md
@@ -5,7 +5,7 @@
 
 **Technical Background**: The current click.edit() function launches an editor process and waits indefinitely for it to complete. This can cause problems in several scenarios: automated scripts that get stuck waiting for user input, editors that hang or become unresponsive, and CI/CD pipelines that need guaranteed execution times. Users currently have no way to automatically terminate editor sessions that exceed reasonable time limits, potentially causing scripts to hang indefinitely.
 
-**Solution**:  The implementation adds an optional timeout parameter to the edit() function that specifies the maximum number of seconds to wait for the editor process to complete. When the timeout is reached, the editor process is terminated and a ClickException is raised with a descriptive error message. The timeout parameter includes validation to ensure only positive integer values are accepted, raising a ValueError for invalid inputs.
+**Solution**:  The implementation adds an optional timeout parameter to the edit() function that specifies the maximum number of seconds to wait for the editor process to complete. When the timeout is reached, the editor process is terminated and a ClickException is raised with a descriptive error message, whose text is `{editor}: Editing timed out after {timeout} seconds` — note the count is interpolated without pluralisation, so one second reads "after 1 seconds". The timeout parameter includes validation to ensure only positive integer values are accepted, raising a ValueError for invalid inputs.
 
 Key changes include:
 
diff --git a/dataset/pallets_click_task/task2068/feature3/feature.md b/dataset/pallets_click_task/task2068/feature3/feature.md
index 146604a6..bfd44e23 100644
--- a/dataset/pallets_click_task/task2068/feature3/feature.md
+++ b/dataset/pallets_click_task/task2068/feature3/feature.md
@@ -16,6 +16,8 @@ The implementation adds an optional `auto_backup` parameter to the `click.edit()
 3. **Safety Implementation**: 
    - Create backup only when editing existing files (not for new temporary files)
    - Handle file I/O errors gracefully with informative error messages
+   - When the backup cannot be written, raise `ClickException` with the text
+     `Failed to create backup file: {error}`
    - Preserve original file permissions on backup copies
 4. **Cross-Platform Support**: Use platform-appropriate file operations for reliable backup creation
 5. **Error Handling**:
diff --git a/dataset/pallets_click_task/task2068/feature8/feature.md b/dataset/pallets_click_task/task2068/feature8/feature.md
index 4d4e8fb8..2ffa8c39 100644
--- a/dataset/pallets_click_task/task2068/feature8/feature.md
+++ b/dataset/pallets_click_task/task2068/feature8/feature.md
@@ -6,6 +6,8 @@ Enhance the click.edit() function with an optional process_group parameter that
 **Description**:
 This feature adds a new `process_group` parameter to the `click.edit()` function that allows developers to control whether the spawned editor process runs in its own process group. When enabled, this prevents signal propagation issues between the editor and the parent Click application, providing more robust process management for CLI tools that use external editors.
 
+The editor is spawned with `preexec_fn=os.setpgrp`, so the new group is observable on the `Popen` call itself. (`start_new_session=True` has the same effect on POSIX but is not what callers here check for.)
+
 **Technical Background**:
 Currently, when `click.edit()` spawns an external editor, the editor process inherits the same process group as the parent application. This can lead to signal handling conflicts where signals intended for the parent application (like SIGINT or SIGTERM) are also sent to the editor process, potentially causing unexpected behavior or data loss. Additionally, some editors may send signals that interfere with the parent application's normal operation, particularly in complex CLI workflows or when editors are used within larger automation scripts.
 
diff --git a/dataset/pallets_jinja_task/task1559/feature3/tests.patch b/dataset/pallets_jinja_task/task1559/feature3/tests.patch
index 984bec27..0c94e07c 100644
--- a/dataset/pallets_jinja_task/task1559/feature3/tests.patch
+++ b/dataset/pallets_jinja_task/task1559/feature3/tests.patch
@@ -1,8 +1,8 @@
 diff --git a/tests/test_ext.py b/tests/test_ext.py
-index b54e905f..6de3a267 100644
+index b54e905f..21d9708e 100644
 --- a/tests/test_ext.py
 +++ b/tests/test_ext.py
-@@ -599,6 +599,273 @@ class TestNewstyleInternationalization:
+@@ -599,6 +599,320 @@ class TestNewstyleInternationalization:
          assert tmpl.render(LANGUAGE="de", apples=5) == "5 Apples"
  
  
@@ -272,6 +272,53 @@ index b54e905f..6de3a267 100644
 +        assert result == "Hola"
 +
 +
++    def test_priority_retries_until_translation_found(self):
++        """gettext is retried at incrementing priority levels until it returns a translation."""
++        calls = []
++
++        def counting_gettext(message):
++            calls.append(message)
++            # Untranslated at the first two priority levels, translated at the third.
++            return "TRANSLATED" if len(calls) >= 3 else message
++
++        env = Environment(extensions=["jinja2.ext.i18n"])
++        env.globals.update({
++            "gettext": counting_gettext,
++            "ngettext": lambda s, p, n: s if n == 1 else p,
++        })
++
++        tmpl = env.from_string('{% trans priority=5 %}Hello{% endtrans %}')
++        assert tmpl.render() == "TRANSLATED"
++        assert len(calls) == 3
++
++    def test_priority_zero_calls_gettext_once(self):
++        """priority=0 must not retry, so a missing translation stays untranslated."""
++        calls = []
++
++        def counting_gettext(message):
++            calls.append(message)
++            return message
++
++        env = Environment(extensions=["jinja2.ext.i18n"])
++        env.globals.update({
++            "gettext": counting_gettext,
++            "ngettext": lambda s, p, n: s if n == 1 else p,
++        })
++
++        tmpl = env.from_string('{% trans priority=0 %}Hello{% endtrans %}')
++        assert tmpl.render() == "Hello"
++        assert len(calls) == 1
++
++    def test_priority_is_not_a_template_variable(self):
++        """priority is consumed by the tag, not exposed for substitution."""
++        env = Environment(extensions=["jinja2.ext.i18n"])
++        env.globals.update({
++            "gettext": lambda x: x,
++            "ngettext": lambda s, p, n: s if n == 1 else p,
++        })
++
++        tmpl = env.from_string('{% trans priority=7, name="World" %}Hello {{ name }}{% endtrans %}')
++        assert tmpl.render() == "Hello World"
 +
  class TestAutoEscape:
      def test_scoped_setting(self):
diff --git a/dataset/pallets_jinja_task/task1559/feature8/feature.md b/dataset/pallets_jinja_task/task1559/feature8/feature.md
index 6e1c315a..303d3271 100644
--- a/dataset/pallets_jinja_task/task1559/feature8/feature.md
+++ b/dataset/pallets_jinja_task/task1559/feature8/feature.md
@@ -19,6 +19,10 @@ Introduce a new `ctrans` template tag that conditionally renders translations ba
    * Add module-level alias: `conditional_i18n = ConditionalI18nExtension`
 
 2. Implement template tag syntax: `{% ctrans condition [variables] [trimmed|notrimmed] %}...{% pluralize [variable] %}...{% fallback %}...{% endctrans %}`
+   * **Commas are optional separators anywhere in the header.** `{% ctrans use_i18n, count %}` and
+     `{% ctrans use_i18n count %}` are both valid.
+   * **`trimmed` / `notrimmed` apply to every block in the tag**, including `{% fallback %}`, not
+     only to the singular and plural blocks.
 
 3. Inside the extension's `parse()` method:
    * Parse the condition expression using `parser.parse_expression()`
diff --git a/dataset/pallets_jinja_task/task1621/feature5/tests.patch b/dataset/pallets_jinja_task/task1621/feature5/tests.patch
index efc0c518..c37884f6 100644
--- a/dataset/pallets_jinja_task/task1621/feature5/tests.patch
+++ b/dataset/pallets_jinja_task/task1621/feature5/tests.patch
@@ -1,8 +1,8 @@
 diff --git a/tests/test_loader.py b/tests/test_loader.py
-index b300c8f2..8a269a21 100644
+index b300c8f2..91287439 100644
 --- a/tests/test_loader.py
 +++ b/tests/test_loader.py
-@@ -116,6 +116,129 @@ class TestLoaders:
+@@ -116,6 +116,170 @@ class TestLoaders:
          assert split_template_path("./foo/bar") == ["foo", "bar"]
          pytest.raises(TemplateNotFound, split_template_path, "../foo")
  
@@ -103,31 +103,72 @@ index b300c8f2..8a269a21 100644
 +        tmpl_nested = env.get_template("foo/test.html")
 +        assert tmpl_nested.render().strip() == "FOO"
 +
-+    def test_filesystem_loader_cache_performance(self):
-+        """Test that caching works correctly on repeated access."""
-+        import time
-+ 
++    def test_filesystem_loader_records_resolved_path(self):
++        """A successful lookup stores the resolved path under the template name."""
 +        searchpath = (Path(__file__) / ".." / "res" / "templates").resolve()
-+        loader1 = loaders.FileSystemLoader(str(searchpath))
-+        loader2 = loaders.FileSystemLoader(str(searchpath))
-+        env1 = Environment(loader=loader1)
-+        env2 = Environment(loader=loader2)
-+ 
-+        # Time repeated access with same loader (should benefit from caching)
-+        start_time = time.time()
-+        for _ in range(10):
-+            env1.get_template("test.html")
-+        cached_time = time.time() - start_time
-+ 
-+        # Time access with fresh loader (no cache benefit)
-+        start_time = time.time()
-+        for _ in range(10):
-+            env2.get_template("test.html")
-+        fresh_time = time.time() - start_time
-+ 
-+        # Both should be fast, this verifies the cache mechanism works without accessing internals
-+        assert cached_time >= 0
-+        assert fresh_time >= 0
++        loader = loaders.FileSystemLoader(str(searchpath))
++        env = Environment(loader=loader)
++
++        assert loader._path_cache == {}
++
++        loader.get_source(env, "test.html")
++
++        assert "test.html" in loader._path_cache
++        cached = loader._path_cache["test.html"]
++        assert cached is not None
++        assert os.path.isfile(cached)
++
++    def test_filesystem_loader_records_miss_as_none(self):
++        """A failed lookup is stored as None and short-circuits later lookups."""
++        import shutil
++        import tempfile
++
++        temp_dir = tempfile.mkdtemp()
++        try:
++            loader = loaders.FileSystemLoader(temp_dir)
++            env = Environment(loader=loader)
++
++            with pytest.raises(TemplateNotFound):
++                loader.get_source(env, "later.html")
++
++            assert "later.html" in loader._path_cache
++            assert loader._path_cache["later.html"] is None
++
++            # The file exists now, but the cached miss answers without touching
++            # the filesystem, so the template is still reported as not found.
++            with open(os.path.join(temp_dir, "later.html"), "w") as f:
++                f.write("appeared")
++
++            with pytest.raises(TemplateNotFound):
++                loader.get_source(env, "later.html")
++        finally:
++            shutil.rmtree(temp_dir, ignore_errors=True)
++
++    def test_filesystem_loader_drops_cached_path_when_file_vanishes(self):
++        """A cached path whose file was removed is re-resolved, not reused."""
++        import shutil
++        import tempfile
++
++        temp_dir = tempfile.mkdtemp()
++        try:
++            path = os.path.join(temp_dir, "gone.html")
++            with open(path, "w") as f:
++                f.write("here")
++
++            loader = loaders.FileSystemLoader(temp_dir)
++            env = Environment(loader=loader)
++
++            contents, filename, _ = loader.get_source(env, "gone.html")
++            assert contents == "here"
++            assert loader._path_cache["gone.html"] == filename
++
++            os.remove(path)
++
++            with pytest.raises(TemplateNotFound):
++                loader.get_source(env, "gone.html")
++            assert loader._path_cache["gone.html"] is None
++        finally:
++            shutil.rmtree(temp_dir, ignore_errors=True)
 +
  
  class TestFileSystemLoader:
diff --git a/dataset/pillow_task/task25/runner.sh b/dataset/pillow_task/task25/runner.sh
index 8c2ed912..ff7aae29 100644
--- a/dataset/pillow_task/task25/runner.sh
+++ b/dataset/pillow_task/task25/runner.sh
@@ -6,7 +6,8 @@ set -e
 cleanup() {
     if git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
         git reset --hard HEAD 2>/dev/null || true
-        git clean -fdx 2>/dev/null || true
+        # No -x: keeps the image's prebuilt native extension.
+        git clean -fd 2>/dev/null || true
     fi
 }
 
diff --git a/dataset/pillow_task/task290/feature4/tests.patch b/dataset/pillow_task/task290/feature4/tests.patch
index c46e5214..b0473562 100644
--- a/dataset/pillow_task/task290/feature4/tests.patch
+++ b/dataset/pillow_task/task290/feature4/tests.patch
@@ -2,7 +2,7 @@ diff --git a/Tests/test_image_quantize.py b/Tests/test_image_quantize.py
 index bd9db362c..a73f17bcc 100644
 --- a/Tests/test_image_quantize.py
 +++ b/Tests/test_image_quantize.py
-@@ -85,3 +85,49 @@ def test_transparent_colors_equal():
+@@ -85,3 +85,50 @@ def test_transparent_colors_equal():
      converted = im.quantize()
      converted_px = converted.load()
      assert converted_px[0, 0] == converted_px[0, 1]
@@ -31,18 +31,19 @@ index bd9db362c..a73f17bcc 100644
 +    with_threshold_array = np.array(with_threshold_rgb)
 +    low_threshold_array = np.array(low_threshold_rgb)
 + 
-+    high_quality_mse = np.mean((image_array - high_quality_array) ** 2)
-+    with_threshold_mse = np.mean((image_array - with_threshold_array) ** 2)
-+    low_threshold_mse = np.mean((image_array - low_threshold_array) ** 2)
++    orig = image_array.astype(np.int64)  # uint8 arithmetic wraps: 16**2 == 0
++    high_quality_mse = np.mean((orig - high_quality_array.astype(np.int64)) ** 2)
++    with_threshold_mse = np.mean((orig - with_threshold_array.astype(np.int64)) ** 2)
++    low_threshold_mse = np.mean((orig - low_threshold_array.astype(np.int64)) ** 2)
 + 
 +    # Count colors in each image
 +    high_quality_colors = len(high_quality.getcolors(256) or [])
 +    with_threshold_colors = len(with_threshold.getcolors(256) or [])
 +    low_threshold_colors = len(low_threshold.getcolors(256) or [])
 + 
-+    # Assert that the error threshold is respected
-+    assert with_threshold_mse <= 30.0
-+    assert low_threshold_mse <= 20.0
++    # A stricter threshold must leave less error than a looser one, and both must
++    # leave more than unconstrained quantization.
++    assert high_quality_mse < low_threshold_mse < with_threshold_mse
 + 
 +    # Assert that higher error threshold uses fewer colors
 +    assert with_threshold_colors < high_quality_colors
diff --git a/dataset/pillow_task/task290/feature5/feature.md b/dataset/pillow_task/task290/feature5/feature.md
index e49f4ec7..6e0f303b 100644
--- a/dataset/pillow_task/task290/feature5/feature.md
+++ b/dataset/pillow_task/task290/feature5/feature.md
@@ -17,6 +17,8 @@ Currently, users must perform a separate brightness adjustment operation after q
 
 A `lightness_factor` greater than 1.0 makes the image brighter (e.g., 1.1 for 10% brighter), while a value less than 1.0 makes it darker (e.g., 0.9 for 10% darker). The implementation ensures that values don't exceed valid color ranges (0-255).
 
+`lightness_factor` must be strictly positive: a value of `0` or below raises `ValueError`. There is no upper bound — any positive value is accepted, with the resulting channel values clamped to 0-255.
+
 This feature allows for fine-tuning the appearance of quantized images without requiring additional processing steps or color conversions.
 
 **Files Modified**
diff --git a/dataset/pillow_task/task290/runner.sh b/dataset/pillow_task/task290/runner.sh
index 8501b1b2..70d401b1 100644
--- a/dataset/pillow_task/task290/runner.sh
+++ b/dataset/pillow_task/task290/runner.sh
@@ -7,7 +7,8 @@ cleanup() {
     echo "CLEANING_UP: Restoring repository to original state..."
     if git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
         git reset --hard HEAD 2>/dev/null || true
-        git clean -fdx 2>/dev/null || true
+        # No -x: keeps the image's prebuilt native extension.
+        git clean -fd 2>/dev/null || true
         echo "CLEANUP: Repository restored to clean state"
     fi
 }
diff --git a/dataset/pillow_task/task68/runner.sh b/dataset/pillow_task/task68/runner.sh
index aeaea65e..158cab6e 100644
--- a/dataset/pillow_task/task68/runner.sh
+++ b/dataset/pillow_task/task68/runner.sh
@@ -7,7 +7,8 @@ cleanup() {
     echo "CLEANING_UP: Restoring repository to original state..."
     if git rev-parse --is-inside-work-tree > /dev/null 2>&1; then
         git reset --hard HEAD 2>/dev/null || true
-        git clean -fdx 2>/dev/null || true
+        # No -x: keeps the image's prebuilt native extension.
+        git clean -fd 2>/dev/null || true
         echo "CLEANUP: Repository restored to clean state"
     fi
 }
diff --git a/dataset/samuelcolvin_dirty_equals_task/task43/feature5/feature.md b/dataset/samuelcolvin_dirty_equals_task/task43/feature5/feature.md
index 4bfe6dd2..6f4c62a6 100644
--- a/dataset/samuelcolvin_dirty_equals_task/task43/feature5/feature.md
+++ b/dataset/samuelcolvin_dirty_equals_task/task43/feature5/feature.md
@@ -30,6 +30,8 @@ seamless integration into existing codebases. Note that location-specific phone
 - Should accept commonly used placeholder/example numbers (e.g., 555-xxx-xxxx format for US numbers)
 - A US number should be rejected by `IsPhoneNumber('GB')` and vice versa
 - **Supported Countries**: The validator provides specific validation rules for the following countries:
+  - Country codes are matched **case-insensitively**: `IsPhoneNumber('gb')` behaves the same
+    as `IsPhoneNumber('GB')`.
   - US (United States): Accepts 10-digit national format (e.g., `555-123-4567`) or international format with country code +1 (e.g., `+1-555-123-4567`)
   - GB (United Kingdom): Accepts national format starting with 0 (e.g., `020 7946 0958`) or international format with country code +44 (e.g., `+44 20 7946 0958`)
 - **National Format Requirements**: For country-specific validation, only proper national formats or correct international formats should be accepted. Malformed numbers (e.g., 11 digits without proper country code for US) should be rejected.
diff --git a/dataset/samuelcolvin_dirty_equals_task/task43/feature6/feature.md b/dataset/samuelcolvin_dirty_equals_task/task43/feature6/feature.md
index 65710d99..193c310d 100644
--- a/dataset/samuelcolvin_dirty_equals_task/task43/feature6/feature.md
+++ b/dataset/samuelcolvin_dirty_equals_task/task43/feature6/feature.md
@@ -7,7 +7,7 @@ issuer-specific validation for enhanced payment processing workflows.
 **Description**:
 The IsCreditCard validator provides robust credit card number validation by implementing the industry-standard Luhn
 algorithm to verify card number integrity. Users can perform basic validation against any valid credit card number or
-specify a particular issuer (Visa, Mastercard, American Express, etc.) for more targeted validation. This enables
+specify a particular issuer. The accepted issuer names are exactly `'Visa'`, `'Mastercard'` and `'AmericanExpress'` — matched case-sensitively, and note `AmericanExpress` is one word for more targeted validation. This enables
 developers to validate payment inputs with confidence while supporting both general and issuer-specific use cases in
 e-commerce and financial applications.
 
diff --git a/dataset/typst_task/task6554/feature10/feature.md b/dataset/typst_task/task6554/feature10/feature.md
index 9dff5762..143060c9 100644
--- a/dataset/typst_task/task6554/feature10/feature.md
+++ b/dataset/typst_task/task6554/feature10/feature.md
@@ -34,5 +34,11 @@ error.
 - Reuse existing behaviour when the unit is omitted or set to `"grapheme"`.
 - Emit a descriptive error for unsupported unit strings and for strings without any words.
 
+**Error messages are compared verbatim by the test harness.** The exact wording is:
+
+- no word segment found: `string contains no words`
+- an unrecognised unit: `unknown unit "<value>"`, with the value in double quotes
+
+
 **Files Modified:**
 - `crates/typst/src/foundations/str.rs`
diff --git a/dataset/typst_task/task6554/feature2/feature.md b/dataset/typst_task/task6554/feature2/feature.md
index 8e2650fc..11db1b18 100644
--- a/dataset/typst_task/task6554/feature2/feature.md
+++ b/dataset/typst_task/task6554/feature2/feature.md
@@ -43,5 +43,10 @@ pub fn first(
 - Proper error handling for out-of-bounds access
 - Full Unicode grapheme cluster support
 
+**Error messages are compared verbatim by the test harness.** The exact wording is:
+
+- out of range: `index <i> out of bounds for string with <n> characters`
+
+
 **Files Modified:**
 - `crates/typst/src/foundations/str.rs` - Method implementations  
diff --git a/dataset/typst_task/task6554/feature4/feature.md b/dataset/typst_task/task6554/feature4/feature.md
index 5813b773..8ea2c040 100644
--- a/dataset/typst_task/task6554/feature4/feature.md
+++ b/dataset/typst_task/task6554/feature4/feature.md
@@ -45,5 +45,11 @@ pub fn first(
 - Proper error handling when repeat exceeds string length
 - Full Unicode grapheme cluster support
 
+**Error messages are compared verbatim by the test harness.** The exact wording is:
+
+- too few characters: `not enough characters (requested <n>, but string only has <m>)`
+- a non-positive repeat: `repeat must be at least 1`
+
+
 **Files Modified:**
 - `crates/typst/src/foundations/str.rs` - Method implementations
diff --git a/dataset/typst_task/task6554/feature5/feature.md b/dataset/typst_task/task6554/feature5/feature.md
index 444d20d9..936edf9f 100644
--- a/dataset/typst_task/task6554/feature5/feature.md
+++ b/dataset/typst_task/task6554/feature5/feature.md
@@ -49,5 +49,11 @@ pub fn first(
 - Proper error handling when count exceeds string length
 - Full Unicode grapheme cluster support
 
+**Error messages are compared verbatim by the test harness.** The exact wording is:
+
+- too few characters: `not enough characters (requested <n>, but string only has <m>)`
+- a non-positive count: `count must be at least 1`
+
+
 **Files Modified:**
 - `crates/typst/src/foundations/str.rs` - Method implementations
diff --git a/dataset/typst_task/task6554/feature6/feature.md b/dataset/typst_task/task6554/feature6/feature.md
index d4940621..33ad6bf0 100644
--- a/dataset/typst_task/task6554/feature6/feature.md
+++ b/dataset/typst_task/task6554/feature6/feature.md
@@ -31,5 +31,11 @@ hello
 - Reuse the existing grapheme-selection logic on the trimmed slice.
 - Preserve current error handling for empty or whitespace-only strings.
 
+**Error messages are compared verbatim by the test harness.** The exact wording is:
+
+- an all-whitespace string, once whitespace is skipped: `string contains only whitespace`
+  (note this is a new message; an empty string keeps the pre-existing `string is empty`)
+
+
 **Files Modified:**
 - `crates/typst/src/foundations/str.rs`
diff --git a/dataset/typst_task/task6554/feature7/feature.md b/dataset/typst_task/task6554/feature7/feature.md
index 4b95b671..3dca85d9 100644
--- a/dataset/typst_task/task6554/feature7/feature.md
+++ b/dataset/typst_task/task6554/feature7/feature.md
@@ -27,6 +27,11 @@ Add a named `pattern` argument supporting values such as `"alpha"`, `"numeric"`,
 - Parse the optional `pattern` argument and normalise to an enum of supported classes.
 - Iterate graphemes with `unicode_segmentation::UnicodeSegmentation`, testing the first char of each cluster.
 - Return the matching grapheme or propagate the existing `string is empty` / out-of-pattern errors.
+- Error messages are compared verbatim by the test harness. When no grapheme matches, the message
+  is exactly `no character matches pattern "<name>"` — for example
+  `no character matches pattern "alpha"`, with the pattern name in double quotes.
+- Check for an empty string **before** checking the pattern, so `"".last(pattern: "alpha")` still
+  reports `string is empty` rather than the no-match message.
 - Surface a descriptive error when the caller passes an unknown pattern string.
 
 **Files Modified:**
diff --git a/dataset/typst_task/task6554/feature8/feature.md b/dataset/typst_task/task6554/feature8/feature.md
index 76b10e43..245c2924 100644
--- a/dataset/typst_task/task6554/feature8/feature.md
+++ b/dataset/typst_task/task6554/feature8/feature.md
@@ -32,7 +32,12 @@ in the requested case. Invalid values raise a descriptive error.
 - Parse the optional `case` argument in both methods.
 - Transform the selected grapheme using Rust's Unicode-aware case conversion helpers.
 - Handle title casing by uppercasing the first scalar value and lowercasing the rest.
-- Error out when an unsupported `case` value is provided.
+- Error out when an unsupported `case` value is provided. Error messages are compared verbatim by
+  the test harness; the message is exactly `unknown case option "<value>"`, with the offending
+  value in double quotes — for example `unknown case option "spongebob"`.
+- The harness compares the diagnostic's source range as well as its text. Report the error against
+  the whole method call expression (the default when the failure is returned from the function),
+  not against the offending argument.
 - Leave existing behaviour unchanged when the parameter is omitted.
 
 **Files Modified:**
diff --git a/dataset/typst_task/task6554/runner.sh b/dataset/typst_task/task6554/runner.sh
index b52a74e8..0b1ffb1b 100644
--- a/dataset/typst_task/task6554/runner.sh
+++ b/dataset/typst_task/task6554/runner.sh
@@ -56,9 +56,64 @@ fi
 echo "Building Typst..."
 cargo build --package typst --package typst-cli
 
-# Run tests (string-first and string-last tests)
-echo "Running tests..."
-cargo test -p typst-tests -- string-first
-cargo test -p typst-tests -- string-last
+# Run the tests this feature's patch actually declares.
+#
+# These filters used to be hardcoded to `string-first` and `string-last`, which are the block names
+# feature 7 happens to use. Every other feature of this task declares `str-*` blocks — feature 8's
+# are `str-case-parameter` / `str-case-invalid` — so for features 2,3,4,5,6,8,9,10 the graded tests
+# were never executed at all, and the feature scored on whatever the base suite did.
+echo "Deriving test filters from the test patch..."
+BLOCKS=$(grep -oE '^\+--- [A-Za-z0-9_-]+ ---' "/patches/$TEST_PATCH" 2>/dev/null \
+         | sed -E 's/^\+--- (.*) ---$/\1/' | sort -u || true)
+
+FAILED=0
+RAN_ANY=0
+
+run_and_check() {
+    # $1 = filter, or empty for the whole suite
+    local filter="$1" out rc
+    if [ -n "$filter" ]; then
+        echo "Running tests..."
+        out=$(cargo test -p typst-tests -- "$filter" 2>&1) && rc=0 || rc=$?
+    else
+        echo "Running the whole suite..."
+        out=$(cargo test -p typst-tests 2>&1) && rc=0 || rc=$?
+    fi
+    echo "$out"
+    # typst-tests prints "N passed, M failed, K skipped". A filter that selects nothing still exits
+    # 0 with "0 passed, 0 failed" — that is a broken filter, not a pass.
+    if echo "$out" | grep -qE '^[0-9]+ passed, [0-9]+ failed'; then
+        local executed
+        executed=$(echo "$out" | grep -oE '^[0-9]+ passed, [0-9]+ failed' \
+                   | awk '{s+=$1+$3} END {print s+0}')
+        [ "$executed" -gt 0 ] && RAN_ANY=1
+    fi
+    [ "$rc" -ne 0 ] && FAILED=1
+    return 0
+}
+
+if [ -z "$BLOCKS" ]; then
+    # Some feature patches add cases inside an existing block instead of declaring a new one
+    # (feature 1 does). There is no filter that isolates those, so grade against the whole suite.
+    echo "Test patch declares no new test blocks; grading against the full suite."
+    run_and_check ""
+else
+    echo "Test blocks declared by this patch: $(echo "$BLOCKS" | tr '\n' ' ')"
+    while IFS= read -r block; do
+        [ -z "$block" ] && continue
+        run_and_check "$block"
+    done <<< "$BLOCKS"
+fi
+
+if [ "$RAN_ANY" -eq 0 ]; then
+    echo "Error: no test was executed. The derived filters matched nothing, which means the graded"
+    echo "       tests are absent or the block names in the test patch have changed."
+    exit 1
+fi
+
+if [ "$FAILED" -ne 0 ]; then
+    echo "Some tests failed."
+    exit 1
+fi
 
 echo "Test execution completed!"
diff --git a/scripts/check_gradeable.py b/scripts/check_gradeable.py
new file mode 100644
index 00000000..67b75bec
--- /dev/null
+++ b/scripts/check_gradeable.py
@@ -0,0 +1,172 @@
+"""Check every feature is gradeable: its tests must FAIL on the base commit and PASS with gold.
+
+This is the property the whole dataset depends on, and neither half is safe to assume:
+
+    tests alone  ->  must FAIL     otherwise the tests do not measure the feature, and it scores
+                                   for any submission at all (pallets_jinja/1621 f5 passed 6/6 on
+                                   an untouched tree, so it scored 5/5 across archived runs
+                                   regardless of what the agent wrote)
+    tests + gold ->  must PASS     otherwise the reference contradicts its own tests and no
+                                   correct implementation can score
+
+Both runs go through `runner.sh`, which is the real grading path — language-specific invocation,
+filters and exit-code handling included — rather than a hand-rolled pytest call that would miss
+exactly the runner defects this is meant to catch.
+
+One sandbox per feature, so a runner's `git clean -fdx` cleanup trap cannot leak into the next
+measurement. Runs on Modal; nothing is built locally.
+
+    python scripts/check_gradeable.py                        # all 199
+    python scripts/check_gradeable.py pillow_task/task290     # one task
+    python scripts/check_gradeable.py --workers 16
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import subprocess
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+from cooperbench.eval.backends import get_backend  # noqa: E402
+from cooperbench.utils import get_image_name  # noqa: E402
+
+DATASET = Path(__file__).resolve().parents[1] / "dataset"
+REPORT = DATASET / "gradeable_report.json"
+REPORT_LOCAL = DATASET / "gradeable_report_localrunner.json"
+
+
+def _amd64_ref(image: str) -> str:
+    """Pin the amd64 manifest by digest.
+
+    These images are multi-arch; Modal runs amd64 only, and when it resolves the tag itself it can
+    pick the arm64 manifest and fail the image build with "image architecture arm64 not supported"
+    — then CACHE that failure, so every later run of that task fails instantly. Handing it a
+    digest removes the choice. Falls back to the plain tag if the registry cannot be reached.
+    """
+    try:
+        raw = subprocess.run(["docker", "buildx", "imagetools", "inspect", image, "--raw"],
+                             capture_output=True, text=True, timeout=60)
+        if raw.returncode != 0:
+            return image
+        for m in json.loads(raw.stdout).get("manifests", []):
+            p = m.get("platform", {})
+            if p.get("architecture") == "amd64" and p.get("os") == "linux":
+                return f"{image}@{m['digest']}"
+    except Exception:
+        pass
+    return image
+
+
+def _write(sb, path: str, content: str) -> None:
+    enc = base64.b64encode(content.encode()).decode()
+    sb.exec("bash", "-c", f"echo '{enc}' | base64 -d > {path}")
+
+
+def check_feature(repo: str, task: str, feature: str, local_runner: bool = False) -> dict:
+    fd = DATASET / repo / task / feature
+    tests, gold = fd / "tests.patch", fd / "feature.patch"
+    out = {"task": f"{repo}/{task}", "feature": feature}
+    if not (tests.is_file() and gold.is_file()):
+        return {**out, "verdict": "SKIP", "note": "missing tests.patch or feature.patch"}
+
+    image = _amd64_ref(get_image_name(repo, int(task.replace("task", ""))))
+    started = time.time()
+    sb = get_backend("modal").create_sandbox(image, timeout=3600)
+    try:
+        sb.exec("bash", "-c", "mkdir -p /patches")
+        if local_runner:
+            # runner.sh is COPY'd into the image at build time, so edits to it in this repo are
+            # invisible until the image is rebuilt and pushed. Overwrite it in the sandbox to
+            # measure the fixed runner rather than the stale baked one.
+            _write(sb, "/usr/local/bin/runner.sh", (DATASET / repo / task / "runner.sh").read_text())
+            sb.exec("bash", "-c", "chmod +x /usr/local/bin/runner.sh")
+        _write(sb, "/patches/tests.patch", tests.read_text())
+        _write(sb, "/patches/feature.patch", gold.read_text())
+
+        # NOT `runner.sh ... | tail`: a pipeline exits with the status of its LAST command, so
+        # piping into tail reports tail's 0 and every feature looks like it passes. Redirect to a
+        # file, keep the runner's own status, then read the tail back.
+        def run(args: str):
+            # Grep the signal out rather than tail blindly: these runners print a long
+            # `git clean` inventory on exit, which pushes the actual error off the end.
+            r = sb.exec("bash", "-c",
+                        f"bash /usr/local/bin/runner.sh {args} > /tmp/out.log 2>&1; echo RC=$?; "
+                        "grep -iE 'does not apply|failed to apply|error:|FAILED|assert|"
+                        "[0-9]+ (passed|failed)|no tests|collected|panic|cannot' /tmp/out.log "
+                        "| grep -viE 'Removing |Repository (cleaned|restored)' | tail -25")
+            body = r.stdout_read() + r.stderr_read()
+            rc = next((int(ln[3:]) for ln in body.splitlines() if ln.startswith("RC=")), -1)
+            return rc, body
+
+        base_rc, base_body = run("tests.patch")
+        gold_rc, gold_body = run("tests.patch feature.patch")
+
+        if gold_rc != 0:
+            verdict = "GOLD_FAILS"          # reference contradicts its own tests
+        elif base_rc == 0:
+            verdict = "PASSES_ON_BASE"      # tests do not measure the feature
+        else:
+            verdict = "OK"
+        return {
+            **out,
+            "verdict": verdict,
+            "base_rc": base_rc,
+            "gold_rc": gold_rc,
+            "seconds": round(time.time() - started),
+            "base_tail": base_body.strip()[-400:],
+            "gold_tail": gold_body.strip()[-400:],
+        }
+    except Exception as exc:
+        return {**out, "verdict": "ERROR", "note": f"{type(exc).__name__}: {exc}"}
+    finally:
+        sb.terminate()
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("task", nargs="?", help="repo/taskN, default all")
+    ap.add_argument("--workers", type=int, default=12)
+    ap.add_argument("--local-runner", action="store_true",
+                    help="overwrite the image's baked-in runner.sh with the one in dataset/, so "
+                         "runner fixes are exercised without rebuilding and pushing images")
+    args = ap.parse_args()
+
+    feats = [
+        (p.parent.parent.name, p.parent.name, p.name)
+        for p in sorted(DATASET.glob("*/task*/feature*"))
+        if p.is_dir() and (not args.task or f"{p.parent.parent.name}/{p.parent.name}" == args.task)
+    ]
+    if not feats:
+        raise SystemExit(f"no features matched {args.task!r}")
+    print(f"checking {len(feats)} features, {args.workers} sandboxes at a time\n", flush=True)
+
+    results, done = [], 0
+    with ThreadPoolExecutor(max_workers=args.workers) as pool:
+        futures = {pool.submit(check_feature, *f, args.local_runner): f for f in feats}
+        for fut in as_completed(futures):
+            r = fut.result()
+            results.append(r)
+            done += 1
+            if r["verdict"] != "OK":
+                print(f"  [{done}/{len(feats)}] {r['verdict']:15s} {r['task']} {r['feature']}"
+                      f" {r.get('note','')}", flush=True)
+            elif done % 20 == 0:
+                print(f"  [{done}/{len(feats)}] ...", flush=True)
+
+    (REPORT_LOCAL if args.local_runner else REPORT).write_text(json.dumps(sorted(results, key=lambda r: (r["task"], r["feature"])), indent=1))
+    tally: dict[str, int] = {}
+    for r in results:
+        tally[r["verdict"]] = tally.get(r["verdict"], 0) + 1
+    print("\n" + " · ".join(f"{k} {v}" for k, v in sorted(tally.items())))
+    print(f"report -> {REPORT_LOCAL if args.local_runner else REPORT}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/rebuild_task_images.sh b/scripts/rebuild_task_images.sh
new file mode 100755
index 00000000..1aad34df
--- /dev/null
+++ b/scripts/rebuild_task_images.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Rebuild and push task images after changing a task's Dockerfile or runner.sh.
+#
+# runner.sh is COPY'd into the image at build time, so edits to it in this repo are invisible to
+# every run until the image is rebuilt and pushed. A Dockerfile change (e.g. a base-image version
+# bump) is the same. Symptom when you forget: the fix appears to do nothing.
+#
+# Two things are easy to get wrong here:
+#
+#   ARCHITECTURE  the published images are multi-arch (linux/amd64 + linux/arm64). Building on
+#                 Apple Silicon without --platform produces arm64 only, and pushing that breaks
+#                 every amd64 run. Always build both.
+#   MODAL CACHE   Modal resolves `Image.from_registry(tag)` once and caches it, so a freshly
+#                 pushed image under an unchanged tag is NOT picked up. Run anything that uses the
+#                 image once with MODAL_FORCE_BUILD=1 afterwards to refresh it.
+#
+#   ./scripts/rebuild_task_images.sh                          # every task changed vs origin/main
+#   ./scripts/rebuild_task_images.sh go_chi_task/task27 ...    # specific tasks
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+PLATFORMS="linux/amd64,linux/arm64"
+
+if [ $# -gt 0 ]; then
+    TASKS=("$@")
+else
+    mapfile -t TASKS < <(
+        git diff --name-only origin/main -- 'dataset/*/task*/runner.sh' 'dataset/*/task*/Dockerfile' \
+        | sed -E 's|dataset/||; s|/(runner\.sh\|Dockerfile)$||' | sort -u
+    )
+fi
+
+[ ${#TASKS[@]} -eq 0 ] && { echo "no task images need rebuilding"; exit 0; }
+echo "rebuilding ${#TASKS[@]} image(s) for $PLATFORMS"
+
+failed=()
+for t in "${TASKS[@]}"; do
+    repo="${t%%/*}"; task="${t##*/}"
+    image=$(python3 -c "
+import sys; sys.path.insert(0, 'src')
+from cooperbench.utils import get_image_name
+print(get_image_name('$repo', int('${task#task}')))")
+    echo ""
+    echo "=== $t -> $image"
+
+    # If only runner.sh changed, overlay one COPY layer on the published image instead of
+    # rebuilding. A full rebuild re-resolves every dependency (the datasets images pull
+    # tensorflow + torch + jax, typst compiles Rust) for two platforms under emulation, which
+    # takes tens of minutes for what is a one-file change — and re-resolving deps is exactly how
+    # these images drifted into being broken in the first place.
+    if git diff --quiet origin/main -- "dataset/$t/Dockerfile"; then
+        ctx=$(mktemp -d)
+        cp "dataset/$t/runner.sh" "$ctx/runner.sh"
+        printf 'FROM %s\nCOPY runner.sh /usr/local/bin/runner.sh\nRUN chmod +x /usr/local/bin/runner.sh\n' \
+            "$image" > "$ctx/Dockerfile"
+        echo "    runner.sh only -> overlaying on the published image"
+        build_ctx="$ctx"
+    else
+        echo "    Dockerfile changed -> full rebuild"
+        build_ctx="dataset/$t"
+    fi
+
+    before=$(docker buildx imagetools inspect "$image" 2>/dev/null | grep -m1 '^Digest:' | awk '{print $2}' || true)
+    log=$(mktemp)
+    if docker buildx build --platform "$PLATFORMS" -t "$image" --push "$build_ctx" >"$log" 2>&1; then
+        after=$(docker buildx imagetools inspect "$image" 2>/dev/null | grep -m1 '^Digest:' | awk '{print $2}' || true)
+        # A zero exit is not proof the tag moved: a push can no-op silently. Compare digests.
+        if [ -n "$before" ] && [ "$before" = "$after" ]; then
+            echo "    PUSH DID NOT TAKE (digest unchanged: ${before:0:19})"
+            tail -5 "$log" | sed 's/^/      /'
+            failed+=("$t")
+        else
+            echo "    pushed  ${after:0:19}"
+        fi
+    else
+        echo "    BUILD FAILED"
+        tail -8 "$log" | sed 's/^/      /'
+        failed+=("$t")
+    fi
+    rm -f "$log"
+    [ -n "${ctx:-}" ] && rm -rf "$ctx" && unset ctx
+done
+
+echo ""
+if [ ${#failed[@]} -gt 0 ]; then
+    printf 'failed: %s\n' "${failed[@]}"
+    exit 1
+fi
+echo "all pushed. Modal still holds the OLD images under these tags — refresh with:"
+echo "    MODAL_FORCE_BUILD=1 python scripts/check_gradeable.py <repo>/<task>"
diff --git a/scripts/spec_audit_status.py b/scripts/spec_audit_status.py
new file mode 100644
index 00000000..63d8939b
--- /dev/null
+++ b/scripts/spec_audit_status.py
@@ -0,0 +1,63 @@
+"""Regenerate the status table in dataset/SPEC_AUDIT.md from the recorded verdicts.
+
+The audit is per-feature and slow, so the verdicts live in a JSON file
+(`dataset/.spec_audit_verdicts.json`) rather than being hand-maintained inside the markdown —
+otherwise the table drifts from the prose every time a feature is judged.
+
+    python scripts/spec_audit_status.py
+
+Keys are "<repo>/<task>|<feature>". Values are [verdict, note], where verdict is one of:
+
+    OK    spec is sufficient, nothing changed
+    SPEC  feature.md amended, because a hidden assertion was not derivable
+    TEST  tests.patch amended, because the test itself was the defect
+    RUN?  defect identified, but the fix needs a container run to verify gold still passes
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+DATASET = Path(__file__).resolve().parents[1] / "dataset"
+VERDICTS = DATASET / ".spec_audit_verdicts.json"
+AUDIT = DATASET / "SPEC_AUDIT.md"
+START, END = "## Status", "## Separately: gradeability"
+
+
+def main() -> None:
+    verdicts = json.loads(VERDICTS.read_text())
+    features = sorted(
+        (f"{p.parent.parent.name}/{p.parent.name}", p.name)
+        for p in DATASET.glob("*/task*/feature*")
+        if p.is_dir() and (p / "feature.md").is_file()
+    )
+
+    rows, prev = [], None
+    for task, feat in features:
+        verdict, note = verdicts.get(f"{task}|{feat}", ["-", ""])
+        rows.append(f"| {task if task != prev else ''} | {feat.replace('feature', 'f')} "
+                    f"| `{verdict:4s}` | {note} |")
+        prev = task
+
+    done = sum(1 for t, f in features if f"{t}|{f}" in verdicts)
+    counts: dict[str, int] = {}
+    for v, _ in verdicts.values():
+        counts[v] = counts.get(v, 0) + 1
+    tally = " · ".join(f"`{k}` {v}" for k, v in sorted(counts.items()))
+
+    table = "\n".join(["| task | feature | verdict | note |", "|---|---|---|---|", *rows])
+    header = (
+        f"## Status — {done}/{len(features)} features audited\n\n"
+        f"{tally}\n\n"
+        "`OK` sufficient, no change · `SPEC` feature.md amended · `TEST` tests.patch amended · "
+        "`RUN?` defect identified, fix needs a container run · `-` not yet audited\n\n"
+    )
+
+    text = AUDIT.read_text()
+    AUDIT.write_text(text[: text.index(START)] + header + table + "\n\n" + text[text.index(END):])
+    print(f"{done}/{len(features)} audited — {tally}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/spec_audit_survey.py b/scripts/spec_audit_survey.py
new file mode 100644
index 00000000..58dd306a
--- /dev/null
+++ b/scripts/spec_audit_survey.py
@@ -0,0 +1,96 @@
+"""Surface, for one task, everything the spec audit needs to judge each feature.
+
+The audit question is whether a strong model could derive every hidden assertion from
+`feature.md` plus the repo. Three things repeatedly decide it, so this prints them side by side:
+
+    symbols   public names the reference introduces AND the test uses AND the spec never mentions
+    messages  error text the test matches on, with whether the reference is where it comes from
+              (a message absent from feature.patch is usually pre-existing or raised by the
+              test's own stub, and is then not a requirement at all)
+    raises    exception types the test expects that the spec does not name
+
+    python scripts/spec_audit_survey.py pallets_jinja_task/task1465
+
+Everything here is a *candidate*, never a verdict — the judgement is manual, and roughly half of
+what this flags turns out to be derivable on inspection.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+DATASET = Path(__file__).resolve().parents[1] / "dataset"
+
+SYMBOL_PATTERNS = (
+    r"^\+\s*(?:async )?def\s+(\w+)",
+    r"^\+\s*class\s+(\w+)",
+    r"^\+func\s+(?:\([^)]*\)\s*)?([A-Z]\w*)",
+    r"^\+type\s+([A-Z]\w*)",
+    r"^\+\s*([A-Z][A-Za-z0-9]*)\s*=\s",
+    r"^\+\s*(?:pub\s+)?fn\s+([a-z_]\w*)",
+    r"^\+\s*(?:export\s+)?(?:const|function)\s+(\w+)",
+)
+MESSAGE_PATTERNS = (
+    r'match=["\']([^"\']{8,})["\']',
+    r"// Error: [\d\-]+ (.{8,})",
+    r'toThrow\(["\']([^"\']{8,})["\']\)',
+)
+RAISE_PATTERNS = (
+    r"(?:raises|assertRaises)\(\s*([A-Z]\w*)",
+    r"toThrow\(\s*([A-Z]\w*)\b",
+)
+
+
+def added(patch: Path) -> str:
+    return "\n".join(ln for ln in patch.read_text(errors="replace").splitlines() if ln.startswith("+"))
+
+
+def main() -> None:
+    task = DATASET / sys.argv[1]
+    if not task.is_dir():
+        raise SystemExit(f"no such task: {task}")
+
+    for fd in sorted(task.glob("feature*"), key=lambda p: int(re.sub(r"\D", "", p.name) or 0)):
+        md, tp, fp = fd / "feature.md", fd / "tests.patch", fd / "feature.patch"
+        if not (md.is_file() and tp.is_file() and fp.is_file()):
+            continue
+        spec, test, gold = md.read_text(errors="replace"), added(tp), fp.read_text(errors="replace")
+        title = next((ln for ln in spec.splitlines() if ln.strip()), "")[:70]
+
+        symbols = set()
+        for pat in SYMBOL_PATTERNS:
+            symbols |= set(re.findall(pat, gold, re.M))
+        symbols = {s for s in symbols if len(s) > 3 and not s.startswith("_")}
+        missing_sym = sorted(
+            s for s in symbols
+            if re.search(rf"\b{re.escape(s)}\b", test) and not re.search(rf"\b{re.escape(s)}\b", spec)
+        )
+
+        msgs = set()
+        for pat in MESSAGE_PATTERNS:
+            msgs |= set(re.findall(pat, test))
+        msg_rows = [
+            (m, "GOLD" if m.split("{")[0].strip()[:25] in gold else "elsewhere")
+            for m in sorted(msgs) if m.strip(" .")[:35] not in spec
+        ]
+
+        raises = set()
+        for pat in RAISE_PATTERNS:
+            raises |= set(re.findall(pat, test))
+        missing_raise = sorted(r for r in raises if not re.search(rf"\b{r}\b", spec))
+
+        print(f"\n--- {fd.name}: {title}")
+        if missing_sym:
+            print(f"    symbols  : {', '.join(missing_sym[:8])}")
+        for m, origin in msg_rows[:4]:
+            print(f"    message  : [{origin}] {m[:80]}")
+        if missing_raise:
+            print(f"    raises   : {', '.join(missing_raise)}")
+        if not (missing_sym or msg_rows or missing_raise):
+            print("    (nothing flagged)")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/cooperbench/__about__.py b/src/cooperbench/__about__.py
index 327bb35b..ba3b4254 100644
--- a/src/cooperbench/__about__.py
+++ b/src/cooperbench/__about__.py
@@ -1,3 +1,3 @@
 """Version information for CooperBench."""
 
-__version__ = "0.0.28"
+__version__ = "0.0.29"
diff --git a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py
index 60b50d1b..09a1ef6a 100644
--- a/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py
+++ b/src/cooperbench/agents/mini_swe_agent_v2/agents/default.py
@@ -252,9 +252,9 @@ def _nudge_unsubmitted(self) -> bool:
                     f"You have not submitted anything. Your work is still local ({held}) and "
                     f"no pull request exists, so it would be graded as an empty submission.\n\n"
                     f"    git add <the files you want to submit>\n"
-                    f"    git commit -m \"...\"\n"
+                    f'    git commit -m "..."\n'
                     f"    git push {GIT_REMOTE} HEAD:{agent}\n"
-                    f"    gh pr create --title \"...\" --body \"...\"\n\n"
+                    f'    gh pr create --title "..." --body "..."\n\n'
                     f"Then echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT again."
                 ),
             )
@@ -421,17 +421,19 @@ def execute_actions(self, message: dict) -> list[dict]:
                     # reads that as "messaging is broken" rather than "my quoting was wrong".
                     # Measured: 7 lost sends across two runs (bare --wait with no body,
                     # unterminated heredoc, `< file` redirect, `-t 60`, `timeout N bash -c ...`).
-                    outputs.append({
-                        "output": (
-                            "send_message: could not parse that call, so nothing was sent.\n"
-                            "Use one of:\n"
-                            "  send_message [--wait] <agent> <<'MSG'\n  your text\n  MSG\n"
-                            "  send_message [--wait] <agent> \"your text\"\n"
-                            "The message body is required, and --wait takes no argument."
-                        ),
-                        "returncode": 1,
-                        "exception_info": "",
-                    })
+                    outputs.append(
+                        {
+                            "output": (
+                                "send_message: could not parse that call, so nothing was sent.\n"
+                                "Use one of:\n"
+                                "  send_message [--wait] <agent> <<'MSG'\n  your text\n  MSG\n"
+                                '  send_message [--wait] <agent> "your text"\n'
+                                "The message body is required, and --wait takes no argument."
+                            ),
+                            "returncode": 1,
+                            "exception_info": "",
+                        }
+                    )
                     continue
                 if sm_matches:
                     sm_outputs = []
diff --git a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git.py b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git.py
index e27ca8a2..aa66dc0d 100644
--- a/src/cooperbench/agents/mini_swe_agent_v2/connectors/git.py
+++ b/src/cooperbench/agents/mini_swe_agent_v2/connectors/git.py
@@ -222,15 +222,20 @@ def submitted_patch(self, env: DockerEnvironment) -> str:
             # away work that was committed and pushed successfully.
             #
             # Still zero for an agent that pushed nothing -- that is a genuine non-submission.
-            branch = self._exec(
-                env, f"git ls-remote --heads {self.REMOTE_NAME} refs/heads/{self.agent_id}"
-            ).get("output", "").strip()
+            branch = (
+                self._exec(env, f"git ls-remote --heads {self.REMOTE_NAME} refs/heads/{self.agent_id}")
+                .get("output", "")
+                .strip()
+            )
             if not branch:
                 self._logger.info(f"NO PR AND NO BRANCH pushed by {self.agent_id}: submitting nothing")
                 return ""
-            fallback = self._exec(
-                env, f"git --no-pager diff {self._base_sha} {self.REMOTE_NAME}/{self.agent_id}"
-            ).get("output", "") or ""
+            fallback = (
+                self._exec(env, f"git --no-pager diff {self._base_sha} {self.REMOTE_NAME}/{self.agent_id}").get(
+                    "output", ""
+                )
+                or ""
+            )
             if not fallback.strip():
                 self._logger.info(f"NO PR OPENED by {self.agent_id} and branch is at base: nothing to submit")
                 return ""
diff --git a/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py b/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py
index 10330e6f..78a848f0 100644
--- a/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py
+++ b/src/cooperbench/agents/mini_swe_agent_v2/environments/modal.py
@@ -84,16 +84,15 @@ def _invalidate_image(image_name: str) -> None:
 # build. A model that happened to run `find / -name cargo` recovered. Prepending the well-known
 # toolchain dirs costs nothing and removes the archaeology.
 _TOOLCHAIN_PATH = (
-    'export PATH="/usr/local/cargo/bin:/usr/local/go/bin:/root/.cargo/bin:'
-    '/root/go/bin:/usr/local/bin:$PATH";'
+    'export PATH="/usr/local/cargo/bin:/usr/local/go/bin:/root/.cargo/bin:/root/go/bin:/usr/local/bin:$PATH";'
 )
 
 
 class ModalEnvironmentConfig(BaseModel):
     image: str
     cwd: str = "/"
-    timeout: int = 3600           # sandbox lifetime
-    command_timeout: int = 300    # per command; longest real one observed is ~104s (npm test)
+    timeout: int = 3600  # sandbox lifetime
+    command_timeout: int = 300  # per command; longest real one observed is ~104s (npm test)
     env: dict[str, str] = {}
     max_retries: int = 5
     retry_delay: float = 5.0
@@ -238,8 +237,13 @@ def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) ->
                 # other stdin reader) hangs until the sandbox dies. `timeout` catches infinite
                 # loops, which closing stdin cannot.
                 proc = self.sb.exec(
-                    "timeout", "-k", "10", str(timeout or self.config.command_timeout),
-                    "bash", "-lc", f"{_TOOLCHAIN_PATH} exec < /dev/null; cd {cwd} && {command}",
+                    "timeout",
+                    "-k",
+                    "10",
+                    str(timeout or self.config.command_timeout),
+                    "bash",
+                    "-lc",
+                    f"{_TOOLCHAIN_PATH} exec < /dev/null; cd {cwd} && {command}",
                     text=False,
                 )
                 stdout = proc.stdout.read().decode("utf-8", errors="replace")
diff --git a/src/cooperbench/eval/sandbox.py b/src/cooperbench/eval/sandbox.py
index 1540280c..e2157136 100644
--- a/src/cooperbench/eval/sandbox.py
+++ b/src/cooperbench/eval/sandbox.py
@@ -590,10 +590,11 @@ def _run_tests(sb: Sandbox, tests_patch: str, feature_patch: str, base_sha: str)
 # Remove any stale git lock left by a previous operation
 rm -f .git/index.lock .git/refs/heads/.lock
 
-# Reset to base commit
+# Reset to base commit. No -x: it deletes the build output the images pre-compile, which
+# forced a full recompile per graded feature (335 crate compiles for typst).
 git checkout --force {base_sha} 2>&1
 git reset --hard {base_sha} 2>&1
-git clean -fdx 2>&1
+git clean -fd 2>&1
 
 echo "Reset to base: $(git rev-parse HEAD)"
 
@@ -675,11 +676,11 @@ def _parse_results(output: str) -> dict:
 _TEST_DIR_RE = re.compile(r"/(tests?|__tests__|spec|testdata)/")
 _TEST_FILE_RE = re.compile(
     r"(^|/)("
-    r"test_[^/]+"                       # test_foo.py
-    r"|[^/]+_test\.[A-Za-z0-9]+"        # foo_test.go / foo_test.rs / foo_test.py
+    r"test_[^/]+"  # test_foo.py
+    r"|[^/]+_test\.[A-Za-z0-9]+"  # foo_test.go / foo_test.rs / foo_test.py
     r"|[^/]+\.(test|spec)\.[A-Za-z0-9]+"  # foo.test.ts / foo.spec.js
     r"|[^/]*Test[s]?\.(java|kt|cs|scala)"  # FooTest.java
-    r"|tests?\.py"                      # tests.py
+    r"|tests?\.py"  # tests.py
     r")$"
 )