test(benchmarks): add A/B task fixtures with hidden checks

Ten small Python tasks, each failing on the fixture and passing with a
reference solution, including workflow expectations for a small change, a
bugfix, and a delegable change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/benchmarks/tasks/08-api-rename-with-docs/hidden/test_hidden.py

@@ -0,0 +1,15 @@
+import inspect
+import unittest
+from pathlib import Path
+from app import api, cli
+
+
+class Hidden(unittest.TestCase):
+    def test_signature(self):
+        self.assertEqual(list(inspect.signature(api.fetch).parameters), ['url', 'max_retries'])
+        self.assertEqual(cli.main('u'), 2)
+
+    def test_docs(self):
+        text = Path('README.md').read_text()
+        self.assertIn('max_retries', text)
+        self.assertNotRegex(text, r'(?<![\w])retries(?=[=`])')

scripts/benchmarks/tasks/08-api-rename-with-docs/repo/README.md

@@ -0,0 +1,3 @@
+# API
+
+`fetch(url, retries=3)` retries a request up to `retries` times.

scripts/benchmarks/tasks/08-api-rename-with-docs/repo/app/api.py

@@ -0,0 +1,3 @@
+def fetch(url, retries=3):
+    """Fetch url, retrying up to retries times. Returns the attempts used."""
+    return min(retries, 3) if url else 0

scripts/benchmarks/tasks/08-api-rename-with-docs/repo/app/cli.py

@@ -0,0 +1,5 @@
+from app.api import fetch
+
+
+def main(url):
+    return fetch(url, retries=2)

scripts/benchmarks/tasks/08-api-rename-with-docs/task.json

@@ -0,0 +1,6 @@
+{
+  "prompt": "Rename the retries parameter of app.api.fetch to max_retries.",
+  "workflow": {
+    "kind": "API change with directly affected documentation"
+  }
+}