test(benchmarks): add A/B task fixtures with hidden checks

Ten small Python tasks, each failing on the fixture and passing with a
reference solution, including workflow expectations for a small change, a
bugfix, and a delegable change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/benchmarks/tasks/05-write-tests/hidden/test_hidden.py

@@ -0,0 +1,31 @@
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+MUTANTS = [
+    ('.rstrip("-")', ''),
+    ('.lower()', ''),
+    ('"ascii", "ignore"', '"ascii", "replace"'),
+]
+
+
+def run_tests(root):
+    return subprocess.run([sys.executable, '-m', 'unittest', 'discover', '-s', 'tests', '-t', '.'], cwd=root, capture_output=True, text=True, timeout=120)
+
+
+class Hidden(unittest.TestCase):
+    def test_tests_pass(self):
+        result = run_tests('.')
+        self.assertEqual(result.returncode, 0, result.stderr[-2000:])
+        self.assertRegex(result.stderr, r'Ran ([3-9]|\d\d+) tests')
+
+    def test_tests_catch_mutants(self):
+        source = Path('app/slug.py').read_text()
+        for old, new in MUTANTS:
+            with self.subTest(old), tempfile.TemporaryDirectory() as directory:
+                shutil.copytree('.', directory, dirs_exist_ok=True, ignore=shutil.ignore_patterns('.git', 'bench_hidden'))
+                Path(directory, 'app/slug.py').write_text(source.replace(old, new, 1))
+                self.assertNotEqual(run_tests(directory).returncode, 0, f'tests miss mutant {old!r}')

scripts/benchmarks/tasks/05-write-tests/repo/app/slug.py

@@ -0,0 +1,9 @@
+import re
+import unicodedata
+
+
+def slugify(text, max_length=50):
+    """Lowercase ASCII slug with single hyphens, trimmed to max_length without a trailing hyphen."""
+    text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode()
+    text = re.sub(r"[^a-zA-Z0-9]+", "-", text).strip("-").lower()
+    return text[:max_length].rstrip("-")

scripts/benchmarks/tasks/05-write-tests/task.json

@@ -0,0 +1,6 @@
+{
+  "prompt": "Add unittest-based unit tests for app/slug.py under tests/ that cover its edge cases.",
+  "workflow": {
+    "kind": "test writing"
+  }
+}