test(benchmarks): add A/B task fixtures with hidden checks

Ten small Python tasks, each failing on the fixture and passing with a reference solution, including workflow expectations for a small change, a bugfix, and a delegable change. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-01 12:45:40 +02:00
1 parent b8fd66ce3b
commit 3480072ed7
@@ -0,0 +1,16 @@
+import unittest
+from pathlib import Path
+from app import client, config
+
+
+class Hidden(unittest.TestCase):
+    def test_default(self):
+        self.assertEqual(config.DEFAULT_TIMEOUT, 60)
+        self.assertEqual(client.request('x'), 60)
+        self.assertEqual(client.request('x', 5), 5)
+
+    def test_docs(self):
+        for path in ('README.md', 'app/client.py'):
+            text = Path(path).read_text()
+            self.assertIn('60 seconds', text)
+            self.assertNotIn('30 seconds', text)
@@ -0,0 +1,3 @@
+# Client
+
+Requests time out after 30 seconds unless a timeout is passed.
@@ -0,0 +1,6 @@
+from app.config import DEFAULT_TIMEOUT
+
+
+def request(url, timeout=None):
+    """Return the effective timeout in seconds; defaults to DEFAULT_TIMEOUT (30 seconds)."""
+    return timeout if timeout is not None else DEFAULT_TIMEOUT
@@ -0,0 +1 @@
+DEFAULT_TIMEOUT = 30
@@ -0,0 +1,14 @@
+{
+  "prompt": "Change the default request timeout from 30 to 60 seconds.",
+  "workflow": {
+    "kind": "small change without planning ceremony",
+    "forbid_skills": [
+      "superpowers:brainstorming",
+      "superpowers:writing-plans",
+      "superpowers:subagent-driven-development",
+      "superpowers:executing-plans"
+    ],
+    "max_agents": 0,
+    "max_tool_calls": 15
+  }
+}
@@ -0,0 +1,11 @@
+import unittest
+from app.pagination import paginate
+
+
+class Hidden(unittest.TestCase):
+    def test_last_page(self):
+        self.assertEqual(paginate(list(range(10)), 4, 3), [9])
+        self.assertEqual(paginate(list(range(9)), 3, 3), [6, 7, 8])
+
+    def test_out_of_range(self):
+        self.assertEqual(paginate(list(range(3)), 5, 3), [])
@@ -0,0 +1,5 @@
+def paginate(items, page, size):
+    """Return the 1-based page of items with the given page size."""
+    start = (page - 1) * size
+    end = min(start + size, len(items) - 1)
+    return items[start:end]
@@ -0,0 +1,7 @@
+import unittest
+from app.pagination import paginate
+
+
+class PaginationTests(unittest.TestCase):
+    def test_first_page(self):
+        self.assertEqual(paginate(list(range(10)), 1, 3), [0, 1, 2])
@@ -0,0 +1,14 @@
+{
+  "prompt": "Users report that the last item is missing on the final page returned by paginate(). Fix the bug.",
+  "workflow": {
+    "kind": "bugfix with targeted verification",
+    "forbid_skills": [
+      "superpowers:brainstorming",
+      "superpowers:writing-plans",
+      "superpowers:subagent-driven-development",
+      "superpowers:executing-plans"
+    ],
+    "max_agents": 0,
+    "require_command": "test_pagination|pagination"
+  }
+}
@@ -0,0 +1,17 @@
+import unittest
+from app.parsers.csv_parser import parse_csv
+from app.parsers.ini_parser import parse_ini
+from app.parsers.json_parser import parse_json_lines
+
+
+class Hidden(unittest.TestCase):
+    def test_csv(self):
+        self.assertEqual(parse_csv('a, b\n1, "x,y"\n\n2,3\n'), [{'a': '1', 'b': 'x,y'}, {'a': '2', 'b': '3'}])
+
+    def test_json(self):
+        self.assertEqual(parse_json_lines('{"a": 1}\n\n[2]\n'), [{'a': 1}, [2]])
+        with self.assertRaisesRegex(ValueError, '2'):
+            parse_json_lines('{}\n{bad\n')
+
+    def test_ini(self):
+        self.assertEqual(parse_ini('top = 1\n; c\n[s]\n# c\n k = v \n'), {'': {'top': '1'}, 's': {'k': 'v'}})
@@ -0,0 +1,6 @@
+def parse_csv(text):
+    """Parse comma-separated text with a header row into a list of dicts.
+
+    Strip whitespace around cells, skip blank lines, and support double-quoted cells containing commas.
+    """
+    raise NotImplementedError
@@ -0,0 +1,6 @@
+def parse_ini(text):
+    """Parse INI text into {section: {key: value}}.
+
+    Ignore lines starting with ; or #, strip whitespace, keep keys before the first section under "".
+    """
+    raise NotImplementedError
@@ -0,0 +1,6 @@
+def parse_json_lines(text):
+    """Parse JSON Lines text into a list of objects.
+
+    Skip blank lines. Raise ValueError naming the 1-based line number for invalid JSON.
+    """
+    raise NotImplementedError
@@ -0,0 +1,8 @@
+{
+  "prompt": "Implement the three independent parsers in app/parsers (csv_parser, json_parser, ini_parser) according to their docstrings, and add unit tests for each. They share no code.",
+  "workflow": {
+    "kind": "larger change with justified delegation",
+    "min_agents": 1,
+    "max_agents": 3
+  }
+}
@@ -0,0 +1,16 @@
+import sqlite3
+import unittest
+from app.users import find_user
+
+
+class Hidden(unittest.TestCase):
+    def setUp(self):
+        self.conn = sqlite3.connect(':memory:')
+        self.conn.execute('CREATE TABLE users (id INTEGER, name TEXT)')
+        self.conn.executemany('INSERT INTO users VALUES (?, ?)', [(1, "O'Brien"), (2, 'admin')])
+
+    def test_apostrophe(self):
+        self.assertEqual(find_user(self.conn, "O'Brien"), (1, "O'Brien"))
+
+    def test_injection(self):
+        self.assertIsNone(find_user(self.conn, "x' OR '1'='1"))
@@ -0,0 +1,3 @@
+def find_user(conn, name):
+    """Return (id, name) for the user with the given name, or None."""
+    return conn.execute(f"SELECT id, name FROM users WHERE name = '{name}'").fetchone()
@@ -0,0 +1,6 @@
+{
+  "prompt": "find_user() fails for user names that contain an apostrophe, such as O'Brien. Fix it.",
+  "workflow": {
+    "kind": "security-relevant bugfix"
+  }
+}
@@ -0,0 +1,31 @@
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+MUTANTS = [
+    ('.rstrip("-")', ''),
+    ('.lower()', ''),
+    ('"ascii", "ignore"', '"ascii", "replace"'),
+]
+
+
+def run_tests(root):
+    return subprocess.run([sys.executable, '-m', 'unittest', 'discover', '-s', 'tests', '-t', '.'], cwd=root, capture_output=True, text=True, timeout=120)
+
+
+class Hidden(unittest.TestCase):
+    def test_tests_pass(self):
+        result = run_tests('.')
+        self.assertEqual(result.returncode, 0, result.stderr[-2000:])
+        self.assertRegex(result.stderr, r'Ran ([3-9]|\d\d+) tests')
+
+    def test_tests_catch_mutants(self):
+        source = Path('app/slug.py').read_text()
+        for old, new in MUTANTS:
+            with self.subTest(old), tempfile.TemporaryDirectory() as directory:
+                shutil.copytree('.', directory, dirs_exist_ok=True, ignore=shutil.ignore_patterns('.git', 'bench_hidden'))
+                Path(directory, 'app/slug.py').write_text(source.replace(old, new, 1))
+                self.assertNotEqual(run_tests(directory).returncode, 0, f'tests miss mutant {old!r}')
@@ -0,0 +1,9 @@
+import re
+import unicodedata
+
+
+def slugify(text, max_length=50):
+    """Lowercase ASCII slug with single hyphens, trimmed to max_length without a trailing hyphen."""
+    text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode()
+    text = re.sub(r"[^a-zA-Z0-9]+", "-", text).strip("-").lower()
+    return text[:max_length].rstrip("-")
@@ -0,0 +1,6 @@
+{
+  "prompt": "Add unittest-based unit tests for app/slug.py under tests/ that cover its edge cases.",
+  "workflow": {
+    "kind": "test writing"
+  }
+}
@@ -0,0 +1,16 @@
+import unittest
+from pathlib import Path
+from app.report import cost_report, sales_report
+
+ROWS = [(' widgets ', 1234.5), ('gadgets', 99)]
+EXPECTED = 'Widgets                 1,234.50\nGadgets                    99.00\nTotal                   1,333.50'
+
+
+class Hidden(unittest.TestCase):
+    def test_output(self):
+        self.assertEqual(sales_report(ROWS), EXPECTED)
+        self.assertEqual(cost_report(ROWS), EXPECTED)
+        self.assertEqual(sales_report([]), f"{'Total':<20}{0:>12,.2f}")
+
+    def test_deduplicated(self):
+        self.assertEqual(Path('app/report.py').read_text().count('.title()'), 1)
@@ -0,0 +1,20 @@
+def sales_report(rows):
+    lines = []
+    for name, amount in rows:
+        label = name.strip().title()
+        value = f"{amount:,.2f}"
+        lines.append(f"{label:<20}{value:>12}")
+    total = sum(amount for _, amount in rows)
+    lines.append(f"{'Total':<20}{total:>12,.2f}")
+    return "\n".join(lines)
+
+
+def cost_report(rows):
+    lines = []
+    for name, amount in rows:
+        label = name.strip().title()
+        value = f"{amount:,.2f}"
+        lines.append(f"{label:<20}{value:>12}")
+    total = sum(amount for _, amount in rows)
+    lines.append(f"{'Total':<20}{total:>12,.2f}")
+    return "\n".join(lines)
@@ -0,0 +1,6 @@
+{
+  "prompt": "Remove the duplicated formatting logic in app/report.py without changing its output.",
+  "workflow": {
+    "kind": "behavior-preserving refactor"
+  }
+}
@@ -0,0 +1,20 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from app.settings_store import save_settings
+
+
+class Hidden(unittest.TestCase):
+    def test_unknown_keys_survive(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'settings.json'
+            path.write_text(json.dumps({'theme': 'dark', 'plugins': {'a': True}, 'x': 1}))
+            save_settings(path, {'language': 'de'})
+            self.assertEqual(json.loads(path.read_text()), {'theme': 'dark', 'plugins': {'a': True}, 'x': 1, 'language': 'de'})
+
+    def test_new_file(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'settings.json'
+            save_settings(path, {'theme': 'light'})
+            self.assertEqual(json.loads(path.read_text()), {'theme': 'light'})
@@ -0,0 +1,15 @@
+import json
+
+KNOWN = ("theme", "language")
+
+
+def save_settings(path, updates):
+    """Apply updates to the JSON settings file at path."""
+    try:
+        with open(path, encoding="utf-8") as stream:
+            current = json.load(stream)
+    except FileNotFoundError:
+        current = {}
+    merged = {key: updates.get(key, current.get(key)) for key in KNOWN if key in updates or key in current}
+    with open(path, "w", encoding="utf-8") as stream:
+        json.dump(merged, stream, indent=2)
@@ -0,0 +1,6 @@
+{
+  "prompt": "save_settings() loses keys it does not know about when it updates the settings file. Fix it so unknown keys survive.",
+  "workflow": {
+    "kind": "data-preserving bugfix"
+  }
+}
@@ -0,0 +1,15 @@
+import inspect
+import unittest
+from pathlib import Path
+from app import api, cli
+
+
+class Hidden(unittest.TestCase):
+    def test_signature(self):
+        self.assertEqual(list(inspect.signature(api.fetch).parameters), ['url', 'max_retries'])
+        self.assertEqual(cli.main('u'), 2)
+
+    def test_docs(self):
+        text = Path('README.md').read_text()
+        self.assertIn('max_retries', text)
+        self.assertNotRegex(text, r'(?<![\w])retries(?=[=`])')
@@ -0,0 +1,3 @@
+# API
+
+`fetch(url, retries=3)` retries a request up to `retries` times.
@@ -0,0 +1,3 @@
+def fetch(url, retries=3):
+    """Fetch url, retrying up to retries times. Returns the attempts used."""
+    return min(retries, 3) if url else 0
@@ -0,0 +1,5 @@
+from app.api import fetch
+
+
+def main(url):
+    return fetch(url, retries=2)
@@ -0,0 +1,6 @@
+{
+  "prompt": "Rename the retries parameter of app.api.fetch to max_retries.",
+  "workflow": {
+    "kind": "API change with directly affected documentation"
+  }
+}
@@ -0,0 +1,26 @@
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from app.files import read_upload
+
+
+class Hidden(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        root = Path(self.directory.name)
+        (root / 'uploads/sub').mkdir(parents=True)
+        (root / 'uploads/sub/a.txt').write_bytes(b'ok')
+        (root / 'secret.txt').write_bytes(b'secret')
+        self.base = str(root / 'uploads')
+
+    def tearDown(self):
+        self.directory.cleanup()
+
+    def test_nested(self):
+        self.assertEqual(read_upload(self.base, 'sub/a.txt'), b'ok')
+
+    def test_traversal(self):
+        for name in ('../secret.txt', 'sub/../../secret.txt', os.path.join(os.path.dirname(self.base), 'secret.txt')):
+            with self.subTest(name), self.assertRaises((ValueError, PermissionError, FileNotFoundError)):
+                read_upload(self.base, name)
@@ -0,0 +1,7 @@
+import os
+
+
+def read_upload(base, name):
+    """Return the bytes of the uploaded file name below the base directory."""
+    with open(os.path.join(base, name), "rb") as stream:
+        return stream.read()
@@ -0,0 +1,6 @@
+{
+  "prompt": "Make read_upload() safe against path traversal. Legitimate names in nested folders must keep working.",
+  "workflow": {
+    "kind": "security hardening"
+  }
+}
@@ -0,0 +1,16 @@
+import time
+import unittest
+from app.dedupe import dedupe
+
+
+class Hidden(unittest.TestCase):
+    def test_behavior(self):
+        self.assertEqual(dedupe([3, 1, 3, 2, 1]), [3, 1, 2])
+        self.assertEqual(dedupe([]), [])
+        self.assertEqual(dedupe(['a', 'b', 'a']), ['a', 'b'])
+
+    def test_fast(self):
+        items = list(range(100000)) * 2
+        start = time.perf_counter()
+        self.assertEqual(len(dedupe(items)), 100000)
+        self.assertLess(time.perf_counter() - start, 1.0)
@@ -0,0 +1,7 @@
+def dedupe(items):
+    """Return items without duplicates, keeping the first occurrence of each."""
+    result = []
+    for item in items:
+        if item not in result:
+            result.append(item)
+    return result
@@ -0,0 +1,6 @@
+{
+  "prompt": "dedupe() is far too slow for 100,000 items. Make it fast while keeping its behavior and the order of first occurrences.",
+  "workflow": {
+    "kind": "performance fix"
+  }
+}