test(benchmarks): add A/B task fixtures with hidden checks

Ten small Python tasks, each failing on the fixture and passing with a
reference solution, including workflow expectations for a small change, a
bugfix, and a delegable change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Julian lechnerandClaude Opus 5.5 committed 2026-10-01 12:45:40 +02:00
1 parent b8fd66ce3b
commit 3480072ed7
60 files changed
+374

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,16 @@
import sqlite3
import unittest
from app.users import find_user
class Hidden(unittest.TestCase):
def setUp(self):
self.conn = sqlite3.connect(':memory:')
self.conn.execute('CREATE TABLE users (id INTEGER, name TEXT)')
self.conn.executemany('INSERT INTO users VALUES (?, ?)', [(1, "O'Brien"), (2, 'admin')])
def test_apostrophe(self):
self.assertEqual(find_user(self.conn, "O'Brien"), (1, "O'Brien"))
def test_injection(self):
self.assertIsNone(find_user(self.conn, "x' OR '1'='1"))
Whitespace-only changes.
@@ -0,0 +1,3 @@
def find_user(conn, name):
"""Return (id, name) for the user with the given name, or None."""
return conn.execute(f"SELECT id, name FROM users WHERE name = '{name}'").fetchone()
@@ -0,0 +1,6 @@
{
"prompt": "find_user() fails for user names that contain an apostrophe, such as O'Brien. Fix it.",
"workflow": {
"kind": "security-relevant bugfix"
}
}