test(benchmarks): add A/B task fixtures with hidden checks

Ten small Python tasks, each failing on the fixture and passing with a
reference solution, including workflow expectations for a small change, a
bugfix, and a delegable change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Julian lechnerandClaude Opus 5.5 committed 2026-10-01 12:45:40 +02:00
1 parent b8fd66ce3b
commit 3480072ed7
60 files changed
+374

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,11 @@
import unittest
from app.pagination import paginate
class Hidden(unittest.TestCase):
def test_last_page(self):
self.assertEqual(paginate(list(range(10)), 4, 3), [9])
self.assertEqual(paginate(list(range(9)), 3, 3), [6, 7, 8])
def test_out_of_range(self):
self.assertEqual(paginate(list(range(3)), 5, 3), [])
Whitespace-only changes.
@@ -0,0 +1,5 @@
def paginate(items, page, size):
"""Return the 1-based page of items with the given page size."""
start = (page - 1) * size
end = min(start + size, len(items) - 1)
return items[start:end]
Whitespace-only changes.
@@ -0,0 +1,7 @@
import unittest
from app.pagination import paginate
class PaginationTests(unittest.TestCase):
def test_first_page(self):
self.assertEqual(paginate(list(range(10)), 1, 3), [0, 1, 2])
@@ -0,0 +1,14 @@
{
"prompt": "Users report that the last item is missing on the final page returned by paginate(). Fix the bug.",
"workflow": {
"kind": "bugfix with targeted verification",
"forbid_skills": [
"superpowers:brainstorming",
"superpowers:writing-plans",
"superpowers:subagent-driven-development",
"superpowers:executing-plans"
],
"max_agents": 0,
"require_command": "test_pagination|pagination"
}
}