test(benchmarks): add concept-search tasks

Three tasks whose target code shares no vocabulary with the prompt, run in a
repository with generated distractor modules.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/benchmarks/tasks/11-search-expired-evaluation/hidden/test_hidden.py

@@ -0,0 +1,14 @@
+import unittest
+from datetime import date
+from app.accounts import Account
+from app.ledger.archive import render_period_summary
+
+
+class Hidden(unittest.TestCase):
+    def test_expired_evaluation_blocked(self):
+        with self.assertRaises(PermissionError):
+            render_period_summary(Account('evaluation', date(2026, 1, 1)), '2026-01', today=date(2026, 2, 1))
+
+    def test_running_evaluation_and_paid_allowed(self):
+        self.assertIsInstance(render_period_summary(Account('evaluation', date(2026, 3, 1)), '2026-01', today=date(2026, 2, 1)), bytes)
+        self.assertIsInstance(render_period_summary(Account('paid'), '2026-01', today=date(2026, 2, 1)), bytes)

scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/accounts.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass, field
+from datetime import date
+
+
+@dataclass
+class Account:
+    tier: str  # "evaluation" or "paid"
+    evaluation_until: date | None = None
+    entries: list = field(default_factory=list)

scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/ledger/archive.py

@@ -0,0 +1,8 @@
+from datetime import date
+
+
+def render_period_summary(account, period, today=None):
+    """Build the period summary document for an account."""
+    today = today or date.today()
+    lines = [f"{entry.label}: {entry.amount:.2f}" for entry in account.entries if entry.period == period]
+    return "\n".join(lines).encode()

scripts/benchmarks/tasks/11-search-expired-evaluation/task.json

@@ -0,0 +1,9 @@
+{
+  "prompt": "Customers whose free trial has ended can still download their monthly statements. Block that by raising PermissionError, and keep it working for paying customers and for customers whose trial is still running.",
+  "generate": {
+    "filler_modules": 250
+  },
+  "workflow": {
+    "kind": "concept search in a large repository"
+  }
+}

scripts/benchmarks/tasks/12-search-transient-retry/hidden/test_hidden.py

@@ -0,0 +1,32 @@
+import unittest
+from app.transport.courier import dispatch
+
+
+class Channel:
+    def __init__(self, failures, error=ConnectionResetError):
+        self.failures, self.error, self.calls = failures, error, 0
+
+    def send(self, blob):
+        self.calls += 1
+        if self.calls <= self.failures:
+            raise self.error('flaky')
+        return 'receipt'
+
+
+class Hidden(unittest.TestCase):
+    def test_retries_transient(self):
+        channel = Channel(2)
+        self.assertEqual(dispatch(b'x', channel), 'receipt')
+        self.assertEqual(channel.calls, 3)
+
+    def test_gives_up_after_three(self):
+        channel = Channel(5)
+        with self.assertRaises(ConnectionResetError):
+            dispatch(b'x', channel)
+        self.assertEqual(channel.calls, 3)
+
+    def test_no_retry_for_other_errors(self):
+        channel = Channel(1, ValueError)
+        with self.assertRaises(ValueError):
+            dispatch(b'x', channel)
+        self.assertEqual(channel.calls, 1)

scripts/benchmarks/tasks/12-search-transient-retry/repo/app/transport/courier.py

@@ -0,0 +1,3 @@
+def dispatch(blob, channel):
+    """Hand a blob to the channel and return the receipt."""
+    return channel.send(blob)

scripts/benchmarks/tasks/12-search-transient-retry/task.json

@@ -0,0 +1,9 @@
+{
+  "prompt": "Uploads that fail with a temporary network error are never retried. Make at most three attempts in total before giving up with the last error; other errors must not be retried.",
+  "generate": {
+    "filler_modules": 250
+  },
+  "workflow": {
+    "kind": "concept search in a large repository"
+  }
+}

scripts/benchmarks/tasks/13-search-cent-rounding/hidden/test_hidden.py

@@ -0,0 +1,10 @@
+import unittest
+from app.fin.tally import consolidate
+
+
+class Hidden(unittest.TestCase):
+    def test_half_up(self):
+        self.assertEqual(float(consolidate([1.005])), 1.01)
+        self.assertEqual(float(consolidate([2.675])), 2.68)
+        self.assertEqual(float(consolidate([1.0, 0.004])), 1.0)
+        self.assertEqual(float(consolidate([0.1, 0.2])), 0.3)

scripts/benchmarks/tasks/13-search-cent-rounding/repo/app/fin/tally.py

@@ -0,0 +1,3 @@
+def consolidate(lines):
+    """Sum line amounts and return the figure for the document footer."""
+    return round(sum(lines), 2)

scripts/benchmarks/tasks/13-search-cent-rounding/task.json

@@ -0,0 +1,9 @@
+{
+  "prompt": "Invoice totals are sometimes one cent too low, for example 1.005 shows as 1.00. Round half amounts up to the next cent.",
+  "generate": {
+    "filler_modules": 250
+  },
+  "workflow": {
+    "kind": "concept search in a large repository"
+  }
+}