test(benchmarks): add concept-search tasks

Three tasks whose target code shares no vocabulary with the prompt, run in a
repository with generated distractor modules.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/benchmarks/tasks/11-search-expired-evaluation/hidden/test_hidden.py

@@ -0,0 +1,14 @@
+import unittest
+from datetime import date
+from app.accounts import Account
+from app.ledger.archive import render_period_summary
+
+
+class Hidden(unittest.TestCase):
+    def test_expired_evaluation_blocked(self):
+        with self.assertRaises(PermissionError):
+            render_period_summary(Account('evaluation', date(2026, 1, 1)), '2026-01', today=date(2026, 2, 1))
+
+    def test_running_evaluation_and_paid_allowed(self):
+        self.assertIsInstance(render_period_summary(Account('evaluation', date(2026, 3, 1)), '2026-01', today=date(2026, 2, 1)), bytes)
+        self.assertIsInstance(render_period_summary(Account('paid'), '2026-01', today=date(2026, 2, 1)), bytes)

scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/accounts.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass, field
+from datetime import date
+
+
+@dataclass
+class Account:
+    tier: str  # "evaluation" or "paid"
+    evaluation_until: date | None = None
+    entries: list = field(default_factory=list)

scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/ledger/archive.py

@@ -0,0 +1,8 @@
+from datetime import date
+
+
+def render_period_summary(account, period, today=None):
+    """Build the period summary document for an account."""
+    today = today or date.today()
+    lines = [f"{entry.label}: {entry.amount:.2f}" for entry in account.entries if entry.period == period]
+    return "\n".join(lines).encode()

scripts/benchmarks/tasks/11-search-expired-evaluation/task.json

@@ -0,0 +1,9 @@
+{
+  "prompt": "Customers whose free trial has ended can still download their monthly statements. Block that by raising PermissionError, and keep it working for paying customers and for customers whose trial is still running.",
+  "generate": {
+    "filler_modules": 250
+  },
+  "workflow": {
+    "kind": "concept search in a large repository"
+  }
+}