test(benchmarks): add concept-search tasks

Three tasks whose target code shares no vocabulary with the prompt, run in a
repository with generated distractor modules.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/benchmarks/tasks/12-search-transient-retry/hidden/test_hidden.py

@@ -0,0 +1,32 @@
+import unittest
+from app.transport.courier import dispatch
+
+
+class Channel:
+    def __init__(self, failures, error=ConnectionResetError):
+        self.failures, self.error, self.calls = failures, error, 0
+
+    def send(self, blob):
+        self.calls += 1
+        if self.calls <= self.failures:
+            raise self.error('flaky')
+        return 'receipt'
+
+
+class Hidden(unittest.TestCase):
+    def test_retries_transient(self):
+        channel = Channel(2)
+        self.assertEqual(dispatch(b'x', channel), 'receipt')
+        self.assertEqual(channel.calls, 3)
+
+    def test_gives_up_after_three(self):
+        channel = Channel(5)
+        with self.assertRaises(ConnectionResetError):
+            dispatch(b'x', channel)
+        self.assertEqual(channel.calls, 3)
+
+    def test_no_retry_for_other_errors(self):
+        channel = Channel(1, ValueError)
+        with self.assertRaises(ValueError):
+            dispatch(b'x', channel)
+        self.assertEqual(channel.calls, 1)

scripts/benchmarks/tasks/12-search-transient-retry/repo/app/transport/courier.py

@@ -0,0 +1,3 @@
+def dispatch(blob, channel):
+    """Hand a blob to the channel and return the receipt."""
+    return channel.send(blob)

scripts/benchmarks/tasks/12-search-transient-retry/task.json

@@ -0,0 +1,9 @@
+{
+  "prompt": "Uploads that fail with a temporary network error are never retried. Make at most three attempts in total before giving up with the last error; other errors must not be retried.",
+  "generate": {
+    "filler_modules": 250
+  },
+  "workflow": {
+    "kind": "concept search in a large repository"
+  }
+}