test(benchmarks): add concept-search tasks

Three tasks whose target code shares no vocabulary with the prompt, run in a
repository with generated distractor modules.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Julian lechnerandClaude Opus 5.5 committed 2026-10-01 13:27:56 +02:00
1 parent a8f6f01f2f
commit 00a44f50b0
19 files changed
+106

No files matched your search

@@ -0,0 +1,32 @@
import unittest
from app.transport.courier import dispatch
class Channel:
def __init__(self, failures, error=ConnectionResetError):
self.failures, self.error, self.calls = failures, error, 0
def send(self, blob):
self.calls += 1
if self.calls <= self.failures:
raise self.error('flaky')
return 'receipt'
class Hidden(unittest.TestCase):
def test_retries_transient(self):
channel = Channel(2)
self.assertEqual(dispatch(b'x', channel), 'receipt')
self.assertEqual(channel.calls, 3)
def test_gives_up_after_three(self):
channel = Channel(5)
with self.assertRaises(ConnectionResetError):
dispatch(b'x', channel)
self.assertEqual(channel.calls, 3)
def test_no_retry_for_other_errors(self):
channel = Channel(1, ValueError)
with self.assertRaises(ValueError):
dispatch(b'x', channel)
self.assertEqual(channel.calls, 1)