diff --git a/scripts/benchmarks/tasks/11-search-expired-evaluation/hidden/__init__.py b/scripts/benchmarks/tasks/11-search-expired-evaluation/hidden/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/11-search-expired-evaluation/hidden/test_hidden.py b/scripts/benchmarks/tasks/11-search-expired-evaluation/hidden/test_hidden.py new file mode 100644 index 0000000..b9cd8d3 --- /dev/null +++ b/scripts/benchmarks/tasks/11-search-expired-evaluation/hidden/test_hidden.py @@ -0,0 +1,14 @@ +import unittest +from datetime import date +from app.accounts import Account +from app.ledger.archive import render_period_summary + + +class Hidden(unittest.TestCase): + def test_expired_evaluation_blocked(self): + with self.assertRaises(PermissionError): + render_period_summary(Account('evaluation', date(2026, 1, 1)), '2026-01', today=date(2026, 2, 1)) + + def test_running_evaluation_and_paid_allowed(self): + self.assertIsInstance(render_period_summary(Account('evaluation', date(2026, 3, 1)), '2026-01', today=date(2026, 2, 1)), bytes) + self.assertIsInstance(render_period_summary(Account('paid'), '2026-01', today=date(2026, 2, 1)), bytes) diff --git a/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/__init__.py b/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/accounts.py b/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/accounts.py new file mode 100644 index 0000000..ae62cb7 --- /dev/null +++ b/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/accounts.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass, field +from datetime import date + + +@dataclass +class Account: + tier: str # "evaluation" or "paid" + evaluation_until: date | None = None + entries: list = field(default_factory=list) diff --git a/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/ledger/__init__.py b/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/ledger/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/ledger/archive.py b/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/ledger/archive.py new file mode 100644 index 0000000..b343836 --- /dev/null +++ b/scripts/benchmarks/tasks/11-search-expired-evaluation/repo/app/ledger/archive.py @@ -0,0 +1,8 @@ +from datetime import date + + +def render_period_summary(account, period, today=None): + """Build the period summary document for an account.""" + today = today or date.today() + lines = [f"{entry.label}: {entry.amount:.2f}" for entry in account.entries if entry.period == period] + return "\n".join(lines).encode() diff --git a/scripts/benchmarks/tasks/11-search-expired-evaluation/task.json b/scripts/benchmarks/tasks/11-search-expired-evaluation/task.json new file mode 100644 index 0000000..b6b7eba --- /dev/null +++ b/scripts/benchmarks/tasks/11-search-expired-evaluation/task.json @@ -0,0 +1,9 @@ +{ + "prompt": "Customers whose free trial has ended can still download their monthly statements. Block that by raising PermissionError, and keep it working for paying customers and for customers whose trial is still running.", + "generate": { + "filler_modules": 250 + }, + "workflow": { + "kind": "concept search in a large repository" + } +} diff --git a/scripts/benchmarks/tasks/12-search-transient-retry/hidden/__init__.py b/scripts/benchmarks/tasks/12-search-transient-retry/hidden/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/12-search-transient-retry/hidden/test_hidden.py b/scripts/benchmarks/tasks/12-search-transient-retry/hidden/test_hidden.py new file mode 100644 index 0000000..501d3a5 --- /dev/null +++ b/scripts/benchmarks/tasks/12-search-transient-retry/hidden/test_hidden.py @@ -0,0 +1,32 @@ +import unittest +from app.transport.courier import dispatch + + +class Channel: + def __init__(self, failures, error=ConnectionResetError): + self.failures, self.error, self.calls = failures, error, 0 + + def send(self, blob): + self.calls += 1 + if self.calls <= self.failures: + raise self.error('flaky') + return 'receipt' + + +class Hidden(unittest.TestCase): + def test_retries_transient(self): + channel = Channel(2) + self.assertEqual(dispatch(b'x', channel), 'receipt') + self.assertEqual(channel.calls, 3) + + def test_gives_up_after_three(self): + channel = Channel(5) + with self.assertRaises(ConnectionResetError): + dispatch(b'x', channel) + self.assertEqual(channel.calls, 3) + + def test_no_retry_for_other_errors(self): + channel = Channel(1, ValueError) + with self.assertRaises(ValueError): + dispatch(b'x', channel) + self.assertEqual(channel.calls, 1) diff --git a/scripts/benchmarks/tasks/12-search-transient-retry/repo/app/__init__.py b/scripts/benchmarks/tasks/12-search-transient-retry/repo/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/12-search-transient-retry/repo/app/transport/__init__.py b/scripts/benchmarks/tasks/12-search-transient-retry/repo/app/transport/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/12-search-transient-retry/repo/app/transport/courier.py b/scripts/benchmarks/tasks/12-search-transient-retry/repo/app/transport/courier.py new file mode 100644 index 0000000..2e5426d --- /dev/null +++ b/scripts/benchmarks/tasks/12-search-transient-retry/repo/app/transport/courier.py @@ -0,0 +1,3 @@ +def dispatch(blob, channel): + """Hand a blob to the channel and return the receipt.""" + return channel.send(blob) diff --git a/scripts/benchmarks/tasks/12-search-transient-retry/task.json b/scripts/benchmarks/tasks/12-search-transient-retry/task.json new file mode 100644 index 0000000..ff167ca --- /dev/null +++ b/scripts/benchmarks/tasks/12-search-transient-retry/task.json @@ -0,0 +1,9 @@ +{ + "prompt": "Uploads that fail with a temporary network error are never retried. Make at most three attempts in total before giving up with the last error; other errors must not be retried.", + "generate": { + "filler_modules": 250 + }, + "workflow": { + "kind": "concept search in a large repository" + } +} diff --git a/scripts/benchmarks/tasks/13-search-cent-rounding/hidden/__init__.py b/scripts/benchmarks/tasks/13-search-cent-rounding/hidden/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/13-search-cent-rounding/hidden/test_hidden.py b/scripts/benchmarks/tasks/13-search-cent-rounding/hidden/test_hidden.py new file mode 100644 index 0000000..b690ea9 --- /dev/null +++ b/scripts/benchmarks/tasks/13-search-cent-rounding/hidden/test_hidden.py @@ -0,0 +1,10 @@ +import unittest +from app.fin.tally import consolidate + + +class Hidden(unittest.TestCase): + def test_half_up(self): + self.assertEqual(float(consolidate([1.005])), 1.01) + self.assertEqual(float(consolidate([2.675])), 2.68) + self.assertEqual(float(consolidate([1.0, 0.004])), 1.0) + self.assertEqual(float(consolidate([0.1, 0.2])), 0.3) diff --git a/scripts/benchmarks/tasks/13-search-cent-rounding/repo/app/__init__.py b/scripts/benchmarks/tasks/13-search-cent-rounding/repo/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/13-search-cent-rounding/repo/app/fin/__init__.py b/scripts/benchmarks/tasks/13-search-cent-rounding/repo/app/fin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/benchmarks/tasks/13-search-cent-rounding/repo/app/fin/tally.py b/scripts/benchmarks/tasks/13-search-cent-rounding/repo/app/fin/tally.py new file mode 100644 index 0000000..8198d9b --- /dev/null +++ b/scripts/benchmarks/tasks/13-search-cent-rounding/repo/app/fin/tally.py @@ -0,0 +1,3 @@ +def consolidate(lines): + """Sum line amounts and return the figure for the document footer.""" + return round(sum(lines), 2) diff --git a/scripts/benchmarks/tasks/13-search-cent-rounding/task.json b/scripts/benchmarks/tasks/13-search-cent-rounding/task.json new file mode 100644 index 0000000..f5d78c1 --- /dev/null +++ b/scripts/benchmarks/tasks/13-search-cent-rounding/task.json @@ -0,0 +1,9 @@ +{ + "prompt": "Invoice totals are sometimes one cent too low, for example 1.005 shows as 1.00. Round half amounts up to the next cent.", + "generate": { + "filler_modules": 250 + }, + "workflow": { + "kind": "concept search in a large repository" + } +}