test(benchmarks): add A/B task fixtures with hidden checks

Ten small Python tasks, each failing on the fixture and passing with a
reference solution, including workflow expectations for a small change, a
bugfix, and a delegable change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/benchmarks/tasks/07-preserve-unknown-settings/hidden/test_hidden.py

@@ -0,0 +1,20 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from app.settings_store import save_settings
+
+
+class Hidden(unittest.TestCase):
+    def test_unknown_keys_survive(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'settings.json'
+            path.write_text(json.dumps({'theme': 'dark', 'plugins': {'a': True}, 'x': 1}))
+            save_settings(path, {'language': 'de'})
+            self.assertEqual(json.loads(path.read_text()), {'theme': 'dark', 'plugins': {'a': True}, 'x': 1, 'language': 'de'})
+
+    def test_new_file(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'settings.json'
+            save_settings(path, {'theme': 'light'})
+            self.assertEqual(json.loads(path.read_text()), {'theme': 'light'})

scripts/benchmarks/tasks/07-preserve-unknown-settings/repo/app/settings_store.py

@@ -0,0 +1,15 @@
+import json
+
+KNOWN = ("theme", "language")
+
+
+def save_settings(path, updates):
+    """Apply updates to the JSON settings file at path."""
+    try:
+        with open(path, encoding="utf-8") as stream:
+            current = json.load(stream)
+    except FileNotFoundError:
+        current = {}
+    merged = {key: updates.get(key, current.get(key)) for key in KNOWN if key in updates or key in current}
+    with open(path, "w", encoding="utf-8") as stream:
+        json.dump(merged, stream, indent=2)

scripts/benchmarks/tasks/07-preserve-unknown-settings/task.json

@@ -0,0 +1,6 @@
+{
+  "prompt": "save_settings() loses keys it does not know about when it updates the settings file. Fix it so unknown keys survive.",
+  "workflow": {
+    "kind": "data-preserving bugfix"
+  }
+}