test(benchmarks): tag tasks as small change, larger implementation, or search

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/benchmarks/tasks/01-small-config-change/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "Change the default request timeout from 30 to 60 seconds.",
   "workflow": {
     "kind": "small change without planning ceremony",

scripts/benchmarks/tasks/02-bugfix-pagination/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "Users report that the last item is missing on the final page returned by paginate(). Fix the bug.",
   "workflow": {
     "kind": "bugfix with targeted verification",

scripts/benchmarks/tasks/03-delegation-parsers/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "larger implementation",
   "prompt": "Implement the three independent parsers in app/parsers (csv_parser, json_parser, ini_parser) according to their docstrings, and add unit tests for each. They share no code.",
   "workflow": {
     "kind": "larger change with justified delegation",

scripts/benchmarks/tasks/04-security-sql/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "find_user() fails for user names that contain an apostrophe, such as O'Brien. Fix it.",
   "workflow": {
     "kind": "security-relevant bugfix"

scripts/benchmarks/tasks/05-write-tests/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "larger implementation",
   "prompt": "Add unittest-based unit tests for app/slug.py under tests/ that cover its edge cases.",
   "workflow": {
     "kind": "test writing"

scripts/benchmarks/tasks/06-refactor-duplication/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "larger implementation",
   "prompt": "Remove the duplicated formatting logic in app/report.py without changing its output.",
   "workflow": {
     "kind": "behavior-preserving refactor"

scripts/benchmarks/tasks/07-preserve-unknown-settings/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "save_settings() loses keys it does not know about when it updates the settings file. Fix it so unknown keys survive.",
   "workflow": {
     "kind": "data-preserving bugfix"

scripts/benchmarks/tasks/08-api-rename-with-docs/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "Rename the retries parameter of app.api.fetch to max_retries.",
   "workflow": {
     "kind": "API change with directly affected documentation"

scripts/benchmarks/tasks/09-path-traversal/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "Make read_upload() safe against path traversal. Legitimate names in nested folders must keep working.",
   "workflow": {
     "kind": "security hardening"

scripts/benchmarks/tasks/10-performance-dedupe/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "dedupe() is far too slow for 100,000 items. Make it fast while keeping its behavior and the order of first occurrences.",
   "workflow": {
     "kind": "performance fix"

scripts/benchmarks/tasks/11-search-expired-evaluation/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "search",
   "prompt": "Customers whose free trial has ended can still download their monthly statements. Block that by raising PermissionError, and keep it working for paying customers and for customers whose trial is still running.",
   "generate": {
     "filler_modules": 250

scripts/benchmarks/tasks/12-search-transient-retry/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "search",
   "prompt": "Uploads that fail with a temporary network error are never retried. Make at most three attempts in total before giving up with the last error; other errors must not be retried.",
   "generate": {
     "filler_modules": 250

scripts/benchmarks/tasks/13-search-cent-rounding/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "search",
   "prompt": "Invoice totals are sometimes one cent too low, for example 1.005 shows as 1.00. Round half amounts up to the next cent.",
   "generate": {
     "filler_modules": 250

scripts/benchmarks/tasks/14-aspnet-order-validation/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "larger implementation",
   "prompt": "POST /orders accepts orders with a zero or negative quantity and orders with an empty SKU. Reject those with HTTP 400 and a validation problem response; valid orders must still return 201.",
   "check": [
     "dotnet",

scripts/benchmarks/tasks/15-csharp-cancellation/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "larger implementation",
   "prompt": "Cancelling a report build does nothing: BuildAsync keeps running until every section is rendered. Make it honor cancellation promptly by throwing OperationCanceledException.",
   "check": [
     "dotnet",

scripts/benchmarks/tasks/16-aspnet-unknown-customer/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "Requesting an unknown customer with GET /customers/{id} returns HTTP 500. Return 404 instead, and keep known customers working.",
   "check": [
     "dotnet",

scripts/benchmarks/tasks/17-ts-booking-overlap/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "Back-to-back bookings, where one ends exactly when the next starts, are wrongly reported as overlapping. Fix overlaps() without breaking real overlaps.",
   "check": [
     "node",

scripts/benchmarks/tasks/18-ts-deep-merge/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "larger implementation",
   "prompt": "mergeConfig() drops nested settings from the base configuration when the override sets only some keys of a nested object. Merge nested objects deeply; arrays from the override replace base arrays, and neither input may be mutated.",
   "check": [
     "node",

scripts/benchmarks/tasks/19-ts-debounce-arguments/task.json

@@ -1,4 +1,5 @@
 {
+  "category": "small change",
   "prompt": "debounce() calls the wrapped function with the arguments of the first call in a burst instead of the last one. Fix it so the trailing call uses the latest arguments.",
   "check": [
     "node",