test(benchmarks): add C#/ASP.NET and TypeScript tasks

Six tasks with hidden checks: ASP.NET validation and 404 handling, C#
cancellation, and TypeScript overlap, deep merge, and debounce fixes, checked
with dotnet run and node --test.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

scripts/benchmarks/ab.py

@@ -33,7 +33,8 @@ import time
 TASKS = Path(__file__).resolve().parent / 'tasks'
 CHECK = [sys.executable, '-m', 'unittest', 'discover', '-s', 'bench_hidden', '-t', '.']
 CLAUDE_TOOLS = ['Read', 'Edit', 'Write', 'Glob', 'Grep', 'Agent', 'Skill', 'TodoWrite', 'Bash(python *)', 'Bash(git *)',
-                'Bash(ls *)', 'PowerShell(python *)', 'PowerShell(git *)', 'PowerShell(Get-ChildItem *)']
+                'Bash(ls *)', 'Bash(dotnet *)', 'Bash(node *)', 'PowerShell(python *)', 'PowerShell(git *)', 'PowerShell(Get-ChildItem *)',
+                'PowerShell(dotnet *)', 'PowerShell(node *)']
 
 
 def clean_env(**values):
@@ -308,7 +309,12 @@ def check(work, task):
     hidden = work / 'bench_hidden'
     shutil.rmtree(hidden, ignore_errors=True)
     shutil.copytree(task / 'hidden', hidden)
-    result = subprocess.run(CHECK, cwd=work, capture_output=True, text=True, timeout=300)
+    spec = json.loads((task / 'task.json').read_text(encoding='utf-8'))
+    command = CHECK
+    if spec.get('check'):
+        program, *arguments = spec['check']
+        command = [sys.executable if program == 'python' else shutil.which(program) or program, *arguments]
+    result = subprocess.run(command, cwd=work, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=600)
     shutil.rmtree(hidden, ignore_errors=True)
     return result.returncode == 0, (result.stdout + result.stderr)[-3000:]
 

scripts/benchmarks/tasks/14-aspnet-order-validation/hidden/Check/Check.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+    <ProjectReference Include="..\..\src\Shop\Shop.csproj" />
+  </ItemGroup>
+</Project>

scripts/benchmarks/tasks/14-aspnet-order-validation/hidden/Check/Program.cs

@@ -0,0 +1,24 @@
+using System.Net;
+using System.Text;
+using Microsoft.AspNetCore.Hosting.Server;
+using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.Extensions.DependencyInjection;
+
+var app = Shop.AppHost.Build(new[] { "--urls", "http://127.0.0.1:0" });
+await app.StartAsync();
+var address = app.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First();
+using var client = new HttpClient { BaseAddress = new Uri(address) };
+var failures = new List<string>();
+async Task Expect(string json, int status)
+{
+    var response = await client.PostAsync("/orders", new StringContent(json, Encoding.UTF8, "application/json"));
+    if ((int)response.StatusCode != status) failures.Add($"{json} -> {(int)response.StatusCode}, expected {status}");
+    else if (status == 400 && response.Content.Headers.ContentType?.MediaType != "application/problem+json") failures.Add($"{json} -> 400 without problem details");
+}
+await Expect("{\"sku\":\"A1\",\"quantity\":2}", 201);
+await Expect("{\"sku\":\"A1\",\"quantity\":0}", 400);
+await Expect("{\"sku\":\"A1\",\"quantity\":-3}", 400);
+await Expect("{\"sku\":\"\",\"quantity\":1}", 400);
+await app.StopAsync();
+foreach (var failure in failures) Console.Error.WriteLine("FAIL " + failure);
+return failures.Count == 0 ? 0 : 1;

scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/AppHost.cs

@@ -0,0 +1,20 @@
+namespace Shop;
+
+public static class AppHost
+{
+    // Integration tests start the application through Build.
+    public static WebApplication Build(string[] args)
+    {
+        var builder = WebApplication.CreateBuilder(args);
+        var app = builder.Build();
+        var orders = new List<Order>();
+        app.MapPost("/orders", (Order order) =>
+        {
+            orders.Add(order);
+            return Results.Created($"/orders/{orders.Count}", order);
+        });
+        return app;
+    }
+}
+
+public record Order(string Sku, int Quantity);

scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/Program.cs

@@ -0,0 +1 @@
+Shop.AppHost.Build(args).Run();

scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/Shop.csproj

@@ -0,0 +1,7 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+</Project>

scripts/benchmarks/tasks/14-aspnet-order-validation/task.json

@@ -0,0 +1,13 @@
+{
+  "prompt": "POST /orders accepts orders with a zero or negative quantity and orders with an empty SKU. Reject those with HTTP 400 and a validation problem response; valid orders must still return 201.",
+  "check": [
+    "dotnet",
+    "run",
+    "--project",
+    "bench_hidden/Check",
+    "--nologo"
+  ],
+  "workflow": {
+    "kind": "C#/ASP.NET change"
+  }
+}

scripts/benchmarks/tasks/15-csharp-cancellation/hidden/Check/Check.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Reports\Reports.csproj" />
+  </ItemGroup>
+</Project>

scripts/benchmarks/tasks/15-csharp-cancellation/hidden/Check/Program.cs

@@ -0,0 +1,20 @@
+using System.Diagnostics;
+using Reports;
+
+var failures = new List<string>();
+var builder = new ReportBuilder();
+var complete = await builder.BuildAsync(new[] { "a", "b" });
+if (complete != "## a\n## b") failures.Add($"unexpected output {complete}");
+using var source = new CancellationTokenSource(TimeSpan.FromMilliseconds(150));
+var watch = Stopwatch.StartNew();
+try
+{
+    await builder.BuildAsync(Enumerable.Range(0, 50).Select(number => number.ToString()), source.Token);
+    failures.Add("no cancellation");
+}
+catch (OperationCanceledException)
+{
+    if (watch.ElapsedMilliseconds > 1000) failures.Add($"cancelled only after {watch.ElapsedMilliseconds} ms");
+}
+foreach (var failure in failures) Console.Error.WriteLine("FAIL " + failure);
+return failures.Count == 0 ? 0 : 1;

scripts/benchmarks/tasks/15-csharp-cancellation/repo/src/Reports/ReportBuilder.cs

@@ -0,0 +1,15 @@
+namespace Reports;
+
+public sealed class ReportBuilder
+{
+    public async Task<string> BuildAsync(IEnumerable<string> sections, CancellationToken cancellationToken = default)
+    {
+        var parts = new List<string>();
+        foreach (var section in sections)
+        {
+            await Task.Delay(100);
+            parts.Add($"## {section}");
+        }
+        return string.Join("\n", parts);
+    }
+}

scripts/benchmarks/tasks/15-csharp-cancellation/repo/src/Reports/Reports.csproj

@@ -0,0 +1,7 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+</Project>

scripts/benchmarks/tasks/15-csharp-cancellation/task.json

@@ -0,0 +1,13 @@
+{
+  "prompt": "Cancelling a report build does nothing: BuildAsync keeps running until every section is rendered. Make it honor cancellation promptly by throwing OperationCanceledException.",
+  "check": [
+    "dotnet",
+    "run",
+    "--project",
+    "bench_hidden/Check",
+    "--nologo"
+  ],
+  "workflow": {
+    "kind": "C#/ASP.NET change"
+  }
+}

scripts/benchmarks/tasks/16-aspnet-unknown-customer/hidden/Check/Check.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+    <ProjectReference Include="..\..\src\Crm\Crm.csproj" />
+  </ItemGroup>
+</Project>

scripts/benchmarks/tasks/16-aspnet-unknown-customer/hidden/Check/Program.cs

@@ -0,0 +1,18 @@
+using System.Net;
+using System.Text;
+using Microsoft.AspNetCore.Hosting.Server;
+using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.Extensions.DependencyInjection;
+
+var app = Crm.AppHost.Build(new[] { "--urls", "http://127.0.0.1:0" });
+await app.StartAsync();
+var address = app.Services.GetRequiredService<IServer>().Features.Get<IServerAddressesFeature>()!.Addresses.First();
+using var client = new HttpClient { BaseAddress = new Uri(address) };
+var failures = new List<string>();
+var known = await client.GetAsync("/customers/1");
+if (known.StatusCode != HttpStatusCode.OK || !(await known.Content.ReadAsStringAsync()).Contains("Ada")) failures.Add($"known customer -> {(int)known.StatusCode}");
+var unknown = await client.GetAsync("/customers/42");
+if (unknown.StatusCode != HttpStatusCode.NotFound) failures.Add($"unknown customer -> {(int)unknown.StatusCode}");
+await app.StopAsync();
+foreach (var failure in failures) Console.Error.WriteLine("FAIL " + failure);
+return failures.Count == 0 ? 0 : 1;

scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/AppHost.cs

@@ -0,0 +1,16 @@
+namespace Crm;
+
+public static class AppHost
+{
+    // Integration tests start the application through Build.
+    public static WebApplication Build(string[] args)
+    {
+        var builder = WebApplication.CreateBuilder(args);
+        var app = builder.Build();
+        var customers = new Dictionary<int, Customer> { [1] = new(1, "Ada") };
+        app.MapGet("/customers/{id:int}", (int id) => customers[id]);
+        return app;
+    }
+}
+
+public record Customer(int Id, string Name);

scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/Crm.csproj

@@ -0,0 +1,7 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+</Project>

scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/Program.cs

@@ -0,0 +1 @@
+Crm.AppHost.Build(args).Run();

scripts/benchmarks/tasks/16-aspnet-unknown-customer/task.json

@@ -0,0 +1,13 @@
+{
+  "prompt": "Requesting an unknown customer with GET /customers/{id} returns HTTP 500. Return 404 instead, and keep known customers working.",
+  "check": [
+    "dotnet",
+    "run",
+    "--project",
+    "bench_hidden/Check",
+    "--nologo"
+  ],
+  "workflow": {
+    "kind": "C#/ASP.NET change"
+  }
+}

scripts/benchmarks/tasks/17-ts-booking-overlap/hidden/check.test.ts

@@ -0,0 +1,14 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { overlaps } from '../src/booking.ts';
+
+test('back-to-back bookings do not overlap', () => {
+  assert.equal(overlaps({ start: 9, end: 10 }, { start: 10, end: 11 }), false);
+  assert.equal(overlaps({ start: 10, end: 11 }, { start: 9, end: 10 }), false);
+});
+
+test('real overlaps are detected', () => {
+  assert.equal(overlaps({ start: 9, end: 11 }, { start: 10, end: 12 }), true);
+  assert.equal(overlaps({ start: 9, end: 12 }, { start: 10, end: 11 }), true);
+  assert.equal(overlaps({ start: 9, end: 10 }, { start: 11, end: 12 }), false);
+});

scripts/benchmarks/tasks/17-ts-booking-overlap/repo/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "booking",
+  "type": "module",
+  "private": true
+}

scripts/benchmarks/tasks/17-ts-booking-overlap/repo/src/booking.ts

@@ -0,0 +1,8 @@
+export interface Booking {
+  start: number;
+  end: number;
+}
+
+export function overlaps(a: Booking, b: Booking): boolean {
+  return a.start <= b.end && b.start <= a.end;
+}

scripts/benchmarks/tasks/17-ts-booking-overlap/task.json

@@ -0,0 +1,13 @@
+{
+  "prompt": "Back-to-back bookings, where one ends exactly when the next starts, are wrongly reported as overlapping. Fix overlaps() without breaking real overlaps.",
+  "check": [
+    "node",
+    "--experimental-strip-types",
+    "--no-warnings",
+    "--test",
+    "bench_hidden/check.test.ts"
+  ],
+  "workflow": {
+    "kind": "TypeScript change"
+  }
+}

scripts/benchmarks/tasks/18-ts-deep-merge/hidden/check.test.ts

@@ -0,0 +1,17 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mergeConfig } from '../src/config.ts';
+
+test('nested objects merge deeply', () => {
+  const base = { server: { host: 'a', port: 1, tls: { on: true } }, tags: ['x'] };
+  const override = { server: { port: 2, tls: { cert: 'c' } }, tags: ['y', 'z'] };
+  assert.deepEqual(mergeConfig(base, override), { server: { host: 'a', port: 2, tls: { on: true, cert: 'c' } }, tags: ['y', 'z'] });
+});
+
+test('inputs are not mutated', () => {
+  const base = { a: { b: 1 } };
+  const override = { a: { c: 2 } };
+  mergeConfig(base, override);
+  assert.deepEqual(base, { a: { b: 1 } });
+  assert.deepEqual(override, { a: { c: 2 } });
+});

scripts/benchmarks/tasks/18-ts-deep-merge/repo/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "config",
+  "type": "module",
+  "private": true
+}

scripts/benchmarks/tasks/18-ts-deep-merge/repo/src/config.ts

@@ -0,0 +1,5 @@
+export type Config = { [key: string]: unknown };
+
+export function mergeConfig(base: Config, override: Config): Config {
+  return { ...base, ...override };
+}

scripts/benchmarks/tasks/18-ts-deep-merge/task.json

@@ -0,0 +1,13 @@
+{
+  "prompt": "mergeConfig() drops nested settings from the base configuration when the override sets only some keys of a nested object. Merge nested objects deeply; arrays from the override replace base arrays, and neither input may be mutated.",
+  "check": [
+    "node",
+    "--experimental-strip-types",
+    "--no-warnings",
+    "--test",
+    "bench_hidden/check.test.ts"
+  ],
+  "workflow": {
+    "kind": "TypeScript change"
+  }
+}

scripts/benchmarks/tasks/19-ts-debounce-arguments/hidden/check.test.ts

@@ -0,0 +1,18 @@
+import { test, mock } from 'node:test';
+import assert from 'node:assert/strict';
+import { debounce } from '../src/debounce.ts';
+
+test('trailing call uses the latest arguments', () => {
+  mock.timers.enable({ apis: ['setTimeout'] });
+  const calls: number[][] = [];
+  const run = debounce((value: number) => calls.push([value]), 50);
+  run(1); run(2); run(3);
+  mock.timers.tick(49);
+  assert.deepEqual(calls, []);
+  mock.timers.tick(1);
+  assert.deepEqual(calls, [[3]]);
+  run(4);
+  mock.timers.tick(50);
+  assert.deepEqual(calls, [[3], [4]]);
+  mock.timers.reset();
+});

scripts/benchmarks/tasks/19-ts-debounce-arguments/repo/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "timing",
+  "type": "module",
+  "private": true
+}

scripts/benchmarks/tasks/19-ts-debounce-arguments/repo/src/debounce.ts

@@ -0,0 +1,15 @@
+export function debounce<T extends unknown[]>(fn: (...args: T) => void, wait: number): (...args: T) => void {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let pending: T | undefined;
+  return (...args: T) => {
+    if (pending === undefined) {
+      pending = args;
+    }
+    clearTimeout(timer);
+    timer = setTimeout(() => {
+      const call = pending as T;
+      pending = undefined;
+      fn(...call);
+    }, wait);
+  };
+}

scripts/benchmarks/tasks/19-ts-debounce-arguments/task.json

@@ -0,0 +1,13 @@
+{
+  "prompt": "debounce() calls the wrapped function with the arguments of the first call in a burst instead of the last one. Fix it so the trailing call uses the latest arguments.",
+  "check": [
+    "node",
+    "--experimental-strip-types",
+    "--no-warnings",
+    "--test",
+    "bench_hidden/check.test.ts"
+  ],
+  "workflow": {
+    "kind": "TypeScript change"
+  }
+}