diff --git a/scripts/benchmarks/ab.py b/scripts/benchmarks/ab.py index 8160d07..2587677 100644 --- a/scripts/benchmarks/ab.py +++ b/scripts/benchmarks/ab.py @@ -33,7 +33,8 @@ import time TASKS = Path(__file__).resolve().parent / 'tasks' CHECK = [sys.executable, '-m', 'unittest', 'discover', '-s', 'bench_hidden', '-t', '.'] CLAUDE_TOOLS = ['Read', 'Edit', 'Write', 'Glob', 'Grep', 'Agent', 'Skill', 'TodoWrite', 'Bash(python *)', 'Bash(git *)', - 'Bash(ls *)', 'PowerShell(python *)', 'PowerShell(git *)', 'PowerShell(Get-ChildItem *)'] + 'Bash(ls *)', 'Bash(dotnet *)', 'Bash(node *)', 'PowerShell(python *)', 'PowerShell(git *)', 'PowerShell(Get-ChildItem *)', + 'PowerShell(dotnet *)', 'PowerShell(node *)'] def clean_env(**values): @@ -308,7 +309,12 @@ def check(work, task): hidden = work / 'bench_hidden' shutil.rmtree(hidden, ignore_errors=True) shutil.copytree(task / 'hidden', hidden) - result = subprocess.run(CHECK, cwd=work, capture_output=True, text=True, timeout=300) + spec = json.loads((task / 'task.json').read_text(encoding='utf-8')) + command = CHECK + if spec.get('check'): + program, *arguments = spec['check'] + command = [sys.executable if program == 'python' else shutil.which(program) or program, *arguments] + result = subprocess.run(command, cwd=work, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=600) shutil.rmtree(hidden, ignore_errors=True) return result.returncode == 0, (result.stdout + result.stderr)[-3000:] diff --git a/scripts/benchmarks/tasks/14-aspnet-order-validation/hidden/Check/Check.csproj b/scripts/benchmarks/tasks/14-aspnet-order-validation/hidden/Check/Check.csproj new file mode 100644 index 0000000..97701d8 --- /dev/null +++ b/scripts/benchmarks/tasks/14-aspnet-order-validation/hidden/Check/Check.csproj @@ -0,0 +1,12 @@ + + + Exe + net10.0 + enable + enable + + + + + + diff --git a/scripts/benchmarks/tasks/14-aspnet-order-validation/hidden/Check/Program.cs b/scripts/benchmarks/tasks/14-aspnet-order-validation/hidden/Check/Program.cs new file mode 100644 index 0000000..d65d1a0 --- /dev/null +++ b/scripts/benchmarks/tasks/14-aspnet-order-validation/hidden/Check/Program.cs @@ -0,0 +1,24 @@ +using System.Net; +using System.Text; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; + +var app = Shop.AppHost.Build(new[] { "--urls", "http://127.0.0.1:0" }); +await app.StartAsync(); +var address = app.Services.GetRequiredService().Features.Get()!.Addresses.First(); +using var client = new HttpClient { BaseAddress = new Uri(address) }; +var failures = new List(); +async Task Expect(string json, int status) +{ + var response = await client.PostAsync("/orders", new StringContent(json, Encoding.UTF8, "application/json")); + if ((int)response.StatusCode != status) failures.Add($"{json} -> {(int)response.StatusCode}, expected {status}"); + else if (status == 400 && response.Content.Headers.ContentType?.MediaType != "application/problem+json") failures.Add($"{json} -> 400 without problem details"); +} +await Expect("{\"sku\":\"A1\",\"quantity\":2}", 201); +await Expect("{\"sku\":\"A1\",\"quantity\":0}", 400); +await Expect("{\"sku\":\"A1\",\"quantity\":-3}", 400); +await Expect("{\"sku\":\"\",\"quantity\":1}", 400); +await app.StopAsync(); +foreach (var failure in failures) Console.Error.WriteLine("FAIL " + failure); +return failures.Count == 0 ? 0 : 1; diff --git a/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/AppHost.cs b/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/AppHost.cs new file mode 100644 index 0000000..ac77e94 --- /dev/null +++ b/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/AppHost.cs @@ -0,0 +1,20 @@ +namespace Shop; + +public static class AppHost +{ + // Integration tests start the application through Build. + public static WebApplication Build(string[] args) + { + var builder = WebApplication.CreateBuilder(args); + var app = builder.Build(); + var orders = new List(); + app.MapPost("/orders", (Order order) => + { + orders.Add(order); + return Results.Created($"/orders/{orders.Count}", order); + }); + return app; + } +} + +public record Order(string Sku, int Quantity); diff --git a/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/Program.cs b/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/Program.cs new file mode 100644 index 0000000..8be4650 --- /dev/null +++ b/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/Program.cs @@ -0,0 +1 @@ +Shop.AppHost.Build(args).Run(); diff --git a/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/Shop.csproj b/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/Shop.csproj new file mode 100644 index 0000000..9a04a31 --- /dev/null +++ b/scripts/benchmarks/tasks/14-aspnet-order-validation/repo/src/Shop/Shop.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/scripts/benchmarks/tasks/14-aspnet-order-validation/task.json b/scripts/benchmarks/tasks/14-aspnet-order-validation/task.json new file mode 100644 index 0000000..3d8ff1d --- /dev/null +++ b/scripts/benchmarks/tasks/14-aspnet-order-validation/task.json @@ -0,0 +1,13 @@ +{ + "prompt": "POST /orders accepts orders with a zero or negative quantity and orders with an empty SKU. Reject those with HTTP 400 and a validation problem response; valid orders must still return 201.", + "check": [ + "dotnet", + "run", + "--project", + "bench_hidden/Check", + "--nologo" + ], + "workflow": { + "kind": "C#/ASP.NET change" + } +} diff --git a/scripts/benchmarks/tasks/15-csharp-cancellation/hidden/Check/Check.csproj b/scripts/benchmarks/tasks/15-csharp-cancellation/hidden/Check/Check.csproj new file mode 100644 index 0000000..d1e7afc --- /dev/null +++ b/scripts/benchmarks/tasks/15-csharp-cancellation/hidden/Check/Check.csproj @@ -0,0 +1,11 @@ + + + Exe + net10.0 + enable + enable + + + + + diff --git a/scripts/benchmarks/tasks/15-csharp-cancellation/hidden/Check/Program.cs b/scripts/benchmarks/tasks/15-csharp-cancellation/hidden/Check/Program.cs new file mode 100644 index 0000000..8b83786 --- /dev/null +++ b/scripts/benchmarks/tasks/15-csharp-cancellation/hidden/Check/Program.cs @@ -0,0 +1,20 @@ +using System.Diagnostics; +using Reports; + +var failures = new List(); +var builder = new ReportBuilder(); +var complete = await builder.BuildAsync(new[] { "a", "b" }); +if (complete != "## a\n## b") failures.Add($"unexpected output {complete}"); +using var source = new CancellationTokenSource(TimeSpan.FromMilliseconds(150)); +var watch = Stopwatch.StartNew(); +try +{ + await builder.BuildAsync(Enumerable.Range(0, 50).Select(number => number.ToString()), source.Token); + failures.Add("no cancellation"); +} +catch (OperationCanceledException) +{ + if (watch.ElapsedMilliseconds > 1000) failures.Add($"cancelled only after {watch.ElapsedMilliseconds} ms"); +} +foreach (var failure in failures) Console.Error.WriteLine("FAIL " + failure); +return failures.Count == 0 ? 0 : 1; diff --git a/scripts/benchmarks/tasks/15-csharp-cancellation/repo/src/Reports/ReportBuilder.cs b/scripts/benchmarks/tasks/15-csharp-cancellation/repo/src/Reports/ReportBuilder.cs new file mode 100644 index 0000000..9d27b4a --- /dev/null +++ b/scripts/benchmarks/tasks/15-csharp-cancellation/repo/src/Reports/ReportBuilder.cs @@ -0,0 +1,15 @@ +namespace Reports; + +public sealed class ReportBuilder +{ + public async Task BuildAsync(IEnumerable sections, CancellationToken cancellationToken = default) + { + var parts = new List(); + foreach (var section in sections) + { + await Task.Delay(100); + parts.Add($"## {section}"); + } + return string.Join("\n", parts); + } +} diff --git a/scripts/benchmarks/tasks/15-csharp-cancellation/repo/src/Reports/Reports.csproj b/scripts/benchmarks/tasks/15-csharp-cancellation/repo/src/Reports/Reports.csproj new file mode 100644 index 0000000..1a57862 --- /dev/null +++ b/scripts/benchmarks/tasks/15-csharp-cancellation/repo/src/Reports/Reports.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/scripts/benchmarks/tasks/15-csharp-cancellation/task.json b/scripts/benchmarks/tasks/15-csharp-cancellation/task.json new file mode 100644 index 0000000..53ee073 --- /dev/null +++ b/scripts/benchmarks/tasks/15-csharp-cancellation/task.json @@ -0,0 +1,13 @@ +{ + "prompt": "Cancelling a report build does nothing: BuildAsync keeps running until every section is rendered. Make it honor cancellation promptly by throwing OperationCanceledException.", + "check": [ + "dotnet", + "run", + "--project", + "bench_hidden/Check", + "--nologo" + ], + "workflow": { + "kind": "C#/ASP.NET change" + } +} diff --git a/scripts/benchmarks/tasks/16-aspnet-unknown-customer/hidden/Check/Check.csproj b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/hidden/Check/Check.csproj new file mode 100644 index 0000000..de75f8f --- /dev/null +++ b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/hidden/Check/Check.csproj @@ -0,0 +1,12 @@ + + + Exe + net10.0 + enable + enable + + + + + + diff --git a/scripts/benchmarks/tasks/16-aspnet-unknown-customer/hidden/Check/Program.cs b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/hidden/Check/Program.cs new file mode 100644 index 0000000..a18c423 --- /dev/null +++ b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/hidden/Check/Program.cs @@ -0,0 +1,18 @@ +using System.Net; +using System.Text; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.Extensions.DependencyInjection; + +var app = Crm.AppHost.Build(new[] { "--urls", "http://127.0.0.1:0" }); +await app.StartAsync(); +var address = app.Services.GetRequiredService().Features.Get()!.Addresses.First(); +using var client = new HttpClient { BaseAddress = new Uri(address) }; +var failures = new List(); +var known = await client.GetAsync("/customers/1"); +if (known.StatusCode != HttpStatusCode.OK || !(await known.Content.ReadAsStringAsync()).Contains("Ada")) failures.Add($"known customer -> {(int)known.StatusCode}"); +var unknown = await client.GetAsync("/customers/42"); +if (unknown.StatusCode != HttpStatusCode.NotFound) failures.Add($"unknown customer -> {(int)unknown.StatusCode}"); +await app.StopAsync(); +foreach (var failure in failures) Console.Error.WriteLine("FAIL " + failure); +return failures.Count == 0 ? 0 : 1; diff --git a/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/AppHost.cs b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/AppHost.cs new file mode 100644 index 0000000..f87a826 --- /dev/null +++ b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/AppHost.cs @@ -0,0 +1,16 @@ +namespace Crm; + +public static class AppHost +{ + // Integration tests start the application through Build. + public static WebApplication Build(string[] args) + { + var builder = WebApplication.CreateBuilder(args); + var app = builder.Build(); + var customers = new Dictionary { [1] = new(1, "Ada") }; + app.MapGet("/customers/{id:int}", (int id) => customers[id]); + return app; + } +} + +public record Customer(int Id, string Name); diff --git a/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/Crm.csproj b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/Crm.csproj new file mode 100644 index 0000000..9a04a31 --- /dev/null +++ b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/Crm.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/Program.cs b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/Program.cs new file mode 100644 index 0000000..ff8ef7a --- /dev/null +++ b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/repo/src/Crm/Program.cs @@ -0,0 +1 @@ +Crm.AppHost.Build(args).Run(); diff --git a/scripts/benchmarks/tasks/16-aspnet-unknown-customer/task.json b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/task.json new file mode 100644 index 0000000..b06d532 --- /dev/null +++ b/scripts/benchmarks/tasks/16-aspnet-unknown-customer/task.json @@ -0,0 +1,13 @@ +{ + "prompt": "Requesting an unknown customer with GET /customers/{id} returns HTTP 500. Return 404 instead, and keep known customers working.", + "check": [ + "dotnet", + "run", + "--project", + "bench_hidden/Check", + "--nologo" + ], + "workflow": { + "kind": "C#/ASP.NET change" + } +} diff --git a/scripts/benchmarks/tasks/17-ts-booking-overlap/hidden/check.test.ts b/scripts/benchmarks/tasks/17-ts-booking-overlap/hidden/check.test.ts new file mode 100644 index 0000000..dc88c44 --- /dev/null +++ b/scripts/benchmarks/tasks/17-ts-booking-overlap/hidden/check.test.ts @@ -0,0 +1,14 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { overlaps } from '../src/booking.ts'; + +test('back-to-back bookings do not overlap', () => { + assert.equal(overlaps({ start: 9, end: 10 }, { start: 10, end: 11 }), false); + assert.equal(overlaps({ start: 10, end: 11 }, { start: 9, end: 10 }), false); +}); + +test('real overlaps are detected', () => { + assert.equal(overlaps({ start: 9, end: 11 }, { start: 10, end: 12 }), true); + assert.equal(overlaps({ start: 9, end: 12 }, { start: 10, end: 11 }), true); + assert.equal(overlaps({ start: 9, end: 10 }, { start: 11, end: 12 }), false); +}); diff --git a/scripts/benchmarks/tasks/17-ts-booking-overlap/repo/package.json b/scripts/benchmarks/tasks/17-ts-booking-overlap/repo/package.json new file mode 100644 index 0000000..2ed09a9 --- /dev/null +++ b/scripts/benchmarks/tasks/17-ts-booking-overlap/repo/package.json @@ -0,0 +1,5 @@ +{ + "name": "booking", + "type": "module", + "private": true +} diff --git a/scripts/benchmarks/tasks/17-ts-booking-overlap/repo/src/booking.ts b/scripts/benchmarks/tasks/17-ts-booking-overlap/repo/src/booking.ts new file mode 100644 index 0000000..3e9bb37 --- /dev/null +++ b/scripts/benchmarks/tasks/17-ts-booking-overlap/repo/src/booking.ts @@ -0,0 +1,8 @@ +export interface Booking { + start: number; + end: number; +} + +export function overlaps(a: Booking, b: Booking): boolean { + return a.start <= b.end && b.start <= a.end; +} diff --git a/scripts/benchmarks/tasks/17-ts-booking-overlap/task.json b/scripts/benchmarks/tasks/17-ts-booking-overlap/task.json new file mode 100644 index 0000000..6671bed --- /dev/null +++ b/scripts/benchmarks/tasks/17-ts-booking-overlap/task.json @@ -0,0 +1,13 @@ +{ + "prompt": "Back-to-back bookings, where one ends exactly when the next starts, are wrongly reported as overlapping. Fix overlaps() without breaking real overlaps.", + "check": [ + "node", + "--experimental-strip-types", + "--no-warnings", + "--test", + "bench_hidden/check.test.ts" + ], + "workflow": { + "kind": "TypeScript change" + } +} diff --git a/scripts/benchmarks/tasks/18-ts-deep-merge/hidden/check.test.ts b/scripts/benchmarks/tasks/18-ts-deep-merge/hidden/check.test.ts new file mode 100644 index 0000000..457e214 --- /dev/null +++ b/scripts/benchmarks/tasks/18-ts-deep-merge/hidden/check.test.ts @@ -0,0 +1,17 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mergeConfig } from '../src/config.ts'; + +test('nested objects merge deeply', () => { + const base = { server: { host: 'a', port: 1, tls: { on: true } }, tags: ['x'] }; + const override = { server: { port: 2, tls: { cert: 'c' } }, tags: ['y', 'z'] }; + assert.deepEqual(mergeConfig(base, override), { server: { host: 'a', port: 2, tls: { on: true, cert: 'c' } }, tags: ['y', 'z'] }); +}); + +test('inputs are not mutated', () => { + const base = { a: { b: 1 } }; + const override = { a: { c: 2 } }; + mergeConfig(base, override); + assert.deepEqual(base, { a: { b: 1 } }); + assert.deepEqual(override, { a: { c: 2 } }); +}); diff --git a/scripts/benchmarks/tasks/18-ts-deep-merge/repo/package.json b/scripts/benchmarks/tasks/18-ts-deep-merge/repo/package.json new file mode 100644 index 0000000..f66847a --- /dev/null +++ b/scripts/benchmarks/tasks/18-ts-deep-merge/repo/package.json @@ -0,0 +1,5 @@ +{ + "name": "config", + "type": "module", + "private": true +} diff --git a/scripts/benchmarks/tasks/18-ts-deep-merge/repo/src/config.ts b/scripts/benchmarks/tasks/18-ts-deep-merge/repo/src/config.ts new file mode 100644 index 0000000..6a4fd8d --- /dev/null +++ b/scripts/benchmarks/tasks/18-ts-deep-merge/repo/src/config.ts @@ -0,0 +1,5 @@ +export type Config = { [key: string]: unknown }; + +export function mergeConfig(base: Config, override: Config): Config { + return { ...base, ...override }; +} diff --git a/scripts/benchmarks/tasks/18-ts-deep-merge/task.json b/scripts/benchmarks/tasks/18-ts-deep-merge/task.json new file mode 100644 index 0000000..0a50772 --- /dev/null +++ b/scripts/benchmarks/tasks/18-ts-deep-merge/task.json @@ -0,0 +1,13 @@ +{ + "prompt": "mergeConfig() drops nested settings from the base configuration when the override sets only some keys of a nested object. Merge nested objects deeply; arrays from the override replace base arrays, and neither input may be mutated.", + "check": [ + "node", + "--experimental-strip-types", + "--no-warnings", + "--test", + "bench_hidden/check.test.ts" + ], + "workflow": { + "kind": "TypeScript change" + } +} diff --git a/scripts/benchmarks/tasks/19-ts-debounce-arguments/hidden/check.test.ts b/scripts/benchmarks/tasks/19-ts-debounce-arguments/hidden/check.test.ts new file mode 100644 index 0000000..ec76474 --- /dev/null +++ b/scripts/benchmarks/tasks/19-ts-debounce-arguments/hidden/check.test.ts @@ -0,0 +1,18 @@ +import { test, mock } from 'node:test'; +import assert from 'node:assert/strict'; +import { debounce } from '../src/debounce.ts'; + +test('trailing call uses the latest arguments', () => { + mock.timers.enable({ apis: ['setTimeout'] }); + const calls: number[][] = []; + const run = debounce((value: number) => calls.push([value]), 50); + run(1); run(2); run(3); + mock.timers.tick(49); + assert.deepEqual(calls, []); + mock.timers.tick(1); + assert.deepEqual(calls, [[3]]); + run(4); + mock.timers.tick(50); + assert.deepEqual(calls, [[3], [4]]); + mock.timers.reset(); +}); diff --git a/scripts/benchmarks/tasks/19-ts-debounce-arguments/repo/package.json b/scripts/benchmarks/tasks/19-ts-debounce-arguments/repo/package.json new file mode 100644 index 0000000..1a6ae49 --- /dev/null +++ b/scripts/benchmarks/tasks/19-ts-debounce-arguments/repo/package.json @@ -0,0 +1,5 @@ +{ + "name": "timing", + "type": "module", + "private": true +} diff --git a/scripts/benchmarks/tasks/19-ts-debounce-arguments/repo/src/debounce.ts b/scripts/benchmarks/tasks/19-ts-debounce-arguments/repo/src/debounce.ts new file mode 100644 index 0000000..f4e9194 --- /dev/null +++ b/scripts/benchmarks/tasks/19-ts-debounce-arguments/repo/src/debounce.ts @@ -0,0 +1,15 @@ +export function debounce(fn: (...args: T) => void, wait: number): (...args: T) => void { + let timer: ReturnType | undefined; + let pending: T | undefined; + return (...args: T) => { + if (pending === undefined) { + pending = args; + } + clearTimeout(timer); + timer = setTimeout(() => { + const call = pending as T; + pending = undefined; + fn(...call); + }, wait); + }; +} diff --git a/scripts/benchmarks/tasks/19-ts-debounce-arguments/task.json b/scripts/benchmarks/tasks/19-ts-debounce-arguments/task.json new file mode 100644 index 0000000..c69ad93 --- /dev/null +++ b/scripts/benchmarks/tasks/19-ts-debounce-arguments/task.json @@ -0,0 +1,13 @@ +{ + "prompt": "debounce() calls the wrapped function with the arguments of the first call in a burst instead of the last one. Fix it so the trailing call uses the latest arguments.", + "check": [ + "node", + "--experimental-strip-types", + "--no-warnings", + "--test", + "bench_hidden/check.test.ts" + ], + "workflow": { + "kind": "TypeScript change" + } +}