feat(prompt-budget): track context KPIs

adapters/prompt-budget-baseline.json

@@ -1 +1,23 @@
-{"Claude permanent context":4000,"Codex permanent context":4000,"Claude permanent context absolute limit":6000,"Codex permanent context absolute limit":6000}
+{
+  "regression_percent": 10,
+  "permanent_context_soft_target_tokens": 4000,
+  "permanent_context_hard_limit_tokens": 6000,
+  "metrics": {
+    "permanent_context_tokens": {
+      "claude": 1481,
+      "codex": 1481
+    },
+    "skill_metadata_tokens": {
+      "claude": 1124,
+      "codex": 1124
+    },
+    "rule_catalog_tokens": {
+      "claude": 28036,
+      "codex": 8058
+    },
+    "agent_metadata_tokens": {
+      "claude": 126,
+      "codex": 52
+    }
+  }
+}

adapters/prompt-budget-baseline.tsv

@@ -1,2 +1,9 @@
-Claude	4000	6000
-Codex	4000	6000
+metric	client	baseline_tokens
+permanent_context_tokens	claude	1481
+permanent_context_tokens	codex	1481
+skill_metadata_tokens	claude	1124
+skill_metadata_tokens	codex	1124
+rule_catalog_tokens	claude	28036
+rule_catalog_tokens	codex	8058
+agent_metadata_tokens	claude	126
+agent_metadata_tokens	codex	52

scripts/commands/prompt-budget.ps1

@@ -2,34 +2,11 @@
 param([switch]$Summary)
 $ErrorActionPreference = 'Stop'
 $root = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent
-$generated = Join-Path $root 'generated'
-if (-not (Test-Path (Join-Path $generated 'claude-bash/CLAUDE.md'))) { throw 'Generated output is missing; run build first.' }
-function Measure-Tree([string]$path) {
-    if (-not (Test-Path $path)) { return 0 }
-    return [int64]((Get-ChildItem $path -File -Recurse | ForEach-Object { $_.Length } | Measure-Object -Sum).Sum)
-}
-function Tokens([int64]$bytes) { return [int][Math]::Ceiling($bytes / 4.0) }
-$claude = (Get-Item (Join-Path $generated 'claude-bash/CLAUDE.md')).Length
-$codex = (Get-Item (Join-Path $generated 'codex-bash/AGENTS.md')).Length
-$skills = Measure-Tree (Join-Path $generated 'claude-bash/skills')
-$lazyItem = Get-ChildItem (Join-Path $root 'shared/rules') -File -Recurse | Sort-Object Length -Descending | Select-Object -First 1
-$lazy = if ($lazyItem) { [int64]$lazyItem.Length } else { 0 }
-$baseline = Join-Path $root 'adapters/prompt-budget-baseline.json'
-$old = if (Test-Path $baseline) { Get-Content $baseline -Raw | ConvertFrom-Json } else { $null }
-$rows = @([pscustomobject]@{Name='Claude permanent context';Bytes=$claude;Tokens=(Tokens $claude)},[pscustomobject]@{Name='Codex permanent context';Bytes=$codex;Tokens=(Tokens $codex)},[pscustomobject]@{Name='Claude skill files (lazy)';Bytes=$skills;Tokens=(Tokens $skills)},[pscustomobject]@{Name='Largest lazy rule';Bytes=$lazy;Tokens=(Tokens $lazy)})
-$fail = $false
-$baselineValues = @{}
-if ($old) {
-    foreach ($row in $rows | Select-Object -First 2) {
-        $previous = [double]$old.($row.Name)
-        $absolute = [double]$old.(($row.Name + ' absolute limit'))
-        $baselineValues[$row.Name] = @($previous, $absolute)
-        if ($previous -gt 0 -and $row.Bytes -gt $previous * 1.15) { Write-Warning "Prompt budget soft target exceeded: $($row.Name) $($row.Bytes) bytes (target $previous, 15% tolerance)." }
-        if ($absolute -gt 0 -and $row.Bytes -gt $absolute) { $fail = $true; Write-Error "Prompt budget hard limit failed: $($row.Name) $($row.Bytes) bytes (hard limit $absolute)." }
-    }
-}
-if ($Summary) { Write-Output 'Prompt Budget'; foreach ($row in $rows | Select-Object -First 2) { $limits = $baselineValues[$row.Name]; $base = if($limits){$limits[0]}else{'-'}; $absolute = if($limits){$limits[1]}else{'-'}; Write-Output ('{0,-28} {1,8} bytes  soft target {2}  hard limit {3}' -f $row.Name,$row.Bytes,$base,$absolute) }; $rows | Select-Object -Skip 2 | ForEach-Object { Write-Output ('{0,-28} {1,8} bytes  {2,6} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } } else { $rows | ForEach-Object { Write-Output ('{0}: {1} bytes / {2} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } }
+$budgetScript = Join-Path $root 'scripts/lib/prompt-budget.py'
+$budgetArgs = @()
+if ($Summary) { $budgetArgs += '--summary' }
+& python $budgetScript @budgetArgs
+$budgetExit = $LASTEXITCODE
 & python (Join-Path $root 'scripts/lib/prompt-inventory.py') --home ([Environment]::GetFolderPath('UserProfile'))
 if ($LASTEXITCODE -ne 0) { throw 'Prompt inventory failed.' }
-if ($fail) { exit 1 }
-exit 0
+exit $budgetExit

scripts/commands/prompt-budget.sh

@@ -3,24 +3,9 @@ set -euo pipefail
 summary=false
 for arg in "$@"; do case "$arg" in --summary) summary=true;; *) printf 'Unknown argument: %s\n' "$arg" >&2; exit 1;; esac; done
 root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)
-generated="$root/generated"
-[ -f "$generated/claude-bash/CLAUDE.md" ] || { printf 'Generated output is missing; run build first.\n' >&2; exit 1; }
-bytes() { wc -c < "$1" | tr -d ' '; }
-tree_bytes() { find "$1" -type f -printf '%s\n' 2>/dev/null | awk '{s+=$1} END {print s+0}'; }
-tokens() { awk -v b="$1" 'BEGIN { printf "%d", (b + 3) / 4 }'; }
-claude=$(bytes "$generated/claude-bash/CLAUDE.md"); codex=$(bytes "$generated/codex-bash/AGENTS.md"); skills=$(tree_bytes "$generated/claude-bash/skills"); lazy=$(find "$root/shared/rules" -type f -printf '%s %p\n' | sort -nr | awk 'NR==1 {print $1+0}'); fail=false
-claude_baseline=0; codex_baseline=0; claude_absolute=0; codex_absolute=0
-if [ -f "$root/adapters/prompt-budget-baseline.tsv" ]; then
-  while IFS=$'\t' read -r name old absolute; do
-    [ "$name" = Claude ] || [ "$name" = Codex ] || continue
-    current=$([ "$name" = Claude ] && printf '%s' "$claude" || printf '%s' "$codex")
-    [ "$name" = Claude ] && claude_baseline=$old && claude_absolute=$absolute
-    [ "$name" = Codex ] && codex_baseline=$old && codex_absolute=$absolute
-    if awk -v n="$current" -v o="$old" 'BEGIN { exit !(o > 0 && n > o * 1.15) }'; then printf 'Prompt budget soft target exceeded: %s %s bytes (target %s, 15%% tolerance).\n' "$name" "$current" "$old" >&2; fi
-    if awk -v n="$current" -v a="$absolute" 'BEGIN { exit !(a > 0 && n > a) }'; then printf 'Prompt budget hard limit failed: %s %s bytes (hard limit %s).\n' "$name" "$current" "$absolute" >&2; fail=true; fi
-  done < "$root/adapters/prompt-budget-baseline.tsv"
-fi
-if [ "$summary" = true ]; then printf 'Prompt Budget\nClaude permanent context    %8s bytes  soft target %s  hard limit %s\nCodex permanent context     %8s bytes  soft target %s  hard limit %s\nClaude skill files (lazy)       %8s bytes  %6s tokens\nLargest lazy rule           %8s bytes  %6s tokens\n' "$claude" "$claude_baseline" "$claude_absolute" "$codex" "$codex_baseline" "$codex_absolute" "$skills" "$(tokens "$skills")" "$lazy" "$(tokens "$lazy")"; else printf 'Prompt budget: Claude %s bytes, Codex %s bytes, skills %s bytes, largest lazy rule %s bytes\n' "$claude" "$codex" "$skills" "$lazy"; fi
+budget_args=()
+[ "$summary" = false ] || budget_args+=(--summary)
+budget_status=0
+python3 "$root/scripts/lib/prompt-budget.py" "${budget_args[@]}" || budget_status=$?
 python3 "$root/scripts/lib/prompt-inventory.py" --home "${HOME:?HOME is required}"
-$fail && { printf 'Prompt budget: permanent context exceeded hard limit.\n' >&2; exit 1; }
-exit 0
+exit "$budget_status"

scripts/lib/prompt-budget.py

@@ -0,0 +1,154 @@
+import argparse
+import csv
+import json
+import math
+import re
+import tomllib
+from pathlib import Path
+
+
+METRICS = (
+    'permanent_context_tokens',
+    'skill_metadata_tokens',
+    'rule_catalog_tokens',
+    'agent_metadata_tokens',
+)
+CLIENTS = ('claude', 'codex')
+TOKEN_BYTES = 4
+
+
+def read_frontmatter(path):
+    content = path.read_text(encoding='utf-8-sig')
+    match = re.match(r'\A---\r?\n(.*?)^---\s*$', content, re.MULTILINE | re.DOTALL)
+    return match.group(1).encode('utf-8') if match else b''
+
+
+def tokens(byte_count):
+    return math.ceil(byte_count / TOKEN_BYTES)
+
+
+def measure_package(package, client):
+    instruction_name = 'CLAUDE.md' if client == 'claude' else 'AGENTS.md'
+    instruction_bytes = (package / instruction_name).stat().st_size
+
+    skill_root = package / 'skills'
+    skill_metadata_bytes = sum(
+        len(read_frontmatter(path))
+        for path in skill_root.rglob('SKILL.md')
+        if path.relative_to(skill_root).parts[0] != 'rules'
+    )
+
+    if client == 'claude':
+        rule_files = (package / 'skills' / 'rules').glob('rules-*/SKILL.md')
+    else:
+        rule_root = package / 'rules'
+        rule_files = (
+            path for path in rule_root.rglob('*.md')
+            if 'references' not in path.relative_to(rule_root).parts
+        )
+    rule_catalog_bytes = sum(path.stat().st_size for path in rule_files)
+
+    agent_root = package / 'agents'
+    if client == 'claude':
+        agent_metadata_bytes = sum(len(read_frontmatter(path)) for path in agent_root.glob('*.md'))
+    else:
+        agent_metadata_bytes = 0
+        for path in agent_root.glob('*.toml'):
+            agent = tomllib.loads(path.read_text(encoding='utf-8-sig'))
+            agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description'))
+
+    return {
+        'permanent_context_tokens': tokens(instruction_bytes),
+        'skill_metadata_tokens': tokens(skill_metadata_bytes),
+        'rule_catalog_tokens': tokens(rule_catalog_bytes),
+        'agent_metadata_tokens': tokens(agent_metadata_bytes),
+    }
+
+
+def measure(root):
+    result = {}
+    for client in CLIENTS:
+        packages = [root / 'generated' / f'{client}-{shell}' for shell in ('bash', 'powershell')]
+        for package in packages:
+            if not package.is_dir():
+                raise FileNotFoundError(f'Generated package missing: {package}')
+        measurements = [measure_package(package, client) for package in packages]
+        result[client] = {
+            metric: max(item[metric] for item in measurements)
+            for metric in METRICS
+        }
+    return result
+
+
+def load_baseline(root):
+    path = root / 'adapters' / 'prompt-budget-baseline.json'
+    baseline = json.loads(path.read_text(encoding='utf-8'))
+    if baseline.get('regression_percent') != 10:
+        raise ValueError('Prompt budget regression threshold must be 10%.')
+    metric_values = baseline.get('metrics', {})
+    expected = {(metric, client): metric_values.get(metric, {}).get(client) for metric in METRICS for client in CLIENTS}
+    if any(not isinstance(value, int) or value <= 0 for value in expected.values()):
+        raise ValueError('Prompt budget baseline is missing a positive integer metric.')
+    for key in ('permanent_context_soft_target_tokens', 'permanent_context_hard_limit_tokens'):
+        if not isinstance(baseline.get(key), int) or baseline[key] <= 0:
+            raise ValueError(f'Prompt budget baseline is missing a positive {key}.')
+
+    tsv_path = root / 'adapters' / 'prompt-budget-baseline.tsv'
+    with tsv_path.open(encoding='utf-8', newline='') as stream:
+        rows = list(csv.DictReader(stream, delimiter='\t'))
+    actual = {(row['metric'], row['client']): int(row['baseline_tokens']) for row in rows}
+    if actual != expected:
+        raise ValueError('Prompt budget JSON and TSV baselines do not match.')
+    return baseline
+
+
+def evaluate(current, baseline):
+    failures = []
+    warnings = []
+    limit_percent = 100 + baseline['regression_percent']
+    for metric in METRICS:
+        for client in CLIENTS:
+            value = current[client][metric]
+            previous = baseline['metrics'][metric][client]
+            if value * 100 > previous * limit_percent:
+                failures.append(f'{metric} {client}: {value} tokens (baseline {previous}, limit +{baseline["regression_percent"]}%).')
+
+    hard_limit = baseline['permanent_context_hard_limit_tokens']
+    soft_target = baseline['permanent_context_soft_target_tokens']
+    for client in CLIENTS:
+        value = current[client]['permanent_context_tokens']
+        if value > soft_target:
+            warnings.append(f'permanent_context_tokens {client}: {value} tokens exceeds the {soft_target}-token soft target.')
+        if value > hard_limit:
+            failures.append(f'permanent_context_tokens {client}: {value} tokens exceeds the {hard_limit}-token hard limit.')
+    return failures, warnings
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--summary', action='store_true')
+    args = parser.parse_args()
+    root = Path(__file__).resolve().parents[2]
+    baseline = load_baseline(root)
+    current = measure(root)
+    failures, warnings = evaluate(current, baseline)
+
+    print('Prompt Budget')
+    if args.summary:
+        for metric in METRICS:
+            for client in CLIENTS:
+                value = current[client][metric]
+                previous = baseline['metrics'][metric][client]
+                print(f'{metric} {client}: {value} tokens (baseline {previous}, fail above +{baseline["regression_percent"]}%)')
+    else:
+        for metric in METRICS:
+            print(f'{metric}: Claude {current["claude"][metric]}, Codex {current["codex"][metric]} tokens')
+    for warning in warnings:
+        print(f'WARN {warning}')
+    for failure in failures:
+        print(f'FAIL {failure}')
+    return 1 if failures else 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())