From 24f4d8e1e51f3af6d0a99c1b147425445ff36346 Mon Sep 17 00:00:00 2001 From: Julian lechner Date: Wed, 30 Sep 2026 09:43:51 +0200 Subject: [PATCH] feat(prompt-budget): bundle startup context estimates --- adapters/prompt-budget-baseline.json | 24 ++++++++++++++---------- adapters/prompt-budget-baseline.tsv | 18 ++++++++++-------- scripts/lib/prompt-budget.py | 12 ++++++++++-- scripts/lib/prompt-inventory.py | 2 +- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/adapters/prompt-budget-baseline.json b/adapters/prompt-budget-baseline.json index ac739ab..37299d9 100644 --- a/adapters/prompt-budget-baseline.json +++ b/adapters/prompt-budget-baseline.json @@ -1,23 +1,27 @@ { "regression_percent": 10, - "permanent_context_soft_target_tokens": 4000, - "permanent_context_hard_limit_tokens": 6000, + "permanent_context_soft_target_tokens": 10000, + "permanent_context_hard_limit_tokens": 15000, "metrics": { "permanent_context_tokens": { - "claude": 4029, - "codex": 2603 + "claude": 14521, + "codex": 12997 }, "instruction_tokens": { - "claude": 1479, - "codex": 1479 + "claude": 1485, + "codex": 1485 }, "skill_metadata_tokens": { - "claude": 2550, - "codex": 1124 + "claude": 12233, + "codex": 10784 + }, + "auto_loaded_rule_tokens": { + "claude": 678, + "codex": 678 }, "rule_catalog_tokens": { - "claude": 28036, - "codex": 8058 + "claude": 28818, + "codex": 8510 }, "agent_metadata_tokens": { "claude": 126, diff --git a/adapters/prompt-budget-baseline.tsv b/adapters/prompt-budget-baseline.tsv index 08e78fd..0abd1da 100644 --- a/adapters/prompt-budget-baseline.tsv +++ b/adapters/prompt-budget-baseline.tsv @@ -1,11 +1,13 @@ metric client baseline_tokens -permanent_context_tokens claude 4029 -permanent_context_tokens codex 2603 -instruction_tokens claude 1479 -instruction_tokens codex 1479 -skill_metadata_tokens claude 2550 -skill_metadata_tokens codex 1124 -rule_catalog_tokens claude 28036 -rule_catalog_tokens codex 8058 +permanent_context_tokens claude 14521 +permanent_context_tokens codex 12997 +instruction_tokens claude 1485 +instruction_tokens codex 1485 +skill_metadata_tokens claude 12233 +skill_metadata_tokens codex 10784 +auto_loaded_rule_tokens claude 678 +auto_loaded_rule_tokens codex 678 +rule_catalog_tokens claude 28818 +rule_catalog_tokens codex 8510 agent_metadata_tokens claude 126 agent_metadata_tokens codex 52 diff --git a/scripts/lib/prompt-budget.py b/scripts/lib/prompt-budget.py index c0c9ae4..84bb7a8 100644 --- a/scripts/lib/prompt-budget.py +++ b/scripts/lib/prompt-budget.py @@ -11,6 +11,7 @@ METRICS = ( 'permanent_context_tokens', 'instruction_tokens', 'skill_metadata_tokens', + 'auto_loaded_rule_tokens', 'rule_catalog_tokens', 'agent_metadata_tokens', ) @@ -40,12 +41,17 @@ def measure_package(package, client): if client == 'claude': rule_files = (package / 'skills' / 'rules').glob('rules-*/SKILL.md') + security_rule = package / 'skills' / 'rules' / 'rules-security' / 'SKILL.md' + security_content = security_rule.read_text(encoding='utf-8-sig') + security_header = re.match(r'\A---\r?\n.*?^---[ \t]*\r?$', security_content, re.MULTILINE | re.DOTALL) + auto_loaded_rule_bytes = len(security_content[security_header.end():].encode('utf-8')) if security_header else len(security_content.encode('utf-8')) else: rule_root = package / 'rules' rule_files = ( path for path in rule_root.rglob('*.md') if 'references' not in path.relative_to(rule_root).parts ) + auto_loaded_rule_bytes = (rule_root / 'security' / 'index.md').stat().st_size rule_catalog_bytes = sum(path.stat().st_size for path in rule_files) agent_root = package / 'agents' @@ -58,9 +64,10 @@ def measure_package(package, client): agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description')) return { - 'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes), + 'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes + agent_metadata_bytes + auto_loaded_rule_bytes), 'instruction_tokens': tokens(instruction_bytes), 'skill_metadata_tokens': tokens(skill_metadata_bytes), + 'auto_loaded_rule_tokens': tokens(auto_loaded_rule_bytes), 'rule_catalog_tokens': tokens(rule_catalog_bytes), 'agent_metadata_tokens': tokens(agent_metadata_bytes), } @@ -135,7 +142,8 @@ def main(): failures, warnings = evaluate(current, baseline) print('Prompt Budget (UTF-8 bytes / 4 estimates; not runtime token measurements)') - print('Permanent context includes global instructions and all skill frontmatter; agent metadata, plugins, and tool schemas are separate or excluded.') + print('Bundled estimate includes global instructions, skill and agent metadata, and the always-loaded security baseline.') + print('Project-specific instructions, external plugin prompt bodies, and live MCP/tool schemas are runtime-dependent and not measured here.') if args.summary: for metric in METRICS: for client in CLIENTS: diff --git a/scripts/lib/prompt-inventory.py b/scripts/lib/prompt-inventory.py index 7747dda..6de0d07 100644 --- a/scripts/lib/prompt-inventory.py +++ b/scripts/lib/prompt-inventory.py @@ -40,7 +40,7 @@ def main(): directories.append(args.home / '.agents' / 'skills') report(f'Installed {client} user catalog', directories) report(f'Cached {client} plugins (all versions)', [args.home / f'.{client}' / 'plugins' / 'cache'], warn_duplicates=False) - print('Coverage: prompt KPIs cover generated permanent instructions, skill metadata, the full rule catalog, and agent metadata. Rule catalog size does not represent rules loaded in a typical session. Counts are bytes/4 estimates, not runtime token telemetry. Installed prompts, tool schemas, dynamic hook output, and inactive plugin versions are excluded.') + print('Coverage: the bundled estimate includes generated global instructions, all skill and agent metadata, and the always-loaded security baseline. The full rule catalog is inventory size, not automatic load. Project-specific instructions, external plugin prompt bodies, and live MCP/tool schemas are runtime-dependent and excluded. Counts are UTF-8 bytes/4 estimates, not runtime token telemetry.') if __name__ == '__main__':