feat(prompt-budget): bundle startup context estimates

This commit is contained in:
Julian lechner committed 2026-09-30 09:43:51 +02:00
1 parent d71db16198
commit 24f4d8e1e5
4 files changed
+35 -21

No files matched your search

+14 -10
View File
@@ -1,23 +1,27 @@
{
"regression_percent": 10,
"permanent_context_soft_target_tokens": 4000,
"permanent_context_hard_limit_tokens": 6000,
"permanent_context_soft_target_tokens": 10000,
"permanent_context_hard_limit_tokens": 15000,
"metrics": {
"permanent_context_tokens": {
"claude": 4029,
"codex": 2603
"claude": 14521,
"codex": 12997
},
"instruction_tokens": {
"claude": 1479,
"codex": 1479
"claude": 1485,
"codex": 1485
},
"skill_metadata_tokens": {
"claude": 2550,
"codex": 1124
"claude": 12233,
"codex": 10784
},
"auto_loaded_rule_tokens": {
"claude": 678,
"codex": 678
},
"rule_catalog_tokens": {
"claude": 28036,
"codex": 8058
"claude": 28818,
"codex": 8510
},
"agent_metadata_tokens": {
"claude": 126,
+10 -8
View File
@@ -1,11 +1,13 @@
metric client baseline_tokens
permanent_context_tokens claude 4029
permanent_context_tokens codex 2603
instruction_tokens claude 1479
instruction_tokens codex 1479
skill_metadata_tokens claude 2550
skill_metadata_tokens codex 1124
rule_catalog_tokens claude 28036
rule_catalog_tokens codex 8058
permanent_context_tokens claude 14521
permanent_context_tokens codex 12997
instruction_tokens claude 1485
instruction_tokens codex 1485
skill_metadata_tokens claude 12233
skill_metadata_tokens codex 10784
auto_loaded_rule_tokens claude 678
auto_loaded_rule_tokens codex 678
rule_catalog_tokens claude 28818
rule_catalog_tokens codex 8510
agent_metadata_tokens claude 126
agent_metadata_tokens codex 52
1 metric client baseline_tokens
2 permanent_context_tokens claude 4029 14521
3 permanent_context_tokens codex 2603 12997
4 instruction_tokens claude 1479 1485
5 instruction_tokens codex 1479 1485
6 skill_metadata_tokens claude 2550 12233
7 skill_metadata_tokens codex 1124 10784
8 rule_catalog_tokens auto_loaded_rule_tokens claude 28036 678
9 rule_catalog_tokens auto_loaded_rule_tokens codex 8058 678
10 rule_catalog_tokens claude 28818
11 rule_catalog_tokens codex 8510
12 agent_metadata_tokens claude 126
13 agent_metadata_tokens codex 52
+10 -2
View File
@@ -11,6 +11,7 @@ METRICS = (
'permanent_context_tokens',
'instruction_tokens',
'skill_metadata_tokens',
'auto_loaded_rule_tokens',
'rule_catalog_tokens',
'agent_metadata_tokens',
)
@@ -40,12 +41,17 @@ def measure_package(package, client):
if client == 'claude':
rule_files = (package / 'skills' / 'rules').glob('rules-*/SKILL.md')
security_rule = package / 'skills' / 'rules' / 'rules-security' / 'SKILL.md'
security_content = security_rule.read_text(encoding='utf-8-sig')
security_header = re.match(r'\A---\r?\n.*?^---[ \t]*\r?$', security_content, re.MULTILINE | re.DOTALL)
auto_loaded_rule_bytes = len(security_content[security_header.end():].encode('utf-8')) if security_header else len(security_content.encode('utf-8'))
else:
rule_root = package / 'rules'
rule_files = (
path for path in rule_root.rglob('*.md')
if 'references' not in path.relative_to(rule_root).parts
)
auto_loaded_rule_bytes = (rule_root / 'security' / 'index.md').stat().st_size
rule_catalog_bytes = sum(path.stat().st_size for path in rule_files)
agent_root = package / 'agents'
@@ -58,9 +64,10 @@ def measure_package(package, client):
agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description'))
return {
'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes),
'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes + agent_metadata_bytes + auto_loaded_rule_bytes),
'instruction_tokens': tokens(instruction_bytes),
'skill_metadata_tokens': tokens(skill_metadata_bytes),
'auto_loaded_rule_tokens': tokens(auto_loaded_rule_bytes),
'rule_catalog_tokens': tokens(rule_catalog_bytes),
'agent_metadata_tokens': tokens(agent_metadata_bytes),
}
@@ -135,7 +142,8 @@ def main():
failures, warnings = evaluate(current, baseline)
print('Prompt Budget (UTF-8 bytes / 4 estimates; not runtime token measurements)')
print('Permanent context includes global instructions and all skill frontmatter; agent metadata, plugins, and tool schemas are separate or excluded.')
print('Bundled estimate includes global instructions, skill and agent metadata, and the always-loaded security baseline.')
print('Project-specific instructions, external plugin prompt bodies, and live MCP/tool schemas are runtime-dependent and not measured here.')
if args.summary:
for metric in METRICS:
for client in CLIENTS:
+1 -1
View File
@@ -40,7 +40,7 @@ def main():
directories.append(args.home / '.agents' / 'skills')
report(f'Installed {client} user catalog', directories)
report(f'Cached {client} plugins (all versions)', [args.home / f'.{client}' / 'plugins' / 'cache'], warn_duplicates=False)
print('Coverage: prompt KPIs cover generated permanent instructions, skill metadata, the full rule catalog, and agent metadata. Rule catalog size does not represent rules loaded in a typical session. Counts are bytes/4 estimates, not runtime token telemetry. Installed prompts, tool schemas, dynamic hook output, and inactive plugin versions are excluded.')
print('Coverage: the bundled estimate includes generated global instructions, all skill and agent metadata, and the always-loaded security baseline. The full rule catalog is inventory size, not automatic load. Project-specific instructions, external plugin prompt bodies, and live MCP/tool schemas are runtime-dependent and excluded. Counts are UTF-8 bytes/4 estimates, not runtime token telemetry.')
if __name__ == '__main__':