feat(prompt-budget): bundle startup context estimates

This commit is contained in:
Julian lechner committed 2026-09-30 09:43:51 +02:00
1 parent d71db16198
commit 24f4d8e1e5
4 files changed
+35 -21

No files matched your search

+14 -10
View File
@@ -1,23 +1,27 @@
{ {
"regression_percent": 10, "regression_percent": 10,
"permanent_context_soft_target_tokens": 4000, "permanent_context_soft_target_tokens": 10000,
"permanent_context_hard_limit_tokens": 6000, "permanent_context_hard_limit_tokens": 15000,
"metrics": { "metrics": {
"permanent_context_tokens": { "permanent_context_tokens": {
"claude": 4029, "claude": 14521,
"codex": 2603 "codex": 12997
}, },
"instruction_tokens": { "instruction_tokens": {
"claude": 1479, "claude": 1485,
"codex": 1479 "codex": 1485
}, },
"skill_metadata_tokens": { "skill_metadata_tokens": {
"claude": 2550, "claude": 12233,
"codex": 1124 "codex": 10784
},
"auto_loaded_rule_tokens": {
"claude": 678,
"codex": 678
}, },
"rule_catalog_tokens": { "rule_catalog_tokens": {
"claude": 28036, "claude": 28818,
"codex": 8058 "codex": 8510
}, },
"agent_metadata_tokens": { "agent_metadata_tokens": {
"claude": 126, "claude": 126,
+10 -8
View File
@@ -1,11 +1,13 @@
metric client baseline_tokens metric client baseline_tokens
permanent_context_tokens claude 4029 permanent_context_tokens claude 14521
permanent_context_tokens codex 2603 permanent_context_tokens codex 12997
instruction_tokens claude 1479 instruction_tokens claude 1485
instruction_tokens codex 1479 instruction_tokens codex 1485
skill_metadata_tokens claude 2550 skill_metadata_tokens claude 12233
skill_metadata_tokens codex 1124 skill_metadata_tokens codex 10784
rule_catalog_tokens claude 28036 auto_loaded_rule_tokens claude 678
rule_catalog_tokens codex 8058 auto_loaded_rule_tokens codex 678
rule_catalog_tokens claude 28818
rule_catalog_tokens codex 8510
agent_metadata_tokens claude 126 agent_metadata_tokens claude 126
agent_metadata_tokens codex 52 agent_metadata_tokens codex 52
1 metric client baseline_tokens
2 permanent_context_tokens claude 4029 14521
3 permanent_context_tokens codex 2603 12997
4 instruction_tokens claude 1479 1485
5 instruction_tokens codex 1479 1485
6 skill_metadata_tokens claude 2550 12233
7 skill_metadata_tokens codex 1124 10784
8 rule_catalog_tokens auto_loaded_rule_tokens claude 28036 678
9 rule_catalog_tokens auto_loaded_rule_tokens codex 8058 678
10 rule_catalog_tokens claude 28818
11 rule_catalog_tokens codex 8510
12 agent_metadata_tokens claude 126
13 agent_metadata_tokens codex 52
+10 -2
View File
@@ -11,6 +11,7 @@ METRICS = (
'permanent_context_tokens', 'permanent_context_tokens',
'instruction_tokens', 'instruction_tokens',
'skill_metadata_tokens', 'skill_metadata_tokens',
'auto_loaded_rule_tokens',
'rule_catalog_tokens', 'rule_catalog_tokens',
'agent_metadata_tokens', 'agent_metadata_tokens',
) )
@@ -40,12 +41,17 @@ def measure_package(package, client):
if client == 'claude': if client == 'claude':
rule_files = (package / 'skills' / 'rules').glob('rules-*/SKILL.md') rule_files = (package / 'skills' / 'rules').glob('rules-*/SKILL.md')
security_rule = package / 'skills' / 'rules' / 'rules-security' / 'SKILL.md'
security_content = security_rule.read_text(encoding='utf-8-sig')
security_header = re.match(r'\A---\r?\n.*?^---[ \t]*\r?$', security_content, re.MULTILINE | re.DOTALL)
auto_loaded_rule_bytes = len(security_content[security_header.end():].encode('utf-8')) if security_header else len(security_content.encode('utf-8'))
else: else:
rule_root = package / 'rules' rule_root = package / 'rules'
rule_files = ( rule_files = (
path for path in rule_root.rglob('*.md') path for path in rule_root.rglob('*.md')
if 'references' not in path.relative_to(rule_root).parts if 'references' not in path.relative_to(rule_root).parts
) )
auto_loaded_rule_bytes = (rule_root / 'security' / 'index.md').stat().st_size
rule_catalog_bytes = sum(path.stat().st_size for path in rule_files) rule_catalog_bytes = sum(path.stat().st_size for path in rule_files)
agent_root = package / 'agents' agent_root = package / 'agents'
@@ -58,9 +64,10 @@ def measure_package(package, client):
agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description')) agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description'))
return { return {
'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes), 'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes + agent_metadata_bytes + auto_loaded_rule_bytes),
'instruction_tokens': tokens(instruction_bytes), 'instruction_tokens': tokens(instruction_bytes),
'skill_metadata_tokens': tokens(skill_metadata_bytes), 'skill_metadata_tokens': tokens(skill_metadata_bytes),
'auto_loaded_rule_tokens': tokens(auto_loaded_rule_bytes),
'rule_catalog_tokens': tokens(rule_catalog_bytes), 'rule_catalog_tokens': tokens(rule_catalog_bytes),
'agent_metadata_tokens': tokens(agent_metadata_bytes), 'agent_metadata_tokens': tokens(agent_metadata_bytes),
} }
@@ -135,7 +142,8 @@ def main():
failures, warnings = evaluate(current, baseline) failures, warnings = evaluate(current, baseline)
print('Prompt Budget (UTF-8 bytes / 4 estimates; not runtime token measurements)') print('Prompt Budget (UTF-8 bytes / 4 estimates; not runtime token measurements)')
print('Permanent context includes global instructions and all skill frontmatter; agent metadata, plugins, and tool schemas are separate or excluded.') print('Bundled estimate includes global instructions, skill and agent metadata, and the always-loaded security baseline.')
print('Project-specific instructions, external plugin prompt bodies, and live MCP/tool schemas are runtime-dependent and not measured here.')
if args.summary: if args.summary:
for metric in METRICS: for metric in METRICS:
for client in CLIENTS: for client in CLIENTS:
+1 -1
View File
@@ -40,7 +40,7 @@ def main():
directories.append(args.home / '.agents' / 'skills') directories.append(args.home / '.agents' / 'skills')
report(f'Installed {client} user catalog', directories) report(f'Installed {client} user catalog', directories)
report(f'Cached {client} plugins (all versions)', [args.home / f'.{client}' / 'plugins' / 'cache'], warn_duplicates=False) report(f'Cached {client} plugins (all versions)', [args.home / f'.{client}' / 'plugins' / 'cache'], warn_duplicates=False)
print('Coverage: prompt KPIs cover generated permanent instructions, skill metadata, the full rule catalog, and agent metadata. Rule catalog size does not represent rules loaded in a typical session. Counts are bytes/4 estimates, not runtime token telemetry. Installed prompts, tool schemas, dynamic hook output, and inactive plugin versions are excluded.') print('Coverage: the bundled estimate includes generated global instructions, all skill and agent metadata, and the always-loaded security baseline. The full rule catalog is inventory size, not automatic load. Project-specific instructions, external plugin prompt bodies, and live MCP/tool schemas are runtime-dependent and excluded. Counts are UTF-8 bytes/4 estimates, not runtime token telemetry.')
if __name__ == '__main__': if __name__ == '__main__':