fix(prompt-budget): count all startup skill metadata

This commit is contained in:
Julian lechner committed 2026-09-30 09:38:41 +02:00
1 parent 17e8be13b7
commit 066cbb7037
3 files changed
+17 -9

No files matched your search

+7 -3
View File
@@ -4,11 +4,15 @@
"permanent_context_hard_limit_tokens": 6000, "permanent_context_hard_limit_tokens": 6000,
"metrics": { "metrics": {
"permanent_context_tokens": { "permanent_context_tokens": {
"claude": 1481, "claude": 4029,
"codex": 1481 "codex": 2603
},
"instruction_tokens": {
"claude": 1479,
"codex": 1479
}, },
"skill_metadata_tokens": { "skill_metadata_tokens": {
"claude": 1124, "claude": 2550,
"codex": 1124 "codex": 1124
}, },
"rule_catalog_tokens": { "rule_catalog_tokens": {
+5 -3
View File
@@ -1,7 +1,9 @@
metric client baseline_tokens metric client baseline_tokens
permanent_context_tokens claude 1481 permanent_context_tokens claude 4029
permanent_context_tokens codex 1481 permanent_context_tokens codex 2603
skill_metadata_tokens claude 1124 instruction_tokens claude 1479
instruction_tokens codex 1479
skill_metadata_tokens claude 2550
skill_metadata_tokens codex 1124 skill_metadata_tokens codex 1124
rule_catalog_tokens claude 28036 rule_catalog_tokens claude 28036
rule_catalog_tokens codex 8058 rule_catalog_tokens codex 8058
1 metric client baseline_tokens
2 permanent_context_tokens claude 1481 4029
3 permanent_context_tokens codex 1481 2603
4 skill_metadata_tokens instruction_tokens claude 1124 1479
5 instruction_tokens codex 1479
6 skill_metadata_tokens claude 2550
7 skill_metadata_tokens codex 1124
8 rule_catalog_tokens claude 28036
9 rule_catalog_tokens codex 8058
+5 -3
View File
@@ -9,6 +9,7 @@ from pathlib import Path
METRICS = ( METRICS = (
'permanent_context_tokens', 'permanent_context_tokens',
'instruction_tokens',
'skill_metadata_tokens', 'skill_metadata_tokens',
'rule_catalog_tokens', 'rule_catalog_tokens',
'agent_metadata_tokens', 'agent_metadata_tokens',
@@ -35,7 +36,6 @@ def measure_package(package, client):
skill_metadata_bytes = sum( skill_metadata_bytes = sum(
len(read_frontmatter(path)) len(read_frontmatter(path))
for path in skill_root.rglob('SKILL.md') for path in skill_root.rglob('SKILL.md')
if path.relative_to(skill_root).parts[0] != 'rules'
) )
if client == 'claude': if client == 'claude':
@@ -58,7 +58,8 @@ def measure_package(package, client):
agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description')) agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description'))
return { return {
'permanent_context_tokens': tokens(instruction_bytes), 'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes),
'instruction_tokens': tokens(instruction_bytes),
'skill_metadata_tokens': tokens(skill_metadata_bytes), 'skill_metadata_tokens': tokens(skill_metadata_bytes),
'rule_catalog_tokens': tokens(rule_catalog_bytes), 'rule_catalog_tokens': tokens(rule_catalog_bytes),
'agent_metadata_tokens': tokens(agent_metadata_bytes), 'agent_metadata_tokens': tokens(agent_metadata_bytes),
@@ -133,7 +134,8 @@ def main():
current = measure(root) current = measure(root)
failures, warnings = evaluate(current, baseline) failures, warnings = evaluate(current, baseline)
print('Prompt Budget') print('Prompt Budget (UTF-8 bytes / 4 estimates; not runtime token measurements)')
print('Permanent context includes global instructions and all skill frontmatter; agent metadata, plugins, and tool schemas are separate or excluded.')
if args.summary: if args.summary:
for metric in METRICS: for metric in METRICS:
for client in CLIENTS: for client in CLIENTS: