feat(prompt-budget): bundle startup context estimates
This commit is contained in:
1 parent
d71db16198
commit
24f4d8e1e5
4 files changed
+35
-21
No files matched your search
@@ -1,23 +1,27 @@
|
|||||||
{
|
{
|
||||||
"regression_percent": 10,
|
"regression_percent": 10,
|
||||||
"permanent_context_soft_target_tokens": 4000,
|
"permanent_context_soft_target_tokens": 10000,
|
||||||
"permanent_context_hard_limit_tokens": 6000,
|
"permanent_context_hard_limit_tokens": 15000,
|
||||||
"metrics": {
|
"metrics": {
|
||||||
"permanent_context_tokens": {
|
"permanent_context_tokens": {
|
||||||
"claude": 4029,
|
"claude": 14521,
|
||||||
"codex": 2603
|
"codex": 12997
|
||||||
},
|
},
|
||||||
"instruction_tokens": {
|
"instruction_tokens": {
|
||||||
"claude": 1479,
|
"claude": 1485,
|
||||||
"codex": 1479
|
"codex": 1485
|
||||||
},
|
},
|
||||||
"skill_metadata_tokens": {
|
"skill_metadata_tokens": {
|
||||||
"claude": 2550,
|
"claude": 12233,
|
||||||
"codex": 1124
|
"codex": 10784
|
||||||
|
},
|
||||||
|
"auto_loaded_rule_tokens": {
|
||||||
|
"claude": 678,
|
||||||
|
"codex": 678
|
||||||
},
|
},
|
||||||
"rule_catalog_tokens": {
|
"rule_catalog_tokens": {
|
||||||
"claude": 28036,
|
"claude": 28818,
|
||||||
"codex": 8058
|
"codex": 8510
|
||||||
},
|
},
|
||||||
"agent_metadata_tokens": {
|
"agent_metadata_tokens": {
|
||||||
"claude": 126,
|
"claude": 126,
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
metric client baseline_tokens
|
metric client baseline_tokens
|
||||||
permanent_context_tokens claude 4029
|
permanent_context_tokens claude 14521
|
||||||
permanent_context_tokens codex 2603
|
permanent_context_tokens codex 12997
|
||||||
instruction_tokens claude 1479
|
instruction_tokens claude 1485
|
||||||
instruction_tokens codex 1479
|
instruction_tokens codex 1485
|
||||||
skill_metadata_tokens claude 2550
|
skill_metadata_tokens claude 12233
|
||||||
skill_metadata_tokens codex 1124
|
skill_metadata_tokens codex 10784
|
||||||
rule_catalog_tokens claude 28036
|
auto_loaded_rule_tokens claude 678
|
||||||
rule_catalog_tokens codex 8058
|
auto_loaded_rule_tokens codex 678
|
||||||
|
rule_catalog_tokens claude 28818
|
||||||
|
rule_catalog_tokens codex 8510
|
||||||
agent_metadata_tokens claude 126
|
agent_metadata_tokens claude 126
|
||||||
agent_metadata_tokens codex 52
|
agent_metadata_tokens codex 52
|
||||||
|
@@ -11,6 +11,7 @@ METRICS = (
|
|||||||
'permanent_context_tokens',
|
'permanent_context_tokens',
|
||||||
'instruction_tokens',
|
'instruction_tokens',
|
||||||
'skill_metadata_tokens',
|
'skill_metadata_tokens',
|
||||||
|
'auto_loaded_rule_tokens',
|
||||||
'rule_catalog_tokens',
|
'rule_catalog_tokens',
|
||||||
'agent_metadata_tokens',
|
'agent_metadata_tokens',
|
||||||
)
|
)
|
||||||
@@ -40,12 +41,17 @@ def measure_package(package, client):
|
|||||||
|
|
||||||
if client == 'claude':
|
if client == 'claude':
|
||||||
rule_files = (package / 'skills' / 'rules').glob('rules-*/SKILL.md')
|
rule_files = (package / 'skills' / 'rules').glob('rules-*/SKILL.md')
|
||||||
|
security_rule = package / 'skills' / 'rules' / 'rules-security' / 'SKILL.md'
|
||||||
|
security_content = security_rule.read_text(encoding='utf-8-sig')
|
||||||
|
security_header = re.match(r'\A---\r?\n.*?^---[ \t]*\r?$', security_content, re.MULTILINE | re.DOTALL)
|
||||||
|
auto_loaded_rule_bytes = len(security_content[security_header.end():].encode('utf-8')) if security_header else len(security_content.encode('utf-8'))
|
||||||
else:
|
else:
|
||||||
rule_root = package / 'rules'
|
rule_root = package / 'rules'
|
||||||
rule_files = (
|
rule_files = (
|
||||||
path for path in rule_root.rglob('*.md')
|
path for path in rule_root.rglob('*.md')
|
||||||
if 'references' not in path.relative_to(rule_root).parts
|
if 'references' not in path.relative_to(rule_root).parts
|
||||||
)
|
)
|
||||||
|
auto_loaded_rule_bytes = (rule_root / 'security' / 'index.md').stat().st_size
|
||||||
rule_catalog_bytes = sum(path.stat().st_size for path in rule_files)
|
rule_catalog_bytes = sum(path.stat().st_size for path in rule_files)
|
||||||
|
|
||||||
agent_root = package / 'agents'
|
agent_root = package / 'agents'
|
||||||
@@ -58,9 +64,10 @@ def measure_package(package, client):
|
|||||||
agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description'))
|
agent_metadata_bytes += sum(len(agent.get(key, '').encode('utf-8')) for key in ('name', 'description'))
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes),
|
'permanent_context_tokens': tokens(instruction_bytes + skill_metadata_bytes + agent_metadata_bytes + auto_loaded_rule_bytes),
|
||||||
'instruction_tokens': tokens(instruction_bytes),
|
'instruction_tokens': tokens(instruction_bytes),
|
||||||
'skill_metadata_tokens': tokens(skill_metadata_bytes),
|
'skill_metadata_tokens': tokens(skill_metadata_bytes),
|
||||||
|
'auto_loaded_rule_tokens': tokens(auto_loaded_rule_bytes),
|
||||||
'rule_catalog_tokens': tokens(rule_catalog_bytes),
|
'rule_catalog_tokens': tokens(rule_catalog_bytes),
|
||||||
'agent_metadata_tokens': tokens(agent_metadata_bytes),
|
'agent_metadata_tokens': tokens(agent_metadata_bytes),
|
||||||
}
|
}
|
||||||
@@ -135,7 +142,8 @@ def main():
|
|||||||
failures, warnings = evaluate(current, baseline)
|
failures, warnings = evaluate(current, baseline)
|
||||||
|
|
||||||
print('Prompt Budget (UTF-8 bytes / 4 estimates; not runtime token measurements)')
|
print('Prompt Budget (UTF-8 bytes / 4 estimates; not runtime token measurements)')
|
||||||
print('Permanent context includes global instructions and all skill frontmatter; agent metadata, plugins, and tool schemas are separate or excluded.')
|
print('Bundled estimate includes global instructions, skill and agent metadata, and the always-loaded security baseline.')
|
||||||
|
print('Project-specific instructions, external plugin prompt bodies, and live MCP/tool schemas are runtime-dependent and not measured here.')
|
||||||
if args.summary:
|
if args.summary:
|
||||||
for metric in METRICS:
|
for metric in METRICS:
|
||||||
for client in CLIENTS:
|
for client in CLIENTS:
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ def main():
|
|||||||
directories.append(args.home / '.agents' / 'skills')
|
directories.append(args.home / '.agents' / 'skills')
|
||||||
report(f'Installed {client} user catalog', directories)
|
report(f'Installed {client} user catalog', directories)
|
||||||
report(f'Cached {client} plugins (all versions)', [args.home / f'.{client}' / 'plugins' / 'cache'], warn_duplicates=False)
|
report(f'Cached {client} plugins (all versions)', [args.home / f'.{client}' / 'plugins' / 'cache'], warn_duplicates=False)
|
||||||
print('Coverage: prompt KPIs cover generated permanent instructions, skill metadata, the full rule catalog, and agent metadata. Rule catalog size does not represent rules loaded in a typical session. Counts are bytes/4 estimates, not runtime token telemetry. Installed prompts, tool schemas, dynamic hook output, and inactive plugin versions are excluded.')
|
print('Coverage: the bundled estimate includes generated global instructions, all skill and agent metadata, and the always-loaded security baseline. The full rule catalog is inventory size, not automatic load. Project-specific instructions, external plugin prompt bodies, and live MCP/tool schemas are runtime-dependent and excluded. Counts are UTF-8 bytes/4 estimates, not runtime token telemetry.')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|||||||
Reference in new issue
Block a user