From bdf98d5b71ae1e33a2fd2c8cdd6ea90ec8d4ad9d Mon Sep 17 00:00:00 2001 From: Julian lechner Date: Thu, 17 Sep 2026 08:32:39 +0200 Subject: [PATCH] feat(telemetry): add local agent efficiency metrics --- scripts/telemetry.ps1 | 14 +++++---- scripts/telemetry.sh | 38 ++++++++++++++++++++++--- scripts/test-telemetry.py | 37 ++++++++++++++++++++++++ shared/hooks/scripts/Record-Compact.ps1 | 8 ++++-- shared/hooks/scripts/record-compact.sh | 3 +- 5 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 scripts/test-telemetry.py diff --git a/scripts/telemetry.ps1 b/scripts/telemetry.ps1 index 266a52c..8b81eaf 100644 --- a/scripts/telemetry.ps1 +++ b/scripts/telemetry.ps1 @@ -1,8 +1,10 @@ [CmdletBinding()] -param([ValidateSet('record','summary','rotate')][string]$Action='record',[string]$Event,[string]$Agent,[string]$Role,[int]$DurationMs,[int]$ToolCalls,[string]$CommandClass,[int64]$Bytes,[int]$Count) +param([ValidateSet('record','summary','rotate')][string]$Action='record',[string]$Event,[string]$Agent,[string]$Role,[Nullable[int]]$DurationMs,[Nullable[int]]$ToolCalls,[string]$CommandClass,[Nullable[int64]]$Bytes,[Nullable[int]]$Count,[Nullable[int]]$HandoffBytes,[Nullable[int]]$FilesRead,[Nullable[int]]$FilesChanged,[string]$Outcome,[Nullable[int]]$InputTokens,[Nullable[int]]$CachedTokens,[Nullable[int]]$OutputTokens,[Nullable[int]]$ReasoningTokens) $ErrorActionPreference = 'Stop'; if ($env:AI_CONFIG_TELEMETRY -eq '0') { exit 0 } -$root = Split-Path $PSScriptRoot -Parent; $dir = Join-Path $root '.ai-session'; $path = Join-Path $dir 'telemetry.jsonl'; New-Item $dir -ItemType Directory -Force | Out-Null -if ($Action -eq 'rotate') { if (Test-Path $path) { Move-Item $path ($path + '.1') -Force }; exit 0 } -if ($Action -eq 'summary') { if (-not (Test-Path $path)) { Write-Output 'Telemetry: no events'; exit 0 }; $events=@(Get-Content $path | ForEach-Object { try { $_ | ConvertFrom-Json } catch {} }); $agents=@($events | Where-Object event -eq 'subagent-stop'); Write-Output ('Telemetry: {0} events | {1} subagents' -f $events.Count,$agents.Count); exit 0 } -if ([string]::IsNullOrWhiteSpace($Event)) { throw 'Event is required.' } -$record=[ordered]@{event=$Event;timestamp=(Get-Date).ToUniversalTime().ToString('o')}; foreach($pair in @(@('agent',$Agent),@('role',$Role),@('durationMs',$DurationMs),@('toolCalls',$ToolCalls),@('commandClass',$CommandClass),@('bytes',$Bytes),@('count',$Count))){if($pair[1]){$record[$pair[0]]=$pair[1]}}; ($record|ConvertTo-Json -Compress)|Add-Content $path; if((Get-Item $path).Length -gt 1048576){Move-Item $path ($path+'.1') -Force} +try { + $root = Split-Path $PSScriptRoot -Parent; $dir = Join-Path $root '.ai-session'; $path = Join-Path $dir 'telemetry.jsonl'; New-Item $dir -ItemType Directory -Force | Out-Null + if ($Action -eq 'rotate') { if (Test-Path $path) { Move-Item $path ($path + '.1') -Force }; exit 0 } + if ($Action -eq 'summary') { if (-not (Test-Path $path)) { Write-Output 'Telemetry: no events'; exit 0 }; $events=@(Get-Content $path | ForEach-Object { try { $_ | ConvertFrom-Json } catch {} }); Write-Output ('Telemetry: {0} events' -f $events.Count); foreach($group in @($events | Where-Object role | Group-Object role | Sort-Object Name)){ $inputs=@($group.Group|Where-Object tokenUsage|ForEach-Object {$_.tokenUsage.input}|Where-Object {$_ -is [int] -or $_ -is [long]}|Sort-Object); $median=if($inputs.Count){$inputs[[int][Math]::Floor($inputs.Count/2)]}else{'-'}; Write-Output ('{0}: runs {1} findings {2} median input {3}' -f $group.Name,$group.Count,@($group.Group|Where-Object outcome -eq findings).Count,$median)}; exit 0 } + if ([string]::IsNullOrWhiteSpace($Event)) { throw 'Event is required.' } + $record=[ordered]@{event=$Event;timestamp=(Get-Date).ToUniversalTime().ToString('o')}; foreach($pair in @(@('agent',$Agent),@('role',$Role),@('durationMs',$DurationMs),@('toolCalls',$ToolCalls),@('commandClass',$CommandClass),@('bytes',$Bytes),@('count',$Count),@('handoffBytes',$HandoffBytes),@('filesRead',$FilesRead),@('filesChanged',$FilesChanged),@('outcome',$Outcome))){if($null -ne $pair[1] -and "$($pair[1])" -ne ''){$record[$pair[0]]=$pair[1]}}; $tokens=[ordered]@{}; foreach($pair in @(@('input',$InputTokens),@('cached',$CachedTokens),@('output',$OutputTokens),@('reasoning',$ReasoningTokens))){if($null -ne $pair[1]){$tokens[$pair[0]]=$pair[1]}}; if($tokens.Count){$record.tokenUsage=$tokens}; ($record|ConvertTo-Json -Compress)|Add-Content $path; if((Get-Item $path).Length -gt 1048576){Move-Item $path ($path+'.1') -Force} +} catch { exit 0 } diff --git a/scripts/telemetry.sh b/scripts/telemetry.sh index e8f7ef0..14b305e 100644 --- a/scripts/telemetry.sh +++ b/scripts/telemetry.sh @@ -2,9 +2,39 @@ set -euo pipefail [ "${AI_CONFIG_TELEMETRY:-1}" = 0 ] && exit 0 root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd); dir="$root/.ai-session"; path="$dir/telemetry.jsonl"; mkdir -p "$dir" -case "${1:-record}" in +action=${1:-record}; shift || true +case "$action" in rotate) [ -f "$path" ] && mv -f "$path" "$path.1"; exit 0;; - summary) [ -f "$path" ] || { printf 'Telemetry: no events\n'; exit 0; }; count=$(wc -l < "$path" | tr -d ' '); agents=$(grep -c '"event":"subagent-stop"' "$path" || true); printf 'Telemetry: %s events | %s subagents\n' "$count" "$agents"; exit 0;; - record) event=${2:?event required};; *) printf 'Unknown action: %s\n' "$1" >&2; exit 1;; + summary) [ -f "$path" ] || { printf 'Telemetry: no events\n'; exit 0; }; python3 - "$path" <<'PY' || exit 0 +import json,sys,statistics +rows=[] +for line in open(sys.argv[1],encoding='utf-8',errors='replace'): + try: rows.append(json.loads(line)) + except Exception: pass +print(f'Telemetry: {len(rows)} events') +for role in sorted({r.get('role') for r in rows if r.get('role')}): + subset=[r for r in rows if r.get('role')==role] + inputs=[r['tokenUsage']['input'] for r in subset if isinstance(r.get('tokenUsage'),dict) and isinstance(r['tokenUsage'].get('input'),(int,float))] + print(f"{role}: runs {len(subset)} findings {sum(r.get('outcome')=='findings' for r in subset)} median input {int(statistics.median(inputs)) if inputs else '-'}") +PY + exit 0;; + record) event=${1:?event required}; shift;; *) printf 'Unknown action: %s\n' "$action" >&2; exit 1;; esac -printf '{"event":"%s","timestamp":"%s"}\n' "$event" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$path"; [ "$(wc -c < "$path")" -le 1048576 ] || mv -f "$path" "$path.1" +python3 - "$path" "$event" "$@" <<'PY' || exit 0 +import json,sys,datetime +path,event=sys.argv[1:3]; args=sys.argv[3:] +record={'event':event,'timestamp':datetime.datetime.now(datetime.timezone.utc).isoformat()} +keys=['role','durationMs','handoffBytes','filesRead','filesChanged','outcome','input','cached','output','reasoning'] +if any(a.startswith('--') for a in args): + parsed={}; i=0 + while i < len(args): + if args[i].startswith('--') and i+1 < len(args): parsed[args[i][2:]]=args[i+1]; i+=2 + else: i+=1 + args=[parsed.get(k,'') for k in keys] +for key,value in zip(keys,args): + if value: record[key]=int(value) if value.lstrip('-').isdigit() else value +tokens={k:record.pop(k) for k in ('input','cached','output','reasoning') if k in record} +if tokens: record['tokenUsage']=tokens +with open(path,'a',encoding='utf-8') as out: out.write(json.dumps(record,separators=(',',':'))+'\n') +PY +if [ "$(wc -c < "$path")" -gt 1048576 ]; then mv -f "$path" "$path.1"; fi diff --git a/scripts/test-telemetry.py b/scripts/test-telemetry.py new file mode 100644 index 0000000..e3cdffa --- /dev/null +++ b/scripts/test-telemetry.py @@ -0,0 +1,37 @@ +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parent.parent + + +class TelemetryTests(unittest.TestCase): + def test_bash_record_disable_and_summary(self): + bash = shutil.which('bash') + if not bash: + self.skipTest('Bash unavailable') + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + script = root / 'scripts/telemetry.sh' + script.parent.mkdir() + shutil.copy(ROOT / 'scripts/telemetry.sh', script) + env = dict(os.environ, AI_CONFIG_TELEMETRY='1') + result = subprocess.run([bash, str(script), 'record', 'subagent-stop', '--role', 'reviewer', '--filesRead', '3', '--outcome', 'findings'], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + event = json.loads((root / '.ai-session/telemetry.jsonl').read_text().strip()) + self.assertEqual(event['role'], 'reviewer') + self.assertEqual(event['filesRead'], 3) + self.assertNotIn('prompt', event) + disabled = dict(env, AI_CONFIG_TELEMETRY='0') + subprocess.run([bash, str(script), 'record', 'ignored'], env=disabled, check=True) + summary = subprocess.run([bash, str(script), 'summary'], env=env, capture_output=True, text=True, check=True) + self.assertIn('reviewer: runs 1 findings 1', summary.stdout) + + +if __name__ == '__main__': + unittest.main() diff --git a/shared/hooks/scripts/Record-Compact.ps1 b/shared/hooks/scripts/Record-Compact.ps1 index fa2cc96..ffb5f9b 100644 --- a/shared/hooks/scripts/Record-Compact.ps1 +++ b/shared/hooks/scripts/Record-Compact.ps1 @@ -2,7 +2,11 @@ param() $root = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent $stateDir = Join-Path $root '.ai-session' -New-Item $stateDir -ItemType Directory -Force | Out-Null -('{"event":"compact","timestamp":"' + (Get-Date).ToUniversalTime().ToString('o') + '"}') | Add-Content (Join-Path $stateDir 'telemetry.jsonl') +if ($env:AI_CONFIG_TELEMETRY -ne '0') { + New-Item $stateDir -ItemType Directory -Force | Out-Null + $path = Join-Path $stateDir 'telemetry.jsonl' + ('{"event":"compact","timestamp":"' + (Get-Date).ToUniversalTime().ToString('o') + '"}') | Add-Content $path + if ((Get-Item $path).Length -gt 1048576) { Move-Item $path ($path + '.1') -Force } +} & python (Join-Path $PSScriptRoot 'session-state.py') --hook exit $LASTEXITCODE diff --git a/shared/hooks/scripts/record-compact.sh b/shared/hooks/scripts/record-compact.sh index 3f81144..ace9734 100644 --- a/shared/hooks/scripts/record-compact.sh +++ b/shared/hooks/scripts/record-compact.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash set -euo pipefail root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) -mkdir -p "$root/.ai-session" -printf '{"event":"compact","timestamp":"%s"}\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$root/.ai-session/telemetry.jsonl" +[ "${AI_CONFIG_TELEMETRY:-1}" = 0 ] || { mkdir -p "$root/.ai-session" && printf '{"event":"compact","timestamp":"%s"}\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$root/.ai-session/telemetry.jsonl"; [ "$(wc -c < "$root/.ai-session/telemetry.jsonl")" -le 1048576 ] || mv -f "$root/.ai-session/telemetry.jsonl" "$root/.ai-session/telemetry.jsonl.1"; } python3 "$root/hooks/scripts/session-state.py" --hook