Compare commits

..
10 Commits
24 changed files with 359 additions and 44 deletions

No files matched your search

+2 -2
View File
@@ -209,8 +209,8 @@ $leftoverPlaceholders = @(Get-ChildItem $output -File -Recurse | ForEach-Object
# __AI_CONFIG_ROOT__ is resolved at install time, once the destination is known; it is # __AI_CONFIG_ROOT__ is resolved at install time, once the destination is known; it is
# expected to remain in generated output. -cmatch keeps this case-sensitive so Python # expected to remain in generated output. -cmatch keeps this case-sensitive so Python
# dunder names (__main__, __name__) in flashbang.sh are not mistaken for placeholders. # dunder names (__main__, __name__) in flashbang.sh are not mistaken for placeholders.
$content = (Get-Content -LiteralPath $_.FullName -Raw) -replace '__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__', '' $content = (Get-Content -LiteralPath $_.FullName -Raw) -replace '__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__|__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', ''
if ($content -cmatch '__[A-Z0-9_]+__' -and $content -notmatch '__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__') { $_.FullName } if ($content -cmatch '__[A-Z0-9_]+__') { $_.FullName }
}) })
if ($leftoverPlaceholders.Count) { throw "Unresolved template placeholders in: $($leftoverPlaceholders -join ', ')" } if ($leftoverPlaceholders.Count) { throw "Unresolved template placeholders in: $($leftoverPlaceholders -join ', ')" }
+1 -1
View File
@@ -36,7 +36,7 @@ foreach ($shell in @('powershell','bash')) {
$expectedSkillCount = if ($client -eq 'claude') { $sourceSkillCount + $ruleSkillCount } else { $sourceSkillCount } $expectedSkillCount = if ($client -eq 'claude') { $sourceSkillCount + $ruleSkillCount } else { $sourceSkillCount }
$skillCount = @(Get-ChildItem (Join-Path $package 'skills') -Filter 'SKILL.md' -Recurse -ErrorAction SilentlyContinue).Count $skillCount = @(Get-ChildItem (Join-Path $package 'skills') -Filter 'SKILL.md' -Recurse -ErrorAction SilentlyContinue).Count
if ($skillCount -eq $expectedSkillCount) { Result 'PASS' "$client-$shell skill count matches source ($skillCount)" } else { Result 'FAIL' "$client-$shell skill count $skillCount does not match source ($expectedSkillCount)" } if ($skillCount -eq $expectedSkillCount) { Result 'PASS' "$client-$shell skill count matches source ($skillCount)" } else { Result 'FAIL' "$client-$shell skill count $skillCount does not match source ($expectedSkillCount)" }
$placeholderHits = @(Get-ChildItem $package -File -Recurse | Where-Object { ((Get-Content -LiteralPath $_.FullName -Raw) -replace '__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__', '') -cmatch '__[A-Z0-9_]+__' }) $placeholderHits = @(Get-ChildItem $package -File -Recurse | Where-Object { ((Get-Content -LiteralPath $_.FullName -Raw) -replace '__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__|__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', '') -cmatch '__[A-Z0-9_]+__' })
if ($placeholderHits.Count -eq 0) { Result 'PASS' "$client-$shell has no unresolved placeholders" } else { Result 'FAIL' "$client-$shell has unresolved placeholders: $($placeholderHits.FullName -join ', ')" } if ($placeholderHits.Count -eq 0) { Result 'PASS' "$client-$shell has no unresolved placeholders" } else { Result 'FAIL' "$client-$shell has unresolved placeholders: $($placeholderHits.FullName -join ', ')" }
} }
} }
+1 -1
View File
@@ -49,7 +49,7 @@ for shell in powershell bash; do
[ "$client" != claude ] || expected_skill_count=$((source_skill_count + rule_skill_count)) [ "$client" != claude ] || expected_skill_count=$((source_skill_count + rule_skill_count))
skill_count=$(find "$package/skills" -name SKILL.md 2>/dev/null | wc -l) skill_count=$(find "$package/skills" -name SKILL.md 2>/dev/null | wc -l)
if [ "$skill_count" -eq "$expected_skill_count" ]; then result PASS "$client-$shell skill count matches source ($skill_count)"; else result FAIL "$client-$shell skill count $skill_count does not match source ($expected_skill_count)"; fi if [ "$skill_count" -eq "$expected_skill_count" ]; then result PASS "$client-$shell skill count matches source ($skill_count)"; else result FAIL "$client-$shell skill count $skill_count does not match source ($expected_skill_count)"; fi
leftover_tokens=$(grep -RohE '__[A-Z0-9_]+__' "$package" 2>/dev/null | sort -u | grep -Ev '^(__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__)$' || true) leftover_tokens=$(grep -RohE '__[A-Z0-9_]+__' "$package" 2>/dev/null | sort -u | grep -Ev '^(__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__|__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__)$' || true)
if [ -n "$leftover_tokens" ]; then result FAIL "$client-$shell has unresolved placeholders"; else result PASS "$client-$shell has no unresolved placeholders"; fi if [ -n "$leftover_tokens" ]; then result FAIL "$client-$shell has unresolved placeholders"; else result PASS "$client-$shell has no unresolved placeholders"; fi
done done
done done
+1 -1
View File
@@ -63,7 +63,7 @@ while IFS='|' read -r source destination; do
done < <(get_install_targets "$generated" "$home_path" "$shell" "$client") done < <(get_install_targets "$generated" "$home_path" "$shell" "$client")
sync_configured_plugins "$root" "$client" "$home_path" "$dry_run" false selected_plugins "$summary" sync_configured_plugins "$root" "$client" "$home_path" "$dry_run" false selected_plugins "$summary"
sync_semantic_retrieval "$root" "$client" "$home_path" "$semantic_retrieval" "$dry_run" false sync_semantic_retrieval "$root" "$client" "$home_path" "$semantic_retrieval" "$dry_run" false "$summary"
if [ "$dry_run" = false ]; then save_update_selection; fi if [ "$dry_run" = false ]; then save_update_selection; fi
if [ "$summary" = true ]; then if [ "$summary" = true ]; then
+1 -1
View File
@@ -29,7 +29,7 @@ function Get-InstallDestinations {
} }
function Get-InstallTargets { function Get-InstallTargets {
param([Parameter(Mandatory=$true)][string]$Generated,[Parameter(Mandatory=$true)][string]$HomePath,[Parameter(Mandatory=$true)][string]$Shell,[ValidateSet('Codex','Claude','Both')][Parameter(Mandatory=$true)][string]$Client) param([Parameter(Mandatory=$true)][string]$Generated,[Parameter(Mandatory=$true)][string]$HomePath,[Parameter(Mandatory=$true)][string]$Shell,[ValidateSet('Codex','Claude','Both')][Parameter(Mandatory=$true)][string]$Client)
$targets=@(); if($Client -in @('Codex','Both')){$targets+=@{Source=(Join-Path $Generated "codex-$($Shell.ToLowerInvariant())");Destination=(Join-Path $HomePath '.codex')};$targets+=@{Source=(Join-Path $Generated "codex-$($Shell.ToLowerInvariant())/skills");Destination=(Join-Path $HomePath '.agents/skills')}}; if($Client -in @('Claude','Both')){$targets+=@{Source=(Join-Path $Generated "claude-$($Shell.ToLowerInvariant())");Destination=(Join-Path $HomePath '.claude')};}; return $targets $targets=@(); if($Client -in @('Codex','Both')){$targets+=@{Source=(Join-Path $Generated "codex-$($Shell.ToLowerInvariant())/skills");Destination=(Join-Path $HomePath '.agents/skills')};$targets+=@{Source=(Join-Path $Generated "codex-$($Shell.ToLowerInvariant())");Destination=(Join-Path $HomePath '.codex')}}; if($Client -in @('Claude','Both')){$targets+=@{Source=(Join-Path $Generated "claude-$($Shell.ToLowerInvariant())");Destination=(Join-Path $HomePath '.claude')};}; return $targets
} }
function Test-AnyManifestPresent { param([Parameter(Mandatory=$true)][string[]]$Destinations); return [bool](@($Destinations|Where-Object{Test-Path(Get-ManagedManifestPath $_)}).Count) } function Test-AnyManifestPresent { param([Parameter(Mandatory=$true)][string[]]$Destinations); return [bool](@($Destinations|Where-Object{Test-Path(Get-ManagedManifestPath $_)}).Count) }
function Test-AllManifestsPresent { param([Parameter(Mandatory=$true)][string[]]$Destinations); return [bool](@($Destinations|Where-Object{-not(Test-Path(Get-ManagedManifestPath $_))}).Count -eq 0) } function Test-AllManifestsPresent { param([Parameter(Mandatory=$true)][string[]]$Destinations); return [bool](@($Destinations|Where-Object{-not(Test-Path(Get-ManagedManifestPath $_))}).Count -eq 0) }
+1 -1
View File
@@ -63,8 +63,8 @@ get_install_targets() {
local generated=$1 home_path=$2 shell=$3 client=$4 local generated=$1 home_path=$2 shell=$3 client=$4
case "$client" in case "$client" in
codex|both) codex|both)
printf '%s/codex-%s|%s/.codex\n' "$generated" "$shell" "$home_path"
printf '%s/codex-%s/skills|%s/.agents/skills\n' "$generated" "$shell" "$home_path" printf '%s/codex-%s/skills|%s/.agents/skills\n' "$generated" "$shell" "$home_path"
printf '%s/codex-%s|%s/.codex\n' "$generated" "$shell" "$home_path"
;; ;;
esac esac
case "$client" in claude|both) printf '%s/claude-%s|%s/.claude\n' "$generated" "$shell" "$home_path" ;; esac case "$client" in claude|both) printf '%s/claude-%s|%s/.claude\n' "$generated" "$shell" "$home_path" ;; esac
+2 -1
View File
@@ -70,6 +70,7 @@ function Sync-ManagedDestination {
Get-ChildItem $Source -File -Recurse | ForEach-Object { Get-ChildItem $Source -File -Recurse | ForEach-Object {
$relative = $_.FullName.Substring($Source.Length).TrimStart([char[]]@('\','/')).Replace('\','/') $relative = $_.FullName.Substring($Source.Length).TrimStart([char[]]@('\','/')).Replace('\','/')
if ((Split-Path $Destination -Leaf) -eq '.codex' -and $relative.StartsWith('skills/')) { return }
$target = Join-Path $Destination $relative $target = Join-Path $Destination $relative
$rawContent = Get-Content -LiteralPath $_.FullName -Raw -Encoding UTF8 $rawContent = Get-Content -LiteralPath $_.FullName -Raw -Encoding UTF8
$filterFlashbang = -not $FlashbangEnabled -and $relative -in @('settings.json', 'config.toml') $filterFlashbang = -not $FlashbangEnabled -and $relative -in @('settings.json', 'config.toml')
@@ -77,7 +78,7 @@ function Sync-ManagedDestination {
$rawContent = (& python (Join-Path $PSScriptRoot 'install-options.py') filter --path $_.FullName --flashbang false | Out-String) $rawContent = (& python (Join-Path $PSScriptRoot 'install-options.py') filter --path $_.FullName --flashbang false | Out-String)
if ($LASTEXITCODE -ne 0) { throw 'Could not configure Flashbang.' } if ($LASTEXITCODE -ne 0) { throw 'Could not configure Flashbang.' }
} }
$needsSubstitution = $filterFlashbang -or $rawContent.Contains('__AI_CONFIG_ROOT__') -or $rawContent.Contains('__HOOK_COMMAND__') -or $rawContent.Contains('__POWERSHELL_HOOK_COMMAND__') -or $rawContent.Contains('__POWERSHELL_COMMAND__') $needsSubstitution = $filterFlashbang -or $rawContent.Contains('__AI_CONFIG_ROOT__') -or $rawContent.Contains('__HOOK_COMMAND__') -or $rawContent.Contains('__POWERSHELL_HOOK_COMMAND__') -or $rawContent.Contains('__POWERSHELL_COMMAND__') -or $rawContent.Contains('__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__')
if ($needsSubstitution) { if ($needsSubstitution) {
$finalContent = $rawContent.Replace('__AI_CONFIG_ROOT__', $AiConfigRoot).Replace('__HOOK_COMMAND__', $ShellCommand).Replace('__POWERSHELL_HOOK_COMMAND__', $PowerShellCommand).Replace('__POWERSHELL_COMMAND__', $PowerShellCommand).Replace('__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', $ClaudeConcurrency).Replace('__HOOK_SCRIPT__', 'flashbang.ps1').Replace('__POWERSHELL_HOOK_SCRIPT__', 'flashbang.ps1') $finalContent = $rawContent.Replace('__AI_CONFIG_ROOT__', $AiConfigRoot).Replace('__HOOK_COMMAND__', $ShellCommand).Replace('__POWERSHELL_HOOK_COMMAND__', $PowerShellCommand).Replace('__POWERSHELL_COMMAND__', $PowerShellCommand).Replace('__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', $ClaudeConcurrency).Replace('__HOOK_SCRIPT__', 'flashbang.ps1').Replace('__POWERSHELL_HOOK_SCRIPT__', 'flashbang.ps1')
$newBytes = [System.Text.Encoding]::UTF8.GetBytes($finalContent) $newBytes = [System.Text.Encoding]::UTF8.GetBytes($finalContent)
+2 -1
View File
@@ -64,6 +64,7 @@ sync_managed_destination() {
while IFS= read -r -d '' source_file; do while IFS= read -r -d '' source_file; do
local relative target content needs_sub new_hash exists current_hash action backup local relative target content needs_sub new_hash exists current_hash action backup
relative=${source_file#"$source/"} relative=${source_file#"$source/"}
if [ "${destination##*/}" = .codex ] && [[ "$relative" = skills/* ]]; then continue; fi
target="$destination/$relative" target="$destination/$relative"
content=$(<"$source_file") content=$(<"$source_file")
needs_sub=false needs_sub=false
@@ -71,7 +72,7 @@ sync_managed_destination() {
content=$(python3 "$(dirname -- "${BASH_SOURCE[0]}")/install-options.py" filter --path "$source_file" --flashbang false) || return content=$(python3 "$(dirname -- "${BASH_SOURCE[0]}")/install-options.py" filter --path "$source_file" --flashbang false) || return
needs_sub=true needs_sub=true
fi fi
if [[ "$needs_sub" = true || "$content" == *'__AI_CONFIG_ROOT__'* || "$content" == *'__HOOK_COMMAND__'* || "$content" == *'__POWERSHELL_HOOK_COMMAND__'* || "$content" == *'__POWERSHELL_COMMAND__'* ]]; then if [[ "$needs_sub" = true || "$content" == *'__AI_CONFIG_ROOT__'* || "$content" == *'__HOOK_COMMAND__'* || "$content" == *'__POWERSHELL_HOOK_COMMAND__'* || "$content" == *'__POWERSHELL_COMMAND__'* || "$content" == *'__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__'* ]]; then
needs_sub=true needs_sub=true
content=${content//__AI_CONFIG_ROOT__/$ai_config_root} content=${content//__AI_CONFIG_ROOT__/$ai_config_root}
content=${content//__HOOK_COMMAND__/$shell_command} content=${content//__HOOK_COMMAND__/$shell_command}
+47
View File
@@ -0,0 +1,47 @@
import argparse
from collections import Counter
from pathlib import Path
import re
def inventory(roots):
names = Counter()
count = metadata_bytes = body_bytes = 0
for root in roots:
for path in sorted(root.rglob('SKILL.md')):
content = path.read_text(encoding='utf-8-sig')
count += 1
body_bytes += len(content.encode('utf-8'))
header = re.match(r'\A---\r?\n(.*?)^---\s*$', content, re.MULTILINE | re.DOTALL)
if header:
metadata_bytes += len(header.group(1).encode('utf-8'))
name = re.search(r'^name:\s*(.+)$', header.group(1), re.MULTILINE)
if name:
names[name.group(1).strip().strip('\"\'')] += 1
return count, metadata_bytes, body_bytes, sorted(name for name, total in names.items() if total > 1)
def report(label, roots, warn_duplicates=True):
count, metadata, bodies, duplicates = inventory(roots)
print(f'{label}: {count} skill files, {metadata} frontmatter bytes (~{(metadata + 3) // 4} tokens), {bodies} lazy file bytes')
if duplicates and warn_duplicates:
print(f'WARN {label}: repeated skill names ({len(duplicates)}): {", ".join(duplicates)}')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--home', type=Path, default=Path.home())
args = parser.parse_args()
root = Path(__file__).resolve().parents[2]
for client in ('codex', 'claude'):
report(f'Generated {client} catalog', [root / 'generated' / f'{client}-bash' / 'skills'])
directories = [args.home / f'.{client}' / 'skills']
if client == 'codex':
directories.append(args.home / '.agents' / 'skills')
report(f'Installed {client} user catalog', directories)
report(f'Cached {client} plugins (all versions)', [args.home / f'.{client}' / 'plugins' / 'cache'], warn_duplicates=False)
print('Coverage: baseline covers generated instructions only. Inventory is not measured session context; plugin caches include inactive versions. Client prompts, tool schemas and dynamic hook output are excluded. Token counts use bytes/4 estimates.')
if __name__ == '__main__':
main()
+1
View File
@@ -4,6 +4,7 @@ function Sync-SemanticRetrieval {
if (-not $Enabled -and -not (Test-Path -LiteralPath $statePath)) { return } if (-not $Enabled -and -not (Test-Path -LiteralPath $statePath)) { return }
$arguments = @((Join-Path $PSScriptRoot 'semantic-retrieval.py'), 'sync', '--root', $RepositoryRoot, '--home', $HomePath, '--client', $Client.ToLowerInvariant(), '--enabled', $Enabled.ToString().ToLowerInvariant()) $arguments = @((Join-Path $PSScriptRoot 'semantic-retrieval.py'), 'sync', '--root', $RepositoryRoot, '--home', $HomePath, '--client', $Client.ToLowerInvariant(), '--enabled', $Enabled.ToString().ToLowerInvariant())
if ($DryRun) { $arguments += '--dry-run' }; if ($Update) { $arguments += '--update' } if ($DryRun) { $arguments += '--dry-run' }; if ($Update) { $arguments += '--update' }
if ($Summary) { $arguments += '--summary' }
& python @arguments & python @arguments
if ($LASTEXITCODE -ne 0) { throw 'Semantic retrieval reconciliation failed.' } if ($LASTEXITCODE -ne 0) { throw 'Semantic retrieval reconciliation failed.' }
} }
+5 -2
View File
@@ -67,7 +67,7 @@ def save_state(path, state, dry_run):
path.write_text(json.dumps(state, indent=2) + '\n', encoding='utf-8') path.write_text(json.dumps(state, indent=2) + '\n', encoding='utf-8')
def sync(root, home, clients, enabled, dry_run, update): def sync(root, home, clients, enabled, dry_run, update, summary=False):
base = home / '.my-ai-configuration' base = home / '.my-ai-configuration'
target = base / 'semantic-retrieval' target = base / 'semantic-retrieval'
state_path = base / 'semantic-retrieval.json' state_path = base / 'semantic-retrieval.json'
@@ -78,6 +78,7 @@ def sync(root, home, clients, enabled, dry_run, update):
if not enabled: if not enabled:
owned = [client for client in state['clients'] if client in clients] owned = [client for client in state['clients'] if client in clients]
if dry_run: if dry_run:
if not summary:
print('DRYRUN disable semantic retrieval: ' + (', '.join(owned) if owned else 'nothing owned')) print('DRYRUN disable semantic retrieval: ' + (', '.join(owned) if owned else 'nothing owned'))
return return
for client in owned: for client in owned:
@@ -96,6 +97,7 @@ def sync(root, home, clients, enabled, dry_run, update):
if target.exists() and not state['files']: if target.exists() and not state['files']:
raise ValueError('Semantic retrieval directory exists without setup ownership; refusing to overwrite it.') raise ValueError('Semantic retrieval directory exists without setup ownership; refusing to overwrite it.')
if dry_run: if dry_run:
if not summary:
print('DRYRUN enable semantic retrieval: Qwen3-Embedding-0.6B + Qwen3-Reranker-0.6B') print('DRYRUN enable semantic retrieval: Qwen3-Embedding-0.6B + Qwen3-Reranker-0.6B')
return return
target.mkdir(parents=True, exist_ok=True) target.mkdir(parents=True, exist_ok=True)
@@ -134,9 +136,10 @@ def main():
parser.add_argument('--enabled', choices=('true', 'false'), required=True) parser.add_argument('--enabled', choices=('true', 'false'), required=True)
parser.add_argument('--dry-run', action='store_true') parser.add_argument('--dry-run', action='store_true')
parser.add_argument('--update', action='store_true') parser.add_argument('--update', action='store_true')
parser.add_argument('--summary', action='store_true')
args = parser.parse_args() args = parser.parse_args()
clients = ('codex', 'claude') if args.client == 'both' else (args.client,) clients = ('codex', 'claude') if args.client == 'both' else (args.client,)
sync(args.root.resolve(), args.home.resolve(), clients, args.enabled == 'true', args.dry_run, args.update) sync(args.root.resolve(), args.home.resolve(), clients, args.enabled == 'true', args.dry_run, args.update, args.summary)
print('SEMANTIC RETRIEVAL: reconciliation complete' + (' (dry run)' if args.dry_run else '')) print('SEMANTIC RETRIEVAL: reconciliation complete' + (' (dry run)' if args.dry_run else ''))
+2 -1
View File
@@ -1,8 +1,9 @@
sync_semantic_retrieval() { sync_semantic_retrieval() {
local root=$1 client=$2 home_path=$3 enabled=$4 dry_run=$5 update=$6 local root=$1 client=$2 home_path=$3 enabled=$4 dry_run=$5 update=$6 summary=${7:-false}
[ "$enabled" = true ] || [ -f "$home_path/.my-ai-configuration/semantic-retrieval.json" ] || return 0 [ "$enabled" = true ] || [ -f "$home_path/.my-ai-configuration/semantic-retrieval.json" ] || return 0
local arguments=("$root/scripts/lib/semantic-retrieval.py" sync --root "$root" --home "$home_path" --client "$client" --enabled "$enabled") local arguments=("$root/scripts/lib/semantic-retrieval.py" sync --root "$root" --home "$home_path" --client "$client" --enabled "$enabled")
[ "$dry_run" != true ] || arguments+=(--dry-run) [ "$dry_run" != true ] || arguments+=(--dry-run)
[ "$update" != true ] || arguments+=(--update) [ "$update" != true ] || arguments+=(--update)
[ "$summary" != true ] || arguments+=(--summary)
python3 "${arguments[@]}" python3 "${arguments[@]}"
} }
+2
View File
@@ -30,5 +30,7 @@ if ($old) {
} }
} }
if ($Summary) { Write-Output 'Prompt Budget'; foreach ($row in $rows | Select-Object -First 2) { $limits = $baselineValues[$row.Name]; $base = if($limits){$limits[0]}else{'-'}; $absolute = if($limits){$limits[1]}else{'-'}; Write-Output ('{0,-28} {1,8} bytes baseline {2} relative 115% absolute {3}' -f $row.Name,$row.Bytes,$base,$absolute) }; $rows | Select-Object -Skip 2 | ForEach-Object { Write-Output ('{0,-28} {1,8} bytes {2,6} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } } else { $rows | ForEach-Object { Write-Output ('{0}: {1} bytes / {2} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } } if ($Summary) { Write-Output 'Prompt Budget'; foreach ($row in $rows | Select-Object -First 2) { $limits = $baselineValues[$row.Name]; $base = if($limits){$limits[0]}else{'-'}; $absolute = if($limits){$limits[1]}else{'-'}; Write-Output ('{0,-28} {1,8} bytes baseline {2} relative 115% absolute {3}' -f $row.Name,$row.Bytes,$base,$absolute) }; $rows | Select-Object -Skip 2 | ForEach-Object { Write-Output ('{0,-28} {1,8} bytes {2,6} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } } else { $rows | ForEach-Object { Write-Output ('{0}: {1} bytes / {2} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } }
& python (Join-Path $PSScriptRoot 'lib/prompt-inventory.py') --home ([Environment]::GetFolderPath('UserProfile'))
if ($LASTEXITCODE -ne 0) { throw 'Prompt inventory failed.' }
if ($fail) { exit 1 } if ($fail) { exit 1 }
exit 0 exit 0
+1
View File
@@ -21,5 +21,6 @@ if [ -f "$root/adapters/prompt-budget-baseline.tsv" ]; then
done < "$root/adapters/prompt-budget-baseline.tsv" done < "$root/adapters/prompt-budget-baseline.tsv"
fi fi
if [ "$summary" = true ]; then printf 'Prompt Budget\nClaude permanent context %8s bytes baseline %s relative 115%% absolute %s\nCodex permanent context %8s bytes baseline %s relative 115%% absolute %s\nClaude skill files (lazy) %8s bytes %6s tokens\nRule trigger metadata %8s bytes %6s tokens\nLargest lazy rule %8s bytes %6s tokens\n' "$claude" "$claude_baseline" "$claude_absolute" "$codex" "$codex_baseline" "$codex_absolute" "$skills" "$(tokens "$skills")" "$triggers" "$(tokens "$triggers")" "$lazy" "$(tokens "$lazy")"; else printf 'Prompt budget: Claude %s bytes, Codex %s bytes, skills %s bytes, triggers %s bytes, largest lazy rule %s bytes\n' "$claude" "$codex" "$skills" "$triggers" "$lazy"; fi if [ "$summary" = true ]; then printf 'Prompt Budget\nClaude permanent context %8s bytes baseline %s relative 115%% absolute %s\nCodex permanent context %8s bytes baseline %s relative 115%% absolute %s\nClaude skill files (lazy) %8s bytes %6s tokens\nRule trigger metadata %8s bytes %6s tokens\nLargest lazy rule %8s bytes %6s tokens\n' "$claude" "$claude_baseline" "$claude_absolute" "$codex" "$codex_baseline" "$codex_absolute" "$skills" "$(tokens "$skills")" "$triggers" "$(tokens "$triggers")" "$lazy" "$(tokens "$lazy")"; else printf 'Prompt budget: Claude %s bytes, Codex %s bytes, skills %s bytes, triggers %s bytes, largest lazy rule %s bytes\n' "$claude" "$codex" "$skills" "$triggers" "$lazy"; fi
python3 "$root/scripts/lib/prompt-inventory.py" --home "${HOME:?HOME is required}"
$fail && { printf 'Prompt budget: permanent context increased by more than 15 percent.\n' >&2; exit 1; } $fail && { printf 'Prompt budget: permanent context increased by more than 15 percent.\n' >&2; exit 1; }
exit 0 exit 0
+9
View File
@@ -24,6 +24,9 @@ function Test-BuildFails([string]$Label, [scriptblock]$Mutate) {
if (-not $failed) { throw "Expected build to fail for case '$Label' but it succeeded." } if (-not $failed) { throw "Expected build to fail for case '$Label' but it succeeded." }
} }
$baseline = New-RepoCopy
& (Join-Path $baseline 'scripts/build.ps1') -Summary | Out-Null
Test-BuildFails 'invalid Claude settings.json' { Test-BuildFails 'invalid Claude settings.json' {
param($copy) param($copy)
Add-Content (Join-Path $copy 'adapters/claude/config/settings.json') '} this is not json {' Add-Content (Join-Path $copy 'adapters/claude/config/settings.json') '} this is not json {'
@@ -47,6 +50,12 @@ Test-BuildFails 'leftover template placeholder' {
Add-Content (Join-Path $copy 'shared/global-instructions.md') "`n__NOT_A_REAL_PLACEHOLDER__" Add-Content (Join-Path $copy 'shared/global-instructions.md') "`n__NOT_A_REAL_PLACEHOLDER__"
} }
Test-BuildFails 'unknown placeholder beside an allowed installer placeholder' {
param($copy)
$path = Join-Path $copy 'adapters/claude/config/settings.json'
(Get-Content $path -Raw).Replace('__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', '__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY____UNKNOWN_PLACEHOLDER__') | Set-Content $path
}
Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue
if ($Summary) { Write-Output 'Tests: PASS | build validation' } else { Write-Output 'PASS build validation: invalid JSON, duplicate agent, duplicate plugin, and leftover placeholders are all rejected' } if ($Summary) { Write-Output 'Tests: PASS | build validation' } else { Write-Output 'PASS build validation: invalid JSON, duplicate agent, duplicate plugin, and leftover placeholders are all rejected' }
exit 0 exit 0
+8
View File
@@ -31,6 +31,9 @@ test_build_fails() {
rm -rf -- "$copy" rm -rf -- "$copy"
} }
baseline=$(new_repo_copy)
bash "$baseline/scripts/build.sh" --summary >/dev/null
mutate_invalid_json() { mutate_invalid_json() {
printf '} this is not json {' >> "$1/adapters/claude/config/settings.json" printf '} this is not json {' >> "$1/adapters/claude/config/settings.json"
} }
@@ -52,4 +55,9 @@ mutate_leftover_placeholder() {
} }
test_build_fails 'leftover template placeholder' mutate_leftover_placeholder test_build_fails 'leftover template placeholder' mutate_leftover_placeholder
mutate_mixed_placeholders() {
sed -i 's/__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__/__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__ __UNKNOWN_PLACEHOLDER__/' "$1/adapters/claude/config/settings.json"
}
test_build_fails 'unknown placeholder beside an allowed installer placeholder' mutate_mixed_placeholders
if [ "$summary" = true ]; then printf 'Tests: PASS | build validation\n'; else printf 'PASS build validation: invalid JSON, duplicate agent, duplicate plugin, and leftover placeholders are all rejected\n'; fi if [ "$summary" = true ]; then printf 'Tests: PASS | build validation\n'; else printf 'PASS build validation: invalid JSON, duplicate agent, duplicate plugin, and leftover placeholders are all rejected\n'; fi
+31 -1
View File
@@ -3,6 +3,7 @@ param([switch]$Summary)
$ErrorActionPreference = 'Stop' $ErrorActionPreference = 'Stop'
$root = Split-Path $PSScriptRoot -Parent $root = Split-Path $PSScriptRoot -Parent
. (Join-Path $PSScriptRoot 'lib/manifest.ps1') . (Join-Path $PSScriptRoot 'lib/manifest.ps1')
. (Join-Path $PSScriptRoot 'lib/install-targets.ps1')
$work = Join-Path ([System.IO.Path]::GetTempPath()) ("ai-config-manifest-test-" + [Guid]::NewGuid()) $work = Join-Path ([System.IO.Path]::GetTempPath()) ("ai-config-manifest-test-" + [Guid]::NewGuid())
New-Item $work -ItemType Directory -Force | Out-Null New-Item $work -ItemType Directory -Force | Out-Null
@@ -72,7 +73,36 @@ try {
if (@($outSummary | Where-Object { $_ -like 'CREATE*' -or $_ -like 'UPDATE*' -or $_ -like 'UNCHANGED*' -or $_ -like 'BACKUP*' }).Count -ne 0) { throw "Summary mode must not print per-file lines: $($outSummary -join '; ')" } if (@($outSummary | Where-Object { $_ -like 'CREATE*' -or $_ -like 'UPDATE*' -or $_ -like 'UNCHANGED*' -or $_ -like 'BACKUP*' }).Count -ne 0) { throw "Summary mode must not print per-file lines: $($outSummary -join '; ')" }
if (@($outSummary | Where-Object { $_ -match '^SYNC .* unchanged' }).Count -ne 1) { throw "Summary mode must print exactly one SYNC tally line: $($outSummary -join '; ')" } if (@($outSummary | Where-Object { $_ -match '^SYNC .* unchanged' }).Count -ne 1) { throw "Summary mode must print exactly one SYNC tally line: $($outSummary -join '; ')" }
if ($Summary) { Write-Output 'Tests: PASS | managed install' } else { Write-Output 'PASS managed manifest: idempotent, stale removal, modified-file protection, foreign files preserved, dry run side-effect free' } $generated = Join-Path $work 'generated'
$package = Join-Path $generated 'codex-powershell'
$testHome = Join-Path $work 'home'
$legacy = Join-Path $testHome '.codex'
New-Item (Join-Path $package 'skills/example') -ItemType Directory -Force | Out-Null
Set-Content (Join-Path $package 'skills/example/SKILL.md') 'managed skill' -NoNewline
New-Item (Join-Path $legacy 'skills/example') -ItemType Directory -Force | Out-Null
Copy-Item (Join-Path $package 'skills/example/SKILL.md') (Join-Path $legacy 'skills/example/SKILL.md')
Set-Content (Join-Path $legacy 'skills/example/custom.md') 'customized' -NoNewline
Set-Content (Join-Path $legacy 'skills/example/foreign.md') 'foreign' -NoNewline
Write-ManagedManifest (Get-ManagedManifestPath $legacy) @{
'skills/example/SKILL.md' = (Get-Sha256Hash (Join-Path $legacy 'skills/example/SKILL.md'))
'skills/example/custom.md' = ('0' * 64)
}
$targets = @(Get-InstallTargets -Generated $generated -HomePath $testHome -Shell PowerShell -Client Codex)
if ($targets[0].Destination -ne (Join-Path $testHome '.agents/skills')) { throw 'Canonical skills must be installed before legacy copies are removed.' }
foreach ($item in $targets) { $null = Sync-ManagedDestination -Source $item.Source -Destination $item.Destination -Stamp 'migrate' }
if (-not (Test-Path (Join-Path $testHome '.agents/skills/example/SKILL.md'))) { throw 'Canonical skill is missing.' }
if (Test-Path (Join-Path $legacy 'skills/example/SKILL.md')) { throw 'Unmodified managed skill must not remain duplicated.' }
if (-not (Test-Path (Join-Path $legacy 'backups/migrate/skills/example/SKILL.md'))) { throw 'Removed legacy skill must have a backup.' }
foreach ($name in @('custom.md', 'foreign.md')) {
if (-not (Test-Path (Join-Path $legacy "skills/example/$name"))) { throw 'Modified and foreign legacy skills must be preserved.' }
}
foreach ($item in $targets) { $null = Sync-ManagedDestination -Source $item.Source -Destination $item.Destination -Stamp 'migrate-repeat' }
if (Test-Path (Join-Path $legacy 'backups/migrate-repeat')) { throw 'Skill migration must be idempotent.' }
Set-Content (Join-Path $source 'concurrency.txt') '__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__' -NoNewline
$null = Sync-ManagedDestination -Source $source -Destination $destination -Stamp 'concurrency' -ClaudeConcurrency '7'
if ((Get-Content (Join-Path $destination 'concurrency.txt') -Raw) -ne '7') { throw 'Standalone concurrency placeholder must be resolved.' }
if ($Summary) { Write-Output 'Tests: PASS | managed install' } else { Write-Output 'PASS managed manifest: idempotent, stale removal, modified-file protection, foreign files preserved, dry run side-effect free, skill migration' }
} finally { } finally {
Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue
} }
+30 -1
View File
@@ -10,6 +10,7 @@ done
root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
. "$root/scripts/lib/manifest.sh" . "$root/scripts/lib/manifest.sh"
. "$root/scripts/lib/install-targets.sh"
work=$(mktemp -d) work=$(mktemp -d)
cleanup() { rm -rf -- "$work"; } cleanup() { rm -rf -- "$work"; }
@@ -85,4 +86,32 @@ if [ "$(printf '%s\n' "$out_summary" | grep -c '^SYNC .* unchanged')" -ne 1 ]; t
exit 1 exit 1
fi fi
if [ "$summary" = true ]; then printf 'Tests: PASS | managed install\n'; else printf 'PASS managed manifest: idempotent, stale removal, modified-file protection, foreign files preserved, dry run side-effect free\n'; fi generated="$work/generated"
package="$generated/codex-bash"
test_home="$work/home"
legacy="$test_home/.codex"
mkdir -p "$package/skills/example" "$legacy/skills/example"
printf 'managed skill' > "$package/skills/example/SKILL.md"
cp "$package/skills/example/SKILL.md" "$legacy/skills/example/SKILL.md"
printf 'customized' > "$legacy/skills/example/custom.md"
printf 'foreign' > "$legacy/skills/example/foreign.md"
declare -A legacy_manifest
legacy_manifest[skills/example/SKILL.md]=$(sha256_of_file "$legacy/skills/example/SKILL.md")
legacy_manifest[skills/example/custom.md]=$(printf '0%.0s' {1..64})
write_managed_manifest "$(manifest_path "$legacy")" legacy_manifest
mapfile -t targets < <(get_install_targets "$generated" "$test_home" bash codex)
[[ "${targets[0]}" = *"|$test_home/.agents/skills" ]] || { printf 'Canonical skills must be installed first.\n' >&2; exit 1; }
for target_pair in "${targets[@]}"; do
sync_managed_destination "${target_pair%%|*}" "${target_pair#*|}" migrate "" "" "" false > /dev/null
done
[ -f "$test_home/.agents/skills/example/SKILL.md" ] && [ ! -f "$legacy/skills/example/SKILL.md" ] && [ -f "$legacy/backups/migrate/skills/example/SKILL.md" ] || { printf 'Skill migration must remove and back up the managed duplicate.\n' >&2; exit 1; }
[ -f "$legacy/skills/example/custom.md" ] && [ -f "$legacy/skills/example/foreign.md" ] || { printf 'Skill migration must preserve modified and foreign files.\n' >&2; exit 1; }
for target_pair in "${targets[@]}"; do
sync_managed_destination "${target_pair%%|*}" "${target_pair#*|}" migrate-repeat "" "" "" false > /dev/null
done
[ ! -d "$legacy/backups/migrate-repeat" ] || { printf 'Skill migration must be idempotent.\n' >&2; exit 1; }
printf '__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__' > "$source_dir/concurrency.txt"
sync_managed_destination "$source_dir" "$destination" concurrency "" "" "" false false true 7 > /dev/null
[ "$(cat "$destination/concurrency.txt")" = 7 ] || { printf 'Standalone concurrency placeholder must be resolved.\n' >&2; exit 1; }
if [ "$summary" = true ]; then printf 'Tests: PASS | managed install\n'; else printf 'PASS managed manifest: idempotent, stale removal, modified-file protection, foreign files preserved, dry run side-effect free, skill migration\n'; fi
+29
View File
@@ -0,0 +1,29 @@
import importlib.util
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
SPEC = importlib.util.spec_from_file_location('prompt_inventory', Path(__file__).parent / 'lib/prompt-inventory.py')
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
class PromptInventoryTests(unittest.TestCase):
def test_counts_metadata_separately_and_reports_duplicate_names(self):
with TemporaryDirectory() as temporary:
root = Path(temporary)
metadata = 'name: "example"\ndescription: Example skill.\n'
for name in ('first', 'second'):
directory = root / name
directory.mkdir()
(directory / 'SKILL.md').write_text(f'---\n{metadata}---\n\nBody\n---\nMore body', encoding='utf-8-sig')
count, header_bytes, body_bytes, duplicates = MODULE.inventory([root, root / 'missing'])
self.assertEqual(count, 2)
self.assertEqual(header_bytes, 2 * len(metadata.encode()))
self.assertGreater(body_bytes, header_bytes)
self.assertEqual(duplicates, ['example'])
if __name__ == '__main__':
unittest.main()
+113
View File
@@ -1,6 +1,13 @@
import importlib.util import importlib.util
from array import array
from concurrent.futures import ThreadPoolExecutor
import os
from pathlib import Path from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
import time
import unittest import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parent.parent ROOT = Path(__file__).resolve().parent.parent
@@ -10,6 +17,112 @@ SPEC.loader.exec_module(MODULE)
class RetrievalServerTests(unittest.TestCase): class RetrievalServerTests(unittest.TestCase):
@unittest.skipUnless(os.environ.get('QWEN_TEST_MODEL_CACHE'), 'Set QWEN_TEST_MODEL_CACHE to run the real-model check.')
def test_real_models(self):
os.environ['HF_HOME'] = os.environ['QWEN_TEST_MODEL_CACHE']
with TemporaryDirectory() as temporary:
root = Path(temporary)
(root / 'authentication.md').write_text('Password reset tokens are created by create_reset_token. The reset_password endpoint sends the user a password reset email.', encoding='utf-8')
(root / 'weather.md').write_text('The weather service fetches temperature and rainfall forecasts for a city.', encoding='utf-8')
index = MODULE.Index(root, root / 'data')
try:
for query, expected in [('Where are password reset tokens created?', 'authentication.md'),
('Which service provides rainfall forecasts?', 'weather.md')]:
started = time.monotonic()
results = index.search(query, 2)
self.assertEqual(results[0]['path'], expected)
self.assertEqual(len(results), 2)
self.assertGreater(results[0]['score'], results[1]['score'])
print(f'Qwen real search: {expected}, {time.monotonic() - started:.2f}s, embedding device {index.encoder().device}, reranker device {index.reranker().device}', flush=True)
finally:
index.db.close()
def test_isolation_freshness_and_incremental_encoding(self):
with TemporaryDirectory() as temporary:
root = Path(temporary)
first, second = root / 'first', root / 'second'
first.mkdir()
second.mkdir()
file = first / 'a.md'
file.write_text('alpha', encoding='utf-8')
(second / 'a.md').write_text('other', encoding='utf-8')
indexes = [MODULE.Index(repository, root / 'data') for repository in (first, second)]
encoded = []
def encode(values, **kwargs):
self.assertEqual(kwargs.get('prompt_name'), 'query' if values == ['query'] else None)
encoded.extend(values)
return [array('f', [1.0]) for _ in values]
for index in indexes:
index.encoder = lambda: SimpleNamespace(encode=encode)
index.rerank = lambda query, rows, limit: rows[:limit]
numpy = SimpleNamespace(dot=lambda a, b: sum(x * y for x, y in zip(a, b)),
frombuffer=lambda blob, **kwargs: array('f', blob), float32=None)
try:
with patch.dict('sys.modules', numpy=numpy):
self.assertEqual(indexes[0].search('query', 5)[0]['text'], 'alpha')
self.assertEqual(indexes[1].search('query', 5)[0]['text'], 'other')
encoded.clear()
with ThreadPoolExecutor(max_workers=1) as executor:
self.assertEqual(executor.submit(indexes[0].search, 'query', 5).result()[0]['text'], 'alpha')
self.assertEqual(encoded, ['query'])
timestamp = file.stat()
file.write_text('bravo', encoding='utf-8')
os.utime(file, ns=(timestamp.st_atime_ns, timestamp.st_mtime_ns))
self.assertEqual(indexes[0].search('query', 5)[0]['text'], 'bravo')
file.rename(first / 'renamed.md')
self.assertEqual(indexes[0].search('query', 5)[0]['path'], 'renamed.md')
(first / 'renamed.md').unlink()
encoded.clear()
self.assertEqual(indexes[0].search('query', 5), [])
self.assertEqual(indexes[0].search('query', 5), [])
self.assertEqual(encoded, [])
self.assertEqual(indexes[1].search('query', 5)[0]['text'], 'other')
finally:
for index in indexes:
index.db.close()
def test_failed_refresh_preserves_previous_index(self):
with TemporaryDirectory() as temporary:
root = Path(temporary)
(root / 'a.md').write_text('before', encoding='utf-8')
index = MODULE.Index(root, root / 'data')
index.encoder = lambda: SimpleNamespace(encode=lambda values, **kwargs: [array('f', [1.0]) for _ in values])
try:
index.rebuild()
(root / 'a.md').write_text('after', encoding='utf-8')
(root / 'b.md').write_text('fail', encoding='utf-8')
def encode(values, **kwargs):
if values == ['fail']:
raise RuntimeError('encoder failed')
return [array('f', [1.0]) for _ in values]
index.encoder = lambda: SimpleNamespace(encode=encode)
with self.assertRaisesRegex(RuntimeError, 'encoder failed'):
index.rebuild()
self.assertEqual(index.db.execute('select path, text from chunks').fetchall(), [('a.md', 'before')])
finally:
index.db.close()
def test_scan_prunes_ignored_directories_and_stops_at_limit(self):
with TemporaryDirectory() as temporary:
root = Path(temporary)
index = MODULE.Index(root, root / 'data')
(root / 'visible.md').write_text('visible', encoding='utf-8')
for name in ('node_modules', '.git', 'generated', 'data'):
directory = root / name
directory.mkdir(exist_ok=True)
(directory / 'hidden.md').write_text('hidden', encoding='utf-8')
try:
self.assertEqual(index.files(), [(root / 'visible.md').resolve()])
filenames = ['visible.md'] * 2001
with patch.object(MODULE.os, 'walk', return_value=iter([(str(root), [], filenames)])):
self.assertEqual(len(index.files()), 2000)
finally:
index.db.close()
def test_rerank_uses_cross_encoder_scores(self): def test_rerank_uses_cross_encoder_scores(self):
index = MODULE.Index.__new__(MODULE.Index) index = MODULE.Index.__new__(MODULE.Index)
+12
View File
@@ -1,6 +1,8 @@
import hashlib import hashlib
import importlib.util import importlib.util
import io
import json import json
from contextlib import redirect_stdout
from pathlib import Path from pathlib import Path
import tempfile import tempfile
import unittest import unittest
@@ -13,6 +15,16 @@ SPEC.loader.exec_module(MODULE)
class SemanticRetrievalTests(unittest.TestCase): class SemanticRetrievalTests(unittest.TestCase):
def test_summary_dry_run_omits_details_without_writes(self):
with tempfile.TemporaryDirectory() as directory:
home = Path(directory) / 'home'
for enabled in (True, False):
output = io.StringIO()
with redirect_stdout(output):
MODULE.sync(ROOT, home, ['codex'], enabled, True, False, summary=True)
self.assertEqual(output.getvalue(), '')
self.assertFalse(home.exists())
def test_claude_registration_places_name_before_environment(self): def test_claude_registration_places_name_before_environment(self):
with tempfile.TemporaryDirectory() as directory: with tempfile.TemporaryDirectory() as directory:
home = Path(directory) / 'home' home = Path(directory) / 'home'
+1 -1
View File
@@ -72,7 +72,7 @@ if [ "$quick" = true ]; then
else else
sync_configured_plugins "$root" "$client" "$home_path" "$dry_run" true selected_plugins "$summary" sync_configured_plugins "$root" "$client" "$home_path" "$dry_run" true selected_plugins "$summary"
fi fi
sync_semantic_retrieval "$root" "$client" "$home_path" "$semantic_retrieval" "$dry_run" true sync_semantic_retrieval "$root" "$client" "$home_path" "$semantic_retrieval" "$dry_run" true "$summary"
if [ "$dry_run" = false ]; then save_update_selection; fi if [ "$dry_run" = false ]; then save_update_selection; fi
if [ "$summary" = true ]; then if [ "$summary" = true ]; then
+54 -28
View File
@@ -4,19 +4,22 @@
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import hashlib
import heapq
import json import json
import os import os
from pathlib import Path from pathlib import Path
import sqlite3 import sqlite3
import threading
MODEL = "Qwen/Qwen3-Embedding-0.6B" MODEL = "Qwen/Qwen3-Embedding-0.6B"
RERANKER_MODEL = "Qwen/Qwen3-Reranker-0.6B" RERANKER_MODEL = "Qwen/Qwen3-Reranker-0.6B"
RETRIEVAL_INSTRUCTION = "Given a codebase question, retrieve relevant code and documentation that answer the question."
TEXT_EXTENSIONS = {".c", ".cpp", ".cs", ".go", ".java", ".js", ".json", ".md", ".py", ".ps1", ".rs", ".sh", ".toml", ".ts", ".tsx", ".txt", ".yaml", ".yml"} TEXT_EXTENSIONS = {".c", ".cpp", ".cs", ".go", ".java", ".js", ".json", ".md", ".py", ".ps1", ".rs", ".sh", ".toml", ".ts", ".tsx", ".txt", ".yaml", ".yml"}
def chunks(path: Path): def chunks(text: str):
text = path.read_text(encoding="utf-8", errors="ignore")
for start in range(0, len(text), 1050): for start in range(0, len(text), 1050):
value = text[start:start + 1200].strip() value = text[start:start + 1200].strip()
if value: if value:
@@ -28,33 +31,44 @@ class Index:
self.root = root.resolve() self.root = root.resolve()
self.data = data.resolve() self.data = data.resolve()
self.data.mkdir(parents=True, exist_ok=True) self.data.mkdir(parents=True, exist_ok=True)
self.db = sqlite3.connect(self.data / "index.sqlite3") identity = os.path.normcase(str(self.root)) + "\n" + MODEL
database = hashlib.sha256(identity.encode()).hexdigest() + ".sqlite3"
self.db = sqlite3.connect(self.data / database, check_same_thread=False, timeout=30)
self.db.execute("create table if not exists chunks (path text, text text, vector blob)") self.db.execute("create table if not exists chunks (path text, text text, vector blob)")
self.db.execute("create index if not exists chunks_path on chunks (path)")
self.db.execute("create table if not exists files (path text primary key, digest text)")
self.db.commit() self.db.commit()
self.lock = threading.RLock()
self.model = None self.model = None
self.reranker_model = None self.reranker_model = None
def files(self): def files(self):
ignored = {".git", ".venv", "generated", "node_modules", ".my-ai-configuration"} ignored = {".git", ".venv", "generated", "node_modules", ".my-ai-configuration"}
result = [] result = []
for path in self.root.rglob("*"): for directory, directories, filenames in os.walk(self.root):
if len(result) >= 2000 or not path.is_file() or path.suffix.lower() not in TEXT_EXTENSIONS or path.stat().st_size > 524288: directories[:] = sorted(name for name in directories if name not in ignored
continue and not (Path(directory) / name).is_symlink()
if ignored.intersection(path.relative_to(self.root).parts): and (Path(directory) / name).resolve().is_relative_to(self.root)
and (Path(directory) / name).resolve() != self.data)
for name in sorted(filenames):
path = Path(directory) / name
if path.is_symlink() or path.suffix.lower() not in TEXT_EXTENSIONS or not path.is_file() or path.stat().st_size > 524288:
continue continue
result.append(path) result.append(path)
if len(result) == 2000:
return result
return result return result
def encoder(self): def encoder(self):
if self.model is None: if self.model is None:
from sentence_transformers import SentenceTransformer from sentence_transformers import SentenceTransformer
self.model = SentenceTransformer(MODEL) self.model = SentenceTransformer(MODEL, prompts={"query": f"Instruct: {RETRIEVAL_INSTRUCTION}\nQuery: "})
return self.model return self.model
def reranker(self): def reranker(self):
if self.reranker_model is None: if self.reranker_model is None:
from sentence_transformers import CrossEncoder from sentence_transformers import CrossEncoder
self.reranker_model = CrossEncoder(RERANKER_MODEL) self.reranker_model = CrossEncoder(RERANKER_MODEL, prompts={"query": RETRIEVAL_INSTRUCTION}, default_prompt_name="query")
return self.reranker_model return self.reranker_model
def rerank(self, query: str, rows: list[dict], limit: int): def rerank(self, query: str, rows: list[dict], limit: int):
@@ -65,31 +79,43 @@ class Index:
return sorted(candidates, key=lambda item: item["score"], reverse=True)[:limit] return sorted(candidates, key=lambda item: item["score"], reverse=True)[:limit]
def rebuild(self): def rebuild(self):
encoder = self.encoder() with self.lock:
values = [] self.refresh(force=True)
return self.db.execute("select count(*) from chunks").fetchone()[0]
def refresh(self, force=False):
previous = dict(self.db.execute("select path, digest from files"))
with self.db:
for path in self.files(): for path in self.files():
for value in chunks(path): relative = path.relative_to(self.root).as_posix()
values.append((str(path.relative_to(self.root)), value)) content = path.read_bytes()
self.db.execute("delete from chunks") digest = hashlib.sha256(content).hexdigest()
if values: old_digest = previous.pop(relative, None)
vectors = encoder.encode([value for _, value in values], normalize_embeddings=True) if not force and digest == old_digest:
self.db.executemany("insert into chunks values (?, ?, ?)", ((path, value, vector.tobytes()) for (path, value), vector in zip(values, vectors))) continue
self.db.commit() values = list(chunks(content.decode("utf-8", errors="ignore")))
return len(values) vectors = self.encoder().encode(values, normalize_embeddings=True) if values else []
self.db.execute("delete from chunks where path = ?", (relative,))
self.db.executemany("insert into chunks values (?, ?, ?)",
((relative, value, vector.tobytes()) for value, vector in zip(values, vectors)))
self.db.execute("insert or replace into files values (?, ?)", (relative, digest))
for relative in previous:
self.db.execute("delete from chunks where path = ?", (relative,))
self.db.execute("delete from files where path = ?", (relative,))
def search(self, query: str, top_k: int): def search(self, query: str, top_k: int):
if not query.strip(): if not query.strip():
raise ValueError("query must not be empty") raise ValueError("query must not be empty")
if self.db.execute("select count(*) from chunks").fetchone()[0] == 0: with self.lock:
self.rebuild() self.refresh()
vector = self.encoder().encode([query], normalize_embeddings=True)[0] if not self.db.execute("select 1 from chunks limit 1").fetchone():
rows = [] return []
for path, text, blob in self.db.execute("select path, text, vector from chunks"):
import numpy as np import numpy as np
score = float(np.dot(vector, np.frombuffer(blob, dtype=np.float32))) vector = self.encoder().encode([query], prompt_name="query", normalize_embeddings=True)[0]
rows.append({"path": path, "score": round(score, 6), "text": text}) rows = ({"path": path, "score": float(np.dot(vector, np.frombuffer(blob, dtype=np.float32))), "text": text}
for path, text, blob in self.db.execute("select path, text, vector from chunks"))
limit = max(1, min(top_k, 20)) limit = max(1, min(top_k, 20))
return self.rerank(query, sorted(rows, key=lambda item: item["score"], reverse=True), limit) return self.rerank(query, heapq.nlargest(20, rows, key=lambda item: item["score"]), limit)
def main(): def main():
@@ -107,7 +133,7 @@ def main():
@mcp.tool() @mcp.tool()
def semantic_search(query: str, top_k: int = 5) -> list[dict]: def semantic_search(query: str, top_k: int = 5) -> list[dict]:
"""Find repository passages with Qwen3 embedding and reranking.""" """Search repository concepts, behavior, and implementation patterns first with Qwen3 embedding and reranking. Verify returned passages against current files; use direct reads for known paths."""
return index.search(query, top_k) return index.search(query, top_k)
@mcp.tool() @mcp.tool()
+3 -1
View File
@@ -4,7 +4,9 @@ Apply instructions in this order: direct user instructions; security and data-lo
Preserve security, data, compatibility, and unrelated user work. Ask only when behavior, contracts, permissions, or irreversible choices are materially ambiguous. Otherwise choose the smallest reversible change. Preserve security, data, compatibility, and unrelated user work. Ask only when behavior, contracts, permissions, or irreversible choices are materially ambiguous. Otherwise choose the smallest reversible change.
Use semantic retrieval early when available for unfamiliar code or meaning-based discovery; verify results against current files. Use direct reads and exact search for known paths and symbols. Continue without retrieval if unavailable. Use one applicable workflow per task. Treat skills and plugins as guidance subordinate to these rules. Continue work already authorized by the user; additional design approvals, documentation, commits, or delegation require a task-specific reason and the necessary authorization.
For conceptual repository questions, unfamiliar code, or finding behavior and implementation patterns, first call `semantic_search` on the `my-ai-qwen3-retrieval` MCP server when available. Verify its results against current files. Use direct reads and `rg` for known paths, symbols, and exact strings. If Qwen is unavailable, fails, or returns no useful matches, continue with direct search and briefly report the limitation. Skip retrieval for tasks that do not need repository discovery.
Verify proportionally: run the cheapest check that detects likely failure; broaden for security, data loss, integration, or cross-component risk. Never claim unrun checks passed. Report failures and material gaps. Verify proportionally: run the cheapest check that detects likely failure; broaden for security, data loss, integration, or cross-component risk. Never claim unrun checks passed. Report failures and material gaps.