Compare commits
10
Commits
a182b9c5c7
...
459c520201
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
459c520201 | ||
|
|
2079792a74 | ||
|
|
5dbb1b21b4 | ||
|
|
b7842234c0 | ||
|
|
6ce4fcc848 | ||
|
|
5ca0dcfb53 | ||
|
|
561e010c41 | ||
|
|
975e1fd5f0 | ||
|
|
14940cc118 | ||
|
|
55c80ab9de |
No files matched your search
+2
-2
@@ -209,8 +209,8 @@ $leftoverPlaceholders = @(Get-ChildItem $output -File -Recurse | ForEach-Object
|
|||||||
# __AI_CONFIG_ROOT__ is resolved at install time, once the destination is known; it is
|
# __AI_CONFIG_ROOT__ is resolved at install time, once the destination is known; it is
|
||||||
# expected to remain in generated output. -cmatch keeps this case-sensitive so Python
|
# expected to remain in generated output. -cmatch keeps this case-sensitive so Python
|
||||||
# dunder names (__main__, __name__) in flashbang.sh are not mistaken for placeholders.
|
# dunder names (__main__, __name__) in flashbang.sh are not mistaken for placeholders.
|
||||||
$content = (Get-Content -LiteralPath $_.FullName -Raw) -replace '__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__', ''
|
$content = (Get-Content -LiteralPath $_.FullName -Raw) -replace '__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__|__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', ''
|
||||||
if ($content -cmatch '__[A-Z0-9_]+__' -and $content -notmatch '__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__') { $_.FullName }
|
if ($content -cmatch '__[A-Z0-9_]+__') { $_.FullName }
|
||||||
})
|
})
|
||||||
if ($leftoverPlaceholders.Count) { throw "Unresolved template placeholders in: $($leftoverPlaceholders -join ', ')" }
|
if ($leftoverPlaceholders.Count) { throw "Unresolved template placeholders in: $($leftoverPlaceholders -join ', ')" }
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -36,7 +36,7 @@ foreach ($shell in @('powershell','bash')) {
|
|||||||
$expectedSkillCount = if ($client -eq 'claude') { $sourceSkillCount + $ruleSkillCount } else { $sourceSkillCount }
|
$expectedSkillCount = if ($client -eq 'claude') { $sourceSkillCount + $ruleSkillCount } else { $sourceSkillCount }
|
||||||
$skillCount = @(Get-ChildItem (Join-Path $package 'skills') -Filter 'SKILL.md' -Recurse -ErrorAction SilentlyContinue).Count
|
$skillCount = @(Get-ChildItem (Join-Path $package 'skills') -Filter 'SKILL.md' -Recurse -ErrorAction SilentlyContinue).Count
|
||||||
if ($skillCount -eq $expectedSkillCount) { Result 'PASS' "$client-$shell skill count matches source ($skillCount)" } else { Result 'FAIL' "$client-$shell skill count $skillCount does not match source ($expectedSkillCount)" }
|
if ($skillCount -eq $expectedSkillCount) { Result 'PASS' "$client-$shell skill count matches source ($skillCount)" } else { Result 'FAIL' "$client-$shell skill count $skillCount does not match source ($expectedSkillCount)" }
|
||||||
$placeholderHits = @(Get-ChildItem $package -File -Recurse | Where-Object { ((Get-Content -LiteralPath $_.FullName -Raw) -replace '__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__', '') -cmatch '__[A-Z0-9_]+__' })
|
$placeholderHits = @(Get-ChildItem $package -File -Recurse | Where-Object { ((Get-Content -LiteralPath $_.FullName -Raw) -replace '__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__|__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', '') -cmatch '__[A-Z0-9_]+__' })
|
||||||
if ($placeholderHits.Count -eq 0) { Result 'PASS' "$client-$shell has no unresolved placeholders" } else { Result 'FAIL' "$client-$shell has unresolved placeholders: $($placeholderHits.FullName -join ', ')" }
|
if ($placeholderHits.Count -eq 0) { Result 'PASS' "$client-$shell has no unresolved placeholders" } else { Result 'FAIL' "$client-$shell has unresolved placeholders: $($placeholderHits.FullName -join ', ')" }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -49,7 +49,7 @@ for shell in powershell bash; do
|
|||||||
[ "$client" != claude ] || expected_skill_count=$((source_skill_count + rule_skill_count))
|
[ "$client" != claude ] || expected_skill_count=$((source_skill_count + rule_skill_count))
|
||||||
skill_count=$(find "$package/skills" -name SKILL.md 2>/dev/null | wc -l)
|
skill_count=$(find "$package/skills" -name SKILL.md 2>/dev/null | wc -l)
|
||||||
if [ "$skill_count" -eq "$expected_skill_count" ]; then result PASS "$client-$shell skill count matches source ($skill_count)"; else result FAIL "$client-$shell skill count $skill_count does not match source ($expected_skill_count)"; fi
|
if [ "$skill_count" -eq "$expected_skill_count" ]; then result PASS "$client-$shell skill count matches source ($skill_count)"; else result FAIL "$client-$shell skill count $skill_count does not match source ($expected_skill_count)"; fi
|
||||||
leftover_tokens=$(grep -RohE '__[A-Z0-9_]+__' "$package" 2>/dev/null | sort -u | grep -Ev '^(__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__)$' || true)
|
leftover_tokens=$(grep -RohE '__[A-Z0-9_]+__' "$package" 2>/dev/null | sort -u | grep -Ev '^(__AI_CONFIG_ROOT__|__POWERSHELL_COMMAND__|__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__)$' || true)
|
||||||
if [ -n "$leftover_tokens" ]; then result FAIL "$client-$shell has unresolved placeholders"; else result PASS "$client-$shell has no unresolved placeholders"; fi
|
if [ -n "$leftover_tokens" ]; then result FAIL "$client-$shell has unresolved placeholders"; else result PASS "$client-$shell has no unresolved placeholders"; fi
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
|
|||||||
+1
-1
@@ -63,7 +63,7 @@ while IFS='|' read -r source destination; do
|
|||||||
done < <(get_install_targets "$generated" "$home_path" "$shell" "$client")
|
done < <(get_install_targets "$generated" "$home_path" "$shell" "$client")
|
||||||
|
|
||||||
sync_configured_plugins "$root" "$client" "$home_path" "$dry_run" false selected_plugins "$summary"
|
sync_configured_plugins "$root" "$client" "$home_path" "$dry_run" false selected_plugins "$summary"
|
||||||
sync_semantic_retrieval "$root" "$client" "$home_path" "$semantic_retrieval" "$dry_run" false
|
sync_semantic_retrieval "$root" "$client" "$home_path" "$semantic_retrieval" "$dry_run" false "$summary"
|
||||||
if [ "$dry_run" = false ]; then save_update_selection; fi
|
if [ "$dry_run" = false ]; then save_update_selection; fi
|
||||||
|
|
||||||
if [ "$summary" = true ]; then
|
if [ "$summary" = true ]; then
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ function Get-InstallDestinations {
|
|||||||
}
|
}
|
||||||
function Get-InstallTargets {
|
function Get-InstallTargets {
|
||||||
param([Parameter(Mandatory=$true)][string]$Generated,[Parameter(Mandatory=$true)][string]$HomePath,[Parameter(Mandatory=$true)][string]$Shell,[ValidateSet('Codex','Claude','Both')][Parameter(Mandatory=$true)][string]$Client)
|
param([Parameter(Mandatory=$true)][string]$Generated,[Parameter(Mandatory=$true)][string]$HomePath,[Parameter(Mandatory=$true)][string]$Shell,[ValidateSet('Codex','Claude','Both')][Parameter(Mandatory=$true)][string]$Client)
|
||||||
$targets=@(); if($Client -in @('Codex','Both')){$targets+=@{Source=(Join-Path $Generated "codex-$($Shell.ToLowerInvariant())");Destination=(Join-Path $HomePath '.codex')};$targets+=@{Source=(Join-Path $Generated "codex-$($Shell.ToLowerInvariant())/skills");Destination=(Join-Path $HomePath '.agents/skills')}}; if($Client -in @('Claude','Both')){$targets+=@{Source=(Join-Path $Generated "claude-$($Shell.ToLowerInvariant())");Destination=(Join-Path $HomePath '.claude')};}; return $targets
|
$targets=@(); if($Client -in @('Codex','Both')){$targets+=@{Source=(Join-Path $Generated "codex-$($Shell.ToLowerInvariant())/skills");Destination=(Join-Path $HomePath '.agents/skills')};$targets+=@{Source=(Join-Path $Generated "codex-$($Shell.ToLowerInvariant())");Destination=(Join-Path $HomePath '.codex')}}; if($Client -in @('Claude','Both')){$targets+=@{Source=(Join-Path $Generated "claude-$($Shell.ToLowerInvariant())");Destination=(Join-Path $HomePath '.claude')};}; return $targets
|
||||||
}
|
}
|
||||||
function Test-AnyManifestPresent { param([Parameter(Mandatory=$true)][string[]]$Destinations); return [bool](@($Destinations|Where-Object{Test-Path(Get-ManagedManifestPath $_)}).Count) }
|
function Test-AnyManifestPresent { param([Parameter(Mandatory=$true)][string[]]$Destinations); return [bool](@($Destinations|Where-Object{Test-Path(Get-ManagedManifestPath $_)}).Count) }
|
||||||
function Test-AllManifestsPresent { param([Parameter(Mandatory=$true)][string[]]$Destinations); return [bool](@($Destinations|Where-Object{-not(Test-Path(Get-ManagedManifestPath $_))}).Count -eq 0) }
|
function Test-AllManifestsPresent { param([Parameter(Mandatory=$true)][string[]]$Destinations); return [bool](@($Destinations|Where-Object{-not(Test-Path(Get-ManagedManifestPath $_))}).Count -eq 0) }
|
||||||
@@ -63,8 +63,8 @@ get_install_targets() {
|
|||||||
local generated=$1 home_path=$2 shell=$3 client=$4
|
local generated=$1 home_path=$2 shell=$3 client=$4
|
||||||
case "$client" in
|
case "$client" in
|
||||||
codex|both)
|
codex|both)
|
||||||
printf '%s/codex-%s|%s/.codex\n' "$generated" "$shell" "$home_path"
|
|
||||||
printf '%s/codex-%s/skills|%s/.agents/skills\n' "$generated" "$shell" "$home_path"
|
printf '%s/codex-%s/skills|%s/.agents/skills\n' "$generated" "$shell" "$home_path"
|
||||||
|
printf '%s/codex-%s|%s/.codex\n' "$generated" "$shell" "$home_path"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
case "$client" in claude|both) printf '%s/claude-%s|%s/.claude\n' "$generated" "$shell" "$home_path" ;; esac
|
case "$client" in claude|both) printf '%s/claude-%s|%s/.claude\n' "$generated" "$shell" "$home_path" ;; esac
|
||||||
|
|||||||
@@ -70,6 +70,7 @@ function Sync-ManagedDestination {
|
|||||||
|
|
||||||
Get-ChildItem $Source -File -Recurse | ForEach-Object {
|
Get-ChildItem $Source -File -Recurse | ForEach-Object {
|
||||||
$relative = $_.FullName.Substring($Source.Length).TrimStart([char[]]@('\','/')).Replace('\','/')
|
$relative = $_.FullName.Substring($Source.Length).TrimStart([char[]]@('\','/')).Replace('\','/')
|
||||||
|
if ((Split-Path $Destination -Leaf) -eq '.codex' -and $relative.StartsWith('skills/')) { return }
|
||||||
$target = Join-Path $Destination $relative
|
$target = Join-Path $Destination $relative
|
||||||
$rawContent = Get-Content -LiteralPath $_.FullName -Raw -Encoding UTF8
|
$rawContent = Get-Content -LiteralPath $_.FullName -Raw -Encoding UTF8
|
||||||
$filterFlashbang = -not $FlashbangEnabled -and $relative -in @('settings.json', 'config.toml')
|
$filterFlashbang = -not $FlashbangEnabled -and $relative -in @('settings.json', 'config.toml')
|
||||||
@@ -77,7 +78,7 @@ function Sync-ManagedDestination {
|
|||||||
$rawContent = (& python (Join-Path $PSScriptRoot 'install-options.py') filter --path $_.FullName --flashbang false | Out-String)
|
$rawContent = (& python (Join-Path $PSScriptRoot 'install-options.py') filter --path $_.FullName --flashbang false | Out-String)
|
||||||
if ($LASTEXITCODE -ne 0) { throw 'Could not configure Flashbang.' }
|
if ($LASTEXITCODE -ne 0) { throw 'Could not configure Flashbang.' }
|
||||||
}
|
}
|
||||||
$needsSubstitution = $filterFlashbang -or $rawContent.Contains('__AI_CONFIG_ROOT__') -or $rawContent.Contains('__HOOK_COMMAND__') -or $rawContent.Contains('__POWERSHELL_HOOK_COMMAND__') -or $rawContent.Contains('__POWERSHELL_COMMAND__')
|
$needsSubstitution = $filterFlashbang -or $rawContent.Contains('__AI_CONFIG_ROOT__') -or $rawContent.Contains('__HOOK_COMMAND__') -or $rawContent.Contains('__POWERSHELL_HOOK_COMMAND__') -or $rawContent.Contains('__POWERSHELL_COMMAND__') -or $rawContent.Contains('__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__')
|
||||||
if ($needsSubstitution) {
|
if ($needsSubstitution) {
|
||||||
$finalContent = $rawContent.Replace('__AI_CONFIG_ROOT__', $AiConfigRoot).Replace('__HOOK_COMMAND__', $ShellCommand).Replace('__POWERSHELL_HOOK_COMMAND__', $PowerShellCommand).Replace('__POWERSHELL_COMMAND__', $PowerShellCommand).Replace('__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', $ClaudeConcurrency).Replace('__HOOK_SCRIPT__', 'flashbang.ps1').Replace('__POWERSHELL_HOOK_SCRIPT__', 'flashbang.ps1')
|
$finalContent = $rawContent.Replace('__AI_CONFIG_ROOT__', $AiConfigRoot).Replace('__HOOK_COMMAND__', $ShellCommand).Replace('__POWERSHELL_HOOK_COMMAND__', $PowerShellCommand).Replace('__POWERSHELL_COMMAND__', $PowerShellCommand).Replace('__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', $ClaudeConcurrency).Replace('__HOOK_SCRIPT__', 'flashbang.ps1').Replace('__POWERSHELL_HOOK_SCRIPT__', 'flashbang.ps1')
|
||||||
$newBytes = [System.Text.Encoding]::UTF8.GetBytes($finalContent)
|
$newBytes = [System.Text.Encoding]::UTF8.GetBytes($finalContent)
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ sync_managed_destination() {
|
|||||||
while IFS= read -r -d '' source_file; do
|
while IFS= read -r -d '' source_file; do
|
||||||
local relative target content needs_sub new_hash exists current_hash action backup
|
local relative target content needs_sub new_hash exists current_hash action backup
|
||||||
relative=${source_file#"$source/"}
|
relative=${source_file#"$source/"}
|
||||||
|
if [ "${destination##*/}" = .codex ] && [[ "$relative" = skills/* ]]; then continue; fi
|
||||||
target="$destination/$relative"
|
target="$destination/$relative"
|
||||||
content=$(<"$source_file")
|
content=$(<"$source_file")
|
||||||
needs_sub=false
|
needs_sub=false
|
||||||
@@ -71,7 +72,7 @@ sync_managed_destination() {
|
|||||||
content=$(python3 "$(dirname -- "${BASH_SOURCE[0]}")/install-options.py" filter --path "$source_file" --flashbang false) || return
|
content=$(python3 "$(dirname -- "${BASH_SOURCE[0]}")/install-options.py" filter --path "$source_file" --flashbang false) || return
|
||||||
needs_sub=true
|
needs_sub=true
|
||||||
fi
|
fi
|
||||||
if [[ "$needs_sub" = true || "$content" == *'__AI_CONFIG_ROOT__'* || "$content" == *'__HOOK_COMMAND__'* || "$content" == *'__POWERSHELL_HOOK_COMMAND__'* || "$content" == *'__POWERSHELL_COMMAND__'* ]]; then
|
if [[ "$needs_sub" = true || "$content" == *'__AI_CONFIG_ROOT__'* || "$content" == *'__HOOK_COMMAND__'* || "$content" == *'__POWERSHELL_HOOK_COMMAND__'* || "$content" == *'__POWERSHELL_COMMAND__'* || "$content" == *'__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__'* ]]; then
|
||||||
needs_sub=true
|
needs_sub=true
|
||||||
content=${content//__AI_CONFIG_ROOT__/$ai_config_root}
|
content=${content//__AI_CONFIG_ROOT__/$ai_config_root}
|
||||||
content=${content//__HOOK_COMMAND__/$shell_command}
|
content=${content//__HOOK_COMMAND__/$shell_command}
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
def inventory(roots):
|
||||||
|
names = Counter()
|
||||||
|
count = metadata_bytes = body_bytes = 0
|
||||||
|
for root in roots:
|
||||||
|
for path in sorted(root.rglob('SKILL.md')):
|
||||||
|
content = path.read_text(encoding='utf-8-sig')
|
||||||
|
count += 1
|
||||||
|
body_bytes += len(content.encode('utf-8'))
|
||||||
|
header = re.match(r'\A---\r?\n(.*?)^---\s*$', content, re.MULTILINE | re.DOTALL)
|
||||||
|
if header:
|
||||||
|
metadata_bytes += len(header.group(1).encode('utf-8'))
|
||||||
|
name = re.search(r'^name:\s*(.+)$', header.group(1), re.MULTILINE)
|
||||||
|
if name:
|
||||||
|
names[name.group(1).strip().strip('\"\'')] += 1
|
||||||
|
return count, metadata_bytes, body_bytes, sorted(name for name, total in names.items() if total > 1)
|
||||||
|
|
||||||
|
|
||||||
|
def report(label, roots, warn_duplicates=True):
|
||||||
|
count, metadata, bodies, duplicates = inventory(roots)
|
||||||
|
print(f'{label}: {count} skill files, {metadata} frontmatter bytes (~{(metadata + 3) // 4} tokens), {bodies} lazy file bytes')
|
||||||
|
if duplicates and warn_duplicates:
|
||||||
|
print(f'WARN {label}: repeated skill names ({len(duplicates)}): {", ".join(duplicates)}')
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--home', type=Path, default=Path.home())
|
||||||
|
args = parser.parse_args()
|
||||||
|
root = Path(__file__).resolve().parents[2]
|
||||||
|
for client in ('codex', 'claude'):
|
||||||
|
report(f'Generated {client} catalog', [root / 'generated' / f'{client}-bash' / 'skills'])
|
||||||
|
directories = [args.home / f'.{client}' / 'skills']
|
||||||
|
if client == 'codex':
|
||||||
|
directories.append(args.home / '.agents' / 'skills')
|
||||||
|
report(f'Installed {client} user catalog', directories)
|
||||||
|
report(f'Cached {client} plugins (all versions)', [args.home / f'.{client}' / 'plugins' / 'cache'], warn_duplicates=False)
|
||||||
|
print('Coverage: baseline covers generated instructions only. Inventory is not measured session context; plugin caches include inactive versions. Client prompts, tool schemas and dynamic hook output are excluded. Token counts use bytes/4 estimates.')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -4,6 +4,7 @@ function Sync-SemanticRetrieval {
|
|||||||
if (-not $Enabled -and -not (Test-Path -LiteralPath $statePath)) { return }
|
if (-not $Enabled -and -not (Test-Path -LiteralPath $statePath)) { return }
|
||||||
$arguments = @((Join-Path $PSScriptRoot 'semantic-retrieval.py'), 'sync', '--root', $RepositoryRoot, '--home', $HomePath, '--client', $Client.ToLowerInvariant(), '--enabled', $Enabled.ToString().ToLowerInvariant())
|
$arguments = @((Join-Path $PSScriptRoot 'semantic-retrieval.py'), 'sync', '--root', $RepositoryRoot, '--home', $HomePath, '--client', $Client.ToLowerInvariant(), '--enabled', $Enabled.ToString().ToLowerInvariant())
|
||||||
if ($DryRun) { $arguments += '--dry-run' }; if ($Update) { $arguments += '--update' }
|
if ($DryRun) { $arguments += '--dry-run' }; if ($Update) { $arguments += '--update' }
|
||||||
|
if ($Summary) { $arguments += '--summary' }
|
||||||
& python @arguments
|
& python @arguments
|
||||||
if ($LASTEXITCODE -ne 0) { throw 'Semantic retrieval reconciliation failed.' }
|
if ($LASTEXITCODE -ne 0) { throw 'Semantic retrieval reconciliation failed.' }
|
||||||
}
|
}
|
||||||
@@ -67,7 +67,7 @@ def save_state(path, state, dry_run):
|
|||||||
path.write_text(json.dumps(state, indent=2) + '\n', encoding='utf-8')
|
path.write_text(json.dumps(state, indent=2) + '\n', encoding='utf-8')
|
||||||
|
|
||||||
|
|
||||||
def sync(root, home, clients, enabled, dry_run, update):
|
def sync(root, home, clients, enabled, dry_run, update, summary=False):
|
||||||
base = home / '.my-ai-configuration'
|
base = home / '.my-ai-configuration'
|
||||||
target = base / 'semantic-retrieval'
|
target = base / 'semantic-retrieval'
|
||||||
state_path = base / 'semantic-retrieval.json'
|
state_path = base / 'semantic-retrieval.json'
|
||||||
@@ -78,7 +78,8 @@ def sync(root, home, clients, enabled, dry_run, update):
|
|||||||
if not enabled:
|
if not enabled:
|
||||||
owned = [client for client in state['clients'] if client in clients]
|
owned = [client for client in state['clients'] if client in clients]
|
||||||
if dry_run:
|
if dry_run:
|
||||||
print('DRYRUN disable semantic retrieval: ' + (', '.join(owned) if owned else 'nothing owned'))
|
if not summary:
|
||||||
|
print('DRYRUN disable semantic retrieval: ' + (', '.join(owned) if owned else 'nothing owned'))
|
||||||
return
|
return
|
||||||
for client in owned:
|
for client in owned:
|
||||||
command(client, [client, 'mcp', 'remove', NAME] + (['--scope', 'user'] if client == 'claude' else []), home, False)
|
command(client, [client, 'mcp', 'remove', NAME] + (['--scope', 'user'] if client == 'claude' else []), home, False)
|
||||||
@@ -96,7 +97,8 @@ def sync(root, home, clients, enabled, dry_run, update):
|
|||||||
if target.exists() and not state['files']:
|
if target.exists() and not state['files']:
|
||||||
raise ValueError('Semantic retrieval directory exists without setup ownership; refusing to overwrite it.')
|
raise ValueError('Semantic retrieval directory exists without setup ownership; refusing to overwrite it.')
|
||||||
if dry_run:
|
if dry_run:
|
||||||
print('DRYRUN enable semantic retrieval: Qwen3-Embedding-0.6B + Qwen3-Reranker-0.6B')
|
if not summary:
|
||||||
|
print('DRYRUN enable semantic retrieval: Qwen3-Embedding-0.6B + Qwen3-Reranker-0.6B')
|
||||||
return
|
return
|
||||||
target.mkdir(parents=True, exist_ok=True)
|
target.mkdir(parents=True, exist_ok=True)
|
||||||
source = root / 'shared' / 'retrieval'
|
source = root / 'shared' / 'retrieval'
|
||||||
@@ -134,9 +136,10 @@ def main():
|
|||||||
parser.add_argument('--enabled', choices=('true', 'false'), required=True)
|
parser.add_argument('--enabled', choices=('true', 'false'), required=True)
|
||||||
parser.add_argument('--dry-run', action='store_true')
|
parser.add_argument('--dry-run', action='store_true')
|
||||||
parser.add_argument('--update', action='store_true')
|
parser.add_argument('--update', action='store_true')
|
||||||
|
parser.add_argument('--summary', action='store_true')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
clients = ('codex', 'claude') if args.client == 'both' else (args.client,)
|
clients = ('codex', 'claude') if args.client == 'both' else (args.client,)
|
||||||
sync(args.root.resolve(), args.home.resolve(), clients, args.enabled == 'true', args.dry_run, args.update)
|
sync(args.root.resolve(), args.home.resolve(), clients, args.enabled == 'true', args.dry_run, args.update, args.summary)
|
||||||
print('SEMANTIC RETRIEVAL: reconciliation complete' + (' (dry run)' if args.dry_run else ''))
|
print('SEMANTIC RETRIEVAL: reconciliation complete' + (' (dry run)' if args.dry_run else ''))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
sync_semantic_retrieval() {
|
sync_semantic_retrieval() {
|
||||||
local root=$1 client=$2 home_path=$3 enabled=$4 dry_run=$5 update=$6
|
local root=$1 client=$2 home_path=$3 enabled=$4 dry_run=$5 update=$6 summary=${7:-false}
|
||||||
[ "$enabled" = true ] || [ -f "$home_path/.my-ai-configuration/semantic-retrieval.json" ] || return 0
|
[ "$enabled" = true ] || [ -f "$home_path/.my-ai-configuration/semantic-retrieval.json" ] || return 0
|
||||||
local arguments=("$root/scripts/lib/semantic-retrieval.py" sync --root "$root" --home "$home_path" --client "$client" --enabled "$enabled")
|
local arguments=("$root/scripts/lib/semantic-retrieval.py" sync --root "$root" --home "$home_path" --client "$client" --enabled "$enabled")
|
||||||
[ "$dry_run" != true ] || arguments+=(--dry-run)
|
[ "$dry_run" != true ] || arguments+=(--dry-run)
|
||||||
[ "$update" != true ] || arguments+=(--update)
|
[ "$update" != true ] || arguments+=(--update)
|
||||||
|
[ "$summary" != true ] || arguments+=(--summary)
|
||||||
python3 "${arguments[@]}"
|
python3 "${arguments[@]}"
|
||||||
}
|
}
|
||||||
@@ -30,5 +30,7 @@ if ($old) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ($Summary) { Write-Output 'Prompt Budget'; foreach ($row in $rows | Select-Object -First 2) { $limits = $baselineValues[$row.Name]; $base = if($limits){$limits[0]}else{'-'}; $absolute = if($limits){$limits[1]}else{'-'}; Write-Output ('{0,-28} {1,8} bytes baseline {2} relative 115% absolute {3}' -f $row.Name,$row.Bytes,$base,$absolute) }; $rows | Select-Object -Skip 2 | ForEach-Object { Write-Output ('{0,-28} {1,8} bytes {2,6} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } } else { $rows | ForEach-Object { Write-Output ('{0}: {1} bytes / {2} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } }
|
if ($Summary) { Write-Output 'Prompt Budget'; foreach ($row in $rows | Select-Object -First 2) { $limits = $baselineValues[$row.Name]; $base = if($limits){$limits[0]}else{'-'}; $absolute = if($limits){$limits[1]}else{'-'}; Write-Output ('{0,-28} {1,8} bytes baseline {2} relative 115% absolute {3}' -f $row.Name,$row.Bytes,$base,$absolute) }; $rows | Select-Object -Skip 2 | ForEach-Object { Write-Output ('{0,-28} {1,8} bytes {2,6} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } } else { $rows | ForEach-Object { Write-Output ('{0}: {1} bytes / {2} tokens' -f $_.Name,$_.Bytes,$_.Tokens) } }
|
||||||
|
& python (Join-Path $PSScriptRoot 'lib/prompt-inventory.py') --home ([Environment]::GetFolderPath('UserProfile'))
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw 'Prompt inventory failed.' }
|
||||||
if ($fail) { exit 1 }
|
if ($fail) { exit 1 }
|
||||||
exit 0
|
exit 0
|
||||||
@@ -21,5 +21,6 @@ if [ -f "$root/adapters/prompt-budget-baseline.tsv" ]; then
|
|||||||
done < "$root/adapters/prompt-budget-baseline.tsv"
|
done < "$root/adapters/prompt-budget-baseline.tsv"
|
||||||
fi
|
fi
|
||||||
if [ "$summary" = true ]; then printf 'Prompt Budget\nClaude permanent context %8s bytes baseline %s relative 115%% absolute %s\nCodex permanent context %8s bytes baseline %s relative 115%% absolute %s\nClaude skill files (lazy) %8s bytes %6s tokens\nRule trigger metadata %8s bytes %6s tokens\nLargest lazy rule %8s bytes %6s tokens\n' "$claude" "$claude_baseline" "$claude_absolute" "$codex" "$codex_baseline" "$codex_absolute" "$skills" "$(tokens "$skills")" "$triggers" "$(tokens "$triggers")" "$lazy" "$(tokens "$lazy")"; else printf 'Prompt budget: Claude %s bytes, Codex %s bytes, skills %s bytes, triggers %s bytes, largest lazy rule %s bytes\n' "$claude" "$codex" "$skills" "$triggers" "$lazy"; fi
|
if [ "$summary" = true ]; then printf 'Prompt Budget\nClaude permanent context %8s bytes baseline %s relative 115%% absolute %s\nCodex permanent context %8s bytes baseline %s relative 115%% absolute %s\nClaude skill files (lazy) %8s bytes %6s tokens\nRule trigger metadata %8s bytes %6s tokens\nLargest lazy rule %8s bytes %6s tokens\n' "$claude" "$claude_baseline" "$claude_absolute" "$codex" "$codex_baseline" "$codex_absolute" "$skills" "$(tokens "$skills")" "$triggers" "$(tokens "$triggers")" "$lazy" "$(tokens "$lazy")"; else printf 'Prompt budget: Claude %s bytes, Codex %s bytes, skills %s bytes, triggers %s bytes, largest lazy rule %s bytes\n' "$claude" "$codex" "$skills" "$triggers" "$lazy"; fi
|
||||||
|
python3 "$root/scripts/lib/prompt-inventory.py" --home "${HOME:?HOME is required}"
|
||||||
$fail && { printf 'Prompt budget: permanent context increased by more than 15 percent.\n' >&2; exit 1; }
|
$fail && { printf 'Prompt budget: permanent context increased by more than 15 percent.\n' >&2; exit 1; }
|
||||||
exit 0
|
exit 0
|
||||||
@@ -24,6 +24,9 @@ function Test-BuildFails([string]$Label, [scriptblock]$Mutate) {
|
|||||||
if (-not $failed) { throw "Expected build to fail for case '$Label' but it succeeded." }
|
if (-not $failed) { throw "Expected build to fail for case '$Label' but it succeeded." }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$baseline = New-RepoCopy
|
||||||
|
& (Join-Path $baseline 'scripts/build.ps1') -Summary | Out-Null
|
||||||
|
|
||||||
Test-BuildFails 'invalid Claude settings.json' {
|
Test-BuildFails 'invalid Claude settings.json' {
|
||||||
param($copy)
|
param($copy)
|
||||||
Add-Content (Join-Path $copy 'adapters/claude/config/settings.json') '} this is not json {'
|
Add-Content (Join-Path $copy 'adapters/claude/config/settings.json') '} this is not json {'
|
||||||
@@ -47,6 +50,12 @@ Test-BuildFails 'leftover template placeholder' {
|
|||||||
Add-Content (Join-Path $copy 'shared/global-instructions.md') "`n__NOT_A_REAL_PLACEHOLDER__"
|
Add-Content (Join-Path $copy 'shared/global-instructions.md') "`n__NOT_A_REAL_PLACEHOLDER__"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Test-BuildFails 'unknown placeholder beside an allowed installer placeholder' {
|
||||||
|
param($copy)
|
||||||
|
$path = Join-Path $copy 'adapters/claude/config/settings.json'
|
||||||
|
(Get-Content $path -Raw).Replace('__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__', '__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY____UNKNOWN_PLACEHOLDER__') | Set-Content $path
|
||||||
|
}
|
||||||
|
|
||||||
Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue
|
Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue
|
||||||
if ($Summary) { Write-Output 'Tests: PASS | build validation' } else { Write-Output 'PASS build validation: invalid JSON, duplicate agent, duplicate plugin, and leftover placeholders are all rejected' }
|
if ($Summary) { Write-Output 'Tests: PASS | build validation' } else { Write-Output 'PASS build validation: invalid JSON, duplicate agent, duplicate plugin, and leftover placeholders are all rejected' }
|
||||||
exit 0
|
exit 0
|
||||||
@@ -31,6 +31,9 @@ test_build_fails() {
|
|||||||
rm -rf -- "$copy"
|
rm -rf -- "$copy"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
baseline=$(new_repo_copy)
|
||||||
|
bash "$baseline/scripts/build.sh" --summary >/dev/null
|
||||||
|
|
||||||
mutate_invalid_json() {
|
mutate_invalid_json() {
|
||||||
printf '} this is not json {' >> "$1/adapters/claude/config/settings.json"
|
printf '} this is not json {' >> "$1/adapters/claude/config/settings.json"
|
||||||
}
|
}
|
||||||
@@ -52,4 +55,9 @@ mutate_leftover_placeholder() {
|
|||||||
}
|
}
|
||||||
test_build_fails 'leftover template placeholder' mutate_leftover_placeholder
|
test_build_fails 'leftover template placeholder' mutate_leftover_placeholder
|
||||||
|
|
||||||
|
mutate_mixed_placeholders() {
|
||||||
|
sed -i 's/__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__/__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__ __UNKNOWN_PLACEHOLDER__/' "$1/adapters/claude/config/settings.json"
|
||||||
|
}
|
||||||
|
test_build_fails 'unknown placeholder beside an allowed installer placeholder' mutate_mixed_placeholders
|
||||||
|
|
||||||
if [ "$summary" = true ]; then printf 'Tests: PASS | build validation\n'; else printf 'PASS build validation: invalid JSON, duplicate agent, duplicate plugin, and leftover placeholders are all rejected\n'; fi
|
if [ "$summary" = true ]; then printf 'Tests: PASS | build validation\n'; else printf 'PASS build validation: invalid JSON, duplicate agent, duplicate plugin, and leftover placeholders are all rejected\n'; fi
|
||||||
@@ -3,6 +3,7 @@ param([switch]$Summary)
|
|||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
$root = Split-Path $PSScriptRoot -Parent
|
$root = Split-Path $PSScriptRoot -Parent
|
||||||
. (Join-Path $PSScriptRoot 'lib/manifest.ps1')
|
. (Join-Path $PSScriptRoot 'lib/manifest.ps1')
|
||||||
|
. (Join-Path $PSScriptRoot 'lib/install-targets.ps1')
|
||||||
|
|
||||||
$work = Join-Path ([System.IO.Path]::GetTempPath()) ("ai-config-manifest-test-" + [Guid]::NewGuid())
|
$work = Join-Path ([System.IO.Path]::GetTempPath()) ("ai-config-manifest-test-" + [Guid]::NewGuid())
|
||||||
New-Item $work -ItemType Directory -Force | Out-Null
|
New-Item $work -ItemType Directory -Force | Out-Null
|
||||||
@@ -72,7 +73,36 @@ try {
|
|||||||
if (@($outSummary | Where-Object { $_ -like 'CREATE*' -or $_ -like 'UPDATE*' -or $_ -like 'UNCHANGED*' -or $_ -like 'BACKUP*' }).Count -ne 0) { throw "Summary mode must not print per-file lines: $($outSummary -join '; ')" }
|
if (@($outSummary | Where-Object { $_ -like 'CREATE*' -or $_ -like 'UPDATE*' -or $_ -like 'UNCHANGED*' -or $_ -like 'BACKUP*' }).Count -ne 0) { throw "Summary mode must not print per-file lines: $($outSummary -join '; ')" }
|
||||||
if (@($outSummary | Where-Object { $_ -match '^SYNC .* unchanged' }).Count -ne 1) { throw "Summary mode must print exactly one SYNC tally line: $($outSummary -join '; ')" }
|
if (@($outSummary | Where-Object { $_ -match '^SYNC .* unchanged' }).Count -ne 1) { throw "Summary mode must print exactly one SYNC tally line: $($outSummary -join '; ')" }
|
||||||
|
|
||||||
if ($Summary) { Write-Output 'Tests: PASS | managed install' } else { Write-Output 'PASS managed manifest: idempotent, stale removal, modified-file protection, foreign files preserved, dry run side-effect free' }
|
$generated = Join-Path $work 'generated'
|
||||||
|
$package = Join-Path $generated 'codex-powershell'
|
||||||
|
$testHome = Join-Path $work 'home'
|
||||||
|
$legacy = Join-Path $testHome '.codex'
|
||||||
|
New-Item (Join-Path $package 'skills/example') -ItemType Directory -Force | Out-Null
|
||||||
|
Set-Content (Join-Path $package 'skills/example/SKILL.md') 'managed skill' -NoNewline
|
||||||
|
New-Item (Join-Path $legacy 'skills/example') -ItemType Directory -Force | Out-Null
|
||||||
|
Copy-Item (Join-Path $package 'skills/example/SKILL.md') (Join-Path $legacy 'skills/example/SKILL.md')
|
||||||
|
Set-Content (Join-Path $legacy 'skills/example/custom.md') 'customized' -NoNewline
|
||||||
|
Set-Content (Join-Path $legacy 'skills/example/foreign.md') 'foreign' -NoNewline
|
||||||
|
Write-ManagedManifest (Get-ManagedManifestPath $legacy) @{
|
||||||
|
'skills/example/SKILL.md' = (Get-Sha256Hash (Join-Path $legacy 'skills/example/SKILL.md'))
|
||||||
|
'skills/example/custom.md' = ('0' * 64)
|
||||||
|
}
|
||||||
|
$targets = @(Get-InstallTargets -Generated $generated -HomePath $testHome -Shell PowerShell -Client Codex)
|
||||||
|
if ($targets[0].Destination -ne (Join-Path $testHome '.agents/skills')) { throw 'Canonical skills must be installed before legacy copies are removed.' }
|
||||||
|
foreach ($item in $targets) { $null = Sync-ManagedDestination -Source $item.Source -Destination $item.Destination -Stamp 'migrate' }
|
||||||
|
if (-not (Test-Path (Join-Path $testHome '.agents/skills/example/SKILL.md'))) { throw 'Canonical skill is missing.' }
|
||||||
|
if (Test-Path (Join-Path $legacy 'skills/example/SKILL.md')) { throw 'Unmodified managed skill must not remain duplicated.' }
|
||||||
|
if (-not (Test-Path (Join-Path $legacy 'backups/migrate/skills/example/SKILL.md'))) { throw 'Removed legacy skill must have a backup.' }
|
||||||
|
foreach ($name in @('custom.md', 'foreign.md')) {
|
||||||
|
if (-not (Test-Path (Join-Path $legacy "skills/example/$name"))) { throw 'Modified and foreign legacy skills must be preserved.' }
|
||||||
|
}
|
||||||
|
foreach ($item in $targets) { $null = Sync-ManagedDestination -Source $item.Source -Destination $item.Destination -Stamp 'migrate-repeat' }
|
||||||
|
if (Test-Path (Join-Path $legacy 'backups/migrate-repeat')) { throw 'Skill migration must be idempotent.' }
|
||||||
|
Set-Content (Join-Path $source 'concurrency.txt') '__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__' -NoNewline
|
||||||
|
$null = Sync-ManagedDestination -Source $source -Destination $destination -Stamp 'concurrency' -ClaudeConcurrency '7'
|
||||||
|
if ((Get-Content (Join-Path $destination 'concurrency.txt') -Raw) -ne '7') { throw 'Standalone concurrency placeholder must be resolved.' }
|
||||||
|
|
||||||
|
if ($Summary) { Write-Output 'Tests: PASS | managed install' } else { Write-Output 'PASS managed manifest: idempotent, stale removal, modified-file protection, foreign files preserved, dry run side-effect free, skill migration' }
|
||||||
} finally {
|
} finally {
|
||||||
Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue
|
Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ done
|
|||||||
|
|
||||||
root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
|
root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)
|
||||||
. "$root/scripts/lib/manifest.sh"
|
. "$root/scripts/lib/manifest.sh"
|
||||||
|
. "$root/scripts/lib/install-targets.sh"
|
||||||
|
|
||||||
work=$(mktemp -d)
|
work=$(mktemp -d)
|
||||||
cleanup() { rm -rf -- "$work"; }
|
cleanup() { rm -rf -- "$work"; }
|
||||||
@@ -85,4 +86,32 @@ if [ "$(printf '%s\n' "$out_summary" | grep -c '^SYNC .* unchanged')" -ne 1 ]; t
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$summary" = true ]; then printf 'Tests: PASS | managed install\n'; else printf 'PASS managed manifest: idempotent, stale removal, modified-file protection, foreign files preserved, dry run side-effect free\n'; fi
|
generated="$work/generated"
|
||||||
|
package="$generated/codex-bash"
|
||||||
|
test_home="$work/home"
|
||||||
|
legacy="$test_home/.codex"
|
||||||
|
mkdir -p "$package/skills/example" "$legacy/skills/example"
|
||||||
|
printf 'managed skill' > "$package/skills/example/SKILL.md"
|
||||||
|
cp "$package/skills/example/SKILL.md" "$legacy/skills/example/SKILL.md"
|
||||||
|
printf 'customized' > "$legacy/skills/example/custom.md"
|
||||||
|
printf 'foreign' > "$legacy/skills/example/foreign.md"
|
||||||
|
declare -A legacy_manifest
|
||||||
|
legacy_manifest[skills/example/SKILL.md]=$(sha256_of_file "$legacy/skills/example/SKILL.md")
|
||||||
|
legacy_manifest[skills/example/custom.md]=$(printf '0%.0s' {1..64})
|
||||||
|
write_managed_manifest "$(manifest_path "$legacy")" legacy_manifest
|
||||||
|
mapfile -t targets < <(get_install_targets "$generated" "$test_home" bash codex)
|
||||||
|
[[ "${targets[0]}" = *"|$test_home/.agents/skills" ]] || { printf 'Canonical skills must be installed first.\n' >&2; exit 1; }
|
||||||
|
for target_pair in "${targets[@]}"; do
|
||||||
|
sync_managed_destination "${target_pair%%|*}" "${target_pair#*|}" migrate "" "" "" false > /dev/null
|
||||||
|
done
|
||||||
|
[ -f "$test_home/.agents/skills/example/SKILL.md" ] && [ ! -f "$legacy/skills/example/SKILL.md" ] && [ -f "$legacy/backups/migrate/skills/example/SKILL.md" ] || { printf 'Skill migration must remove and back up the managed duplicate.\n' >&2; exit 1; }
|
||||||
|
[ -f "$legacy/skills/example/custom.md" ] && [ -f "$legacy/skills/example/foreign.md" ] || { printf 'Skill migration must preserve modified and foreign files.\n' >&2; exit 1; }
|
||||||
|
for target_pair in "${targets[@]}"; do
|
||||||
|
sync_managed_destination "${target_pair%%|*}" "${target_pair#*|}" migrate-repeat "" "" "" false > /dev/null
|
||||||
|
done
|
||||||
|
[ ! -d "$legacy/backups/migrate-repeat" ] || { printf 'Skill migration must be idempotent.\n' >&2; exit 1; }
|
||||||
|
printf '__CLAUDE_CODE_MAX_TOOL_USE_CONCURRENCY__' > "$source_dir/concurrency.txt"
|
||||||
|
sync_managed_destination "$source_dir" "$destination" concurrency "" "" "" false false true 7 > /dev/null
|
||||||
|
[ "$(cat "$destination/concurrency.txt")" = 7 ] || { printf 'Standalone concurrency placeholder must be resolved.\n' >&2; exit 1; }
|
||||||
|
|
||||||
|
if [ "$summary" = true ]; then printf 'Tests: PASS | managed install\n'; else printf 'PASS managed manifest: idempotent, stale removal, modified-file protection, foreign files preserved, dry run side-effect free, skill migration\n'; fi
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
SPEC = importlib.util.spec_from_file_location('prompt_inventory', Path(__file__).parent / 'lib/prompt-inventory.py')
|
||||||
|
MODULE = importlib.util.module_from_spec(SPEC)
|
||||||
|
SPEC.loader.exec_module(MODULE)
|
||||||
|
|
||||||
|
|
||||||
|
class PromptInventoryTests(unittest.TestCase):
|
||||||
|
def test_counts_metadata_separately_and_reports_duplicate_names(self):
|
||||||
|
with TemporaryDirectory() as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
metadata = 'name: "example"\ndescription: Example skill.\n'
|
||||||
|
for name in ('first', 'second'):
|
||||||
|
directory = root / name
|
||||||
|
directory.mkdir()
|
||||||
|
(directory / 'SKILL.md').write_text(f'---\n{metadata}---\n\nBody\n---\nMore body', encoding='utf-8-sig')
|
||||||
|
count, header_bytes, body_bytes, duplicates = MODULE.inventory([root, root / 'missing'])
|
||||||
|
self.assertEqual(count, 2)
|
||||||
|
self.assertEqual(header_bytes, 2 * len(metadata.encode()))
|
||||||
|
self.assertGreater(body_bytes, header_bytes)
|
||||||
|
self.assertEqual(duplicates, ['example'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@@ -1,6 +1,13 @@
|
|||||||
import importlib.util
|
import importlib.util
|
||||||
|
from array import array
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parent.parent
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
@@ -10,6 +17,112 @@ SPEC.loader.exec_module(MODULE)
|
|||||||
|
|
||||||
|
|
||||||
class RetrievalServerTests(unittest.TestCase):
|
class RetrievalServerTests(unittest.TestCase):
|
||||||
|
@unittest.skipUnless(os.environ.get('QWEN_TEST_MODEL_CACHE'), 'Set QWEN_TEST_MODEL_CACHE to run the real-model check.')
|
||||||
|
def test_real_models(self):
|
||||||
|
os.environ['HF_HOME'] = os.environ['QWEN_TEST_MODEL_CACHE']
|
||||||
|
with TemporaryDirectory() as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
(root / 'authentication.md').write_text('Password reset tokens are created by create_reset_token. The reset_password endpoint sends the user a password reset email.', encoding='utf-8')
|
||||||
|
(root / 'weather.md').write_text('The weather service fetches temperature and rainfall forecasts for a city.', encoding='utf-8')
|
||||||
|
index = MODULE.Index(root, root / 'data')
|
||||||
|
try:
|
||||||
|
for query, expected in [('Where are password reset tokens created?', 'authentication.md'),
|
||||||
|
('Which service provides rainfall forecasts?', 'weather.md')]:
|
||||||
|
started = time.monotonic()
|
||||||
|
results = index.search(query, 2)
|
||||||
|
self.assertEqual(results[0]['path'], expected)
|
||||||
|
self.assertEqual(len(results), 2)
|
||||||
|
self.assertGreater(results[0]['score'], results[1]['score'])
|
||||||
|
print(f'Qwen real search: {expected}, {time.monotonic() - started:.2f}s, embedding device {index.encoder().device}, reranker device {index.reranker().device}', flush=True)
|
||||||
|
finally:
|
||||||
|
index.db.close()
|
||||||
|
|
||||||
|
def test_isolation_freshness_and_incremental_encoding(self):
|
||||||
|
with TemporaryDirectory() as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
first, second = root / 'first', root / 'second'
|
||||||
|
first.mkdir()
|
||||||
|
second.mkdir()
|
||||||
|
file = first / 'a.md'
|
||||||
|
file.write_text('alpha', encoding='utf-8')
|
||||||
|
(second / 'a.md').write_text('other', encoding='utf-8')
|
||||||
|
indexes = [MODULE.Index(repository, root / 'data') for repository in (first, second)]
|
||||||
|
encoded = []
|
||||||
|
|
||||||
|
def encode(values, **kwargs):
|
||||||
|
self.assertEqual(kwargs.get('prompt_name'), 'query' if values == ['query'] else None)
|
||||||
|
encoded.extend(values)
|
||||||
|
return [array('f', [1.0]) for _ in values]
|
||||||
|
|
||||||
|
for index in indexes:
|
||||||
|
index.encoder = lambda: SimpleNamespace(encode=encode)
|
||||||
|
index.rerank = lambda query, rows, limit: rows[:limit]
|
||||||
|
numpy = SimpleNamespace(dot=lambda a, b: sum(x * y for x, y in zip(a, b)),
|
||||||
|
frombuffer=lambda blob, **kwargs: array('f', blob), float32=None)
|
||||||
|
try:
|
||||||
|
with patch.dict('sys.modules', numpy=numpy):
|
||||||
|
self.assertEqual(indexes[0].search('query', 5)[0]['text'], 'alpha')
|
||||||
|
self.assertEqual(indexes[1].search('query', 5)[0]['text'], 'other')
|
||||||
|
encoded.clear()
|
||||||
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||||
|
self.assertEqual(executor.submit(indexes[0].search, 'query', 5).result()[0]['text'], 'alpha')
|
||||||
|
self.assertEqual(encoded, ['query'])
|
||||||
|
timestamp = file.stat()
|
||||||
|
file.write_text('bravo', encoding='utf-8')
|
||||||
|
os.utime(file, ns=(timestamp.st_atime_ns, timestamp.st_mtime_ns))
|
||||||
|
self.assertEqual(indexes[0].search('query', 5)[0]['text'], 'bravo')
|
||||||
|
file.rename(first / 'renamed.md')
|
||||||
|
self.assertEqual(indexes[0].search('query', 5)[0]['path'], 'renamed.md')
|
||||||
|
(first / 'renamed.md').unlink()
|
||||||
|
encoded.clear()
|
||||||
|
self.assertEqual(indexes[0].search('query', 5), [])
|
||||||
|
self.assertEqual(indexes[0].search('query', 5), [])
|
||||||
|
self.assertEqual(encoded, [])
|
||||||
|
self.assertEqual(indexes[1].search('query', 5)[0]['text'], 'other')
|
||||||
|
finally:
|
||||||
|
for index in indexes:
|
||||||
|
index.db.close()
|
||||||
|
|
||||||
|
def test_failed_refresh_preserves_previous_index(self):
|
||||||
|
with TemporaryDirectory() as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
(root / 'a.md').write_text('before', encoding='utf-8')
|
||||||
|
index = MODULE.Index(root, root / 'data')
|
||||||
|
index.encoder = lambda: SimpleNamespace(encode=lambda values, **kwargs: [array('f', [1.0]) for _ in values])
|
||||||
|
try:
|
||||||
|
index.rebuild()
|
||||||
|
(root / 'a.md').write_text('after', encoding='utf-8')
|
||||||
|
(root / 'b.md').write_text('fail', encoding='utf-8')
|
||||||
|
|
||||||
|
def encode(values, **kwargs):
|
||||||
|
if values == ['fail']:
|
||||||
|
raise RuntimeError('encoder failed')
|
||||||
|
return [array('f', [1.0]) for _ in values]
|
||||||
|
|
||||||
|
index.encoder = lambda: SimpleNamespace(encode=encode)
|
||||||
|
with self.assertRaisesRegex(RuntimeError, 'encoder failed'):
|
||||||
|
index.rebuild()
|
||||||
|
self.assertEqual(index.db.execute('select path, text from chunks').fetchall(), [('a.md', 'before')])
|
||||||
|
finally:
|
||||||
|
index.db.close()
|
||||||
|
|
||||||
|
def test_scan_prunes_ignored_directories_and_stops_at_limit(self):
|
||||||
|
with TemporaryDirectory() as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
index = MODULE.Index(root, root / 'data')
|
||||||
|
(root / 'visible.md').write_text('visible', encoding='utf-8')
|
||||||
|
for name in ('node_modules', '.git', 'generated', 'data'):
|
||||||
|
directory = root / name
|
||||||
|
directory.mkdir(exist_ok=True)
|
||||||
|
(directory / 'hidden.md').write_text('hidden', encoding='utf-8')
|
||||||
|
try:
|
||||||
|
self.assertEqual(index.files(), [(root / 'visible.md').resolve()])
|
||||||
|
filenames = ['visible.md'] * 2001
|
||||||
|
with patch.object(MODULE.os, 'walk', return_value=iter([(str(root), [], filenames)])):
|
||||||
|
self.assertEqual(len(index.files()), 2000)
|
||||||
|
finally:
|
||||||
|
index.db.close()
|
||||||
|
|
||||||
def test_rerank_uses_cross_encoder_scores(self):
|
def test_rerank_uses_cross_encoder_scores(self):
|
||||||
index = MODULE.Index.__new__(MODULE.Index)
|
index = MODULE.Index.__new__(MODULE.Index)
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import io
|
||||||
import json
|
import json
|
||||||
|
from contextlib import redirect_stdout
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
@@ -13,6 +15,16 @@ SPEC.loader.exec_module(MODULE)
|
|||||||
|
|
||||||
|
|
||||||
class SemanticRetrievalTests(unittest.TestCase):
|
class SemanticRetrievalTests(unittest.TestCase):
|
||||||
|
def test_summary_dry_run_omits_details_without_writes(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
home = Path(directory) / 'home'
|
||||||
|
for enabled in (True, False):
|
||||||
|
output = io.StringIO()
|
||||||
|
with redirect_stdout(output):
|
||||||
|
MODULE.sync(ROOT, home, ['codex'], enabled, True, False, summary=True)
|
||||||
|
self.assertEqual(output.getvalue(), '')
|
||||||
|
self.assertFalse(home.exists())
|
||||||
|
|
||||||
def test_claude_registration_places_name_before_environment(self):
|
def test_claude_registration_places_name_before_environment(self):
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
home = Path(directory) / 'home'
|
home = Path(directory) / 'home'
|
||||||
|
|||||||
+1
-1
@@ -72,7 +72,7 @@ if [ "$quick" = true ]; then
|
|||||||
else
|
else
|
||||||
sync_configured_plugins "$root" "$client" "$home_path" "$dry_run" true selected_plugins "$summary"
|
sync_configured_plugins "$root" "$client" "$home_path" "$dry_run" true selected_plugins "$summary"
|
||||||
fi
|
fi
|
||||||
sync_semantic_retrieval "$root" "$client" "$home_path" "$semantic_retrieval" "$dry_run" true
|
sync_semantic_retrieval "$root" "$client" "$home_path" "$semantic_retrieval" "$dry_run" true "$summary"
|
||||||
if [ "$dry_run" = false ]; then save_update_selection; fi
|
if [ "$dry_run" = false ]; then save_update_selection; fi
|
||||||
|
|
||||||
if [ "$summary" = true ]; then
|
if [ "$summary" = true ]; then
|
||||||
|
|||||||
+58
-32
@@ -4,19 +4,22 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import heapq
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
|
||||||
|
|
||||||
MODEL = "Qwen/Qwen3-Embedding-0.6B"
|
MODEL = "Qwen/Qwen3-Embedding-0.6B"
|
||||||
RERANKER_MODEL = "Qwen/Qwen3-Reranker-0.6B"
|
RERANKER_MODEL = "Qwen/Qwen3-Reranker-0.6B"
|
||||||
|
RETRIEVAL_INSTRUCTION = "Given a codebase question, retrieve relevant code and documentation that answer the question."
|
||||||
TEXT_EXTENSIONS = {".c", ".cpp", ".cs", ".go", ".java", ".js", ".json", ".md", ".py", ".ps1", ".rs", ".sh", ".toml", ".ts", ".tsx", ".txt", ".yaml", ".yml"}
|
TEXT_EXTENSIONS = {".c", ".cpp", ".cs", ".go", ".java", ".js", ".json", ".md", ".py", ".ps1", ".rs", ".sh", ".toml", ".ts", ".tsx", ".txt", ".yaml", ".yml"}
|
||||||
|
|
||||||
|
|
||||||
def chunks(path: Path):
|
def chunks(text: str):
|
||||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
|
||||||
for start in range(0, len(text), 1050):
|
for start in range(0, len(text), 1050):
|
||||||
value = text[start:start + 1200].strip()
|
value = text[start:start + 1200].strip()
|
||||||
if value:
|
if value:
|
||||||
@@ -28,33 +31,44 @@ class Index:
|
|||||||
self.root = root.resolve()
|
self.root = root.resolve()
|
||||||
self.data = data.resolve()
|
self.data = data.resolve()
|
||||||
self.data.mkdir(parents=True, exist_ok=True)
|
self.data.mkdir(parents=True, exist_ok=True)
|
||||||
self.db = sqlite3.connect(self.data / "index.sqlite3")
|
identity = os.path.normcase(str(self.root)) + "\n" + MODEL
|
||||||
|
database = hashlib.sha256(identity.encode()).hexdigest() + ".sqlite3"
|
||||||
|
self.db = sqlite3.connect(self.data / database, check_same_thread=False, timeout=30)
|
||||||
self.db.execute("create table if not exists chunks (path text, text text, vector blob)")
|
self.db.execute("create table if not exists chunks (path text, text text, vector blob)")
|
||||||
|
self.db.execute("create index if not exists chunks_path on chunks (path)")
|
||||||
|
self.db.execute("create table if not exists files (path text primary key, digest text)")
|
||||||
self.db.commit()
|
self.db.commit()
|
||||||
|
self.lock = threading.RLock()
|
||||||
self.model = None
|
self.model = None
|
||||||
self.reranker_model = None
|
self.reranker_model = None
|
||||||
|
|
||||||
def files(self):
|
def files(self):
|
||||||
ignored = {".git", ".venv", "generated", "node_modules", ".my-ai-configuration"}
|
ignored = {".git", ".venv", "generated", "node_modules", ".my-ai-configuration"}
|
||||||
result = []
|
result = []
|
||||||
for path in self.root.rglob("*"):
|
for directory, directories, filenames in os.walk(self.root):
|
||||||
if len(result) >= 2000 or not path.is_file() or path.suffix.lower() not in TEXT_EXTENSIONS or path.stat().st_size > 524288:
|
directories[:] = sorted(name for name in directories if name not in ignored
|
||||||
continue
|
and not (Path(directory) / name).is_symlink()
|
||||||
if ignored.intersection(path.relative_to(self.root).parts):
|
and (Path(directory) / name).resolve().is_relative_to(self.root)
|
||||||
continue
|
and (Path(directory) / name).resolve() != self.data)
|
||||||
result.append(path)
|
for name in sorted(filenames):
|
||||||
|
path = Path(directory) / name
|
||||||
|
if path.is_symlink() or path.suffix.lower() not in TEXT_EXTENSIONS or not path.is_file() or path.stat().st_size > 524288:
|
||||||
|
continue
|
||||||
|
result.append(path)
|
||||||
|
if len(result) == 2000:
|
||||||
|
return result
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def encoder(self):
|
def encoder(self):
|
||||||
if self.model is None:
|
if self.model is None:
|
||||||
from sentence_transformers import SentenceTransformer
|
from sentence_transformers import SentenceTransformer
|
||||||
self.model = SentenceTransformer(MODEL)
|
self.model = SentenceTransformer(MODEL, prompts={"query": f"Instruct: {RETRIEVAL_INSTRUCTION}\nQuery: "})
|
||||||
return self.model
|
return self.model
|
||||||
|
|
||||||
def reranker(self):
|
def reranker(self):
|
||||||
if self.reranker_model is None:
|
if self.reranker_model is None:
|
||||||
from sentence_transformers import CrossEncoder
|
from sentence_transformers import CrossEncoder
|
||||||
self.reranker_model = CrossEncoder(RERANKER_MODEL)
|
self.reranker_model = CrossEncoder(RERANKER_MODEL, prompts={"query": RETRIEVAL_INSTRUCTION}, default_prompt_name="query")
|
||||||
return self.reranker_model
|
return self.reranker_model
|
||||||
|
|
||||||
def rerank(self, query: str, rows: list[dict], limit: int):
|
def rerank(self, query: str, rows: list[dict], limit: int):
|
||||||
@@ -65,31 +79,43 @@ class Index:
|
|||||||
return sorted(candidates, key=lambda item: item["score"], reverse=True)[:limit]
|
return sorted(candidates, key=lambda item: item["score"], reverse=True)[:limit]
|
||||||
|
|
||||||
def rebuild(self):
|
def rebuild(self):
|
||||||
encoder = self.encoder()
|
with self.lock:
|
||||||
values = []
|
self.refresh(force=True)
|
||||||
for path in self.files():
|
return self.db.execute("select count(*) from chunks").fetchone()[0]
|
||||||
for value in chunks(path):
|
|
||||||
values.append((str(path.relative_to(self.root)), value))
|
def refresh(self, force=False):
|
||||||
self.db.execute("delete from chunks")
|
previous = dict(self.db.execute("select path, digest from files"))
|
||||||
if values:
|
with self.db:
|
||||||
vectors = encoder.encode([value for _, value in values], normalize_embeddings=True)
|
for path in self.files():
|
||||||
self.db.executemany("insert into chunks values (?, ?, ?)", ((path, value, vector.tobytes()) for (path, value), vector in zip(values, vectors)))
|
relative = path.relative_to(self.root).as_posix()
|
||||||
self.db.commit()
|
content = path.read_bytes()
|
||||||
return len(values)
|
digest = hashlib.sha256(content).hexdigest()
|
||||||
|
old_digest = previous.pop(relative, None)
|
||||||
|
if not force and digest == old_digest:
|
||||||
|
continue
|
||||||
|
values = list(chunks(content.decode("utf-8", errors="ignore")))
|
||||||
|
vectors = self.encoder().encode(values, normalize_embeddings=True) if values else []
|
||||||
|
self.db.execute("delete from chunks where path = ?", (relative,))
|
||||||
|
self.db.executemany("insert into chunks values (?, ?, ?)",
|
||||||
|
((relative, value, vector.tobytes()) for value, vector in zip(values, vectors)))
|
||||||
|
self.db.execute("insert or replace into files values (?, ?)", (relative, digest))
|
||||||
|
for relative in previous:
|
||||||
|
self.db.execute("delete from chunks where path = ?", (relative,))
|
||||||
|
self.db.execute("delete from files where path = ?", (relative,))
|
||||||
|
|
||||||
def search(self, query: str, top_k: int):
|
def search(self, query: str, top_k: int):
|
||||||
if not query.strip():
|
if not query.strip():
|
||||||
raise ValueError("query must not be empty")
|
raise ValueError("query must not be empty")
|
||||||
if self.db.execute("select count(*) from chunks").fetchone()[0] == 0:
|
with self.lock:
|
||||||
self.rebuild()
|
self.refresh()
|
||||||
vector = self.encoder().encode([query], normalize_embeddings=True)[0]
|
if not self.db.execute("select 1 from chunks limit 1").fetchone():
|
||||||
rows = []
|
return []
|
||||||
for path, text, blob in self.db.execute("select path, text, vector from chunks"):
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
score = float(np.dot(vector, np.frombuffer(blob, dtype=np.float32)))
|
vector = self.encoder().encode([query], prompt_name="query", normalize_embeddings=True)[0]
|
||||||
rows.append({"path": path, "score": round(score, 6), "text": text})
|
rows = ({"path": path, "score": float(np.dot(vector, np.frombuffer(blob, dtype=np.float32))), "text": text}
|
||||||
limit = max(1, min(top_k, 20))
|
for path, text, blob in self.db.execute("select path, text, vector from chunks"))
|
||||||
return self.rerank(query, sorted(rows, key=lambda item: item["score"], reverse=True), limit)
|
limit = max(1, min(top_k, 20))
|
||||||
|
return self.rerank(query, heapq.nlargest(20, rows, key=lambda item: item["score"]), limit)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
@@ -107,7 +133,7 @@ def main():
|
|||||||
|
|
||||||
@mcp.tool()
|
@mcp.tool()
|
||||||
def semantic_search(query: str, top_k: int = 5) -> list[dict]:
|
def semantic_search(query: str, top_k: int = 5) -> list[dict]:
|
||||||
"""Find repository passages with Qwen3 embedding and reranking."""
|
"""Search repository concepts, behavior, and implementation patterns first with Qwen3 embedding and reranking. Verify returned passages against current files; use direct reads for known paths."""
|
||||||
return index.search(query, top_k)
|
return index.search(query, top_k)
|
||||||
|
|
||||||
@mcp.tool()
|
@mcp.tool()
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ Apply instructions in this order: direct user instructions; security and data-lo
|
|||||||
|
|
||||||
Preserve security, data, compatibility, and unrelated user work. Ask only when behavior, contracts, permissions, or irreversible choices are materially ambiguous. Otherwise choose the smallest reversible change.
|
Preserve security, data, compatibility, and unrelated user work. Ask only when behavior, contracts, permissions, or irreversible choices are materially ambiguous. Otherwise choose the smallest reversible change.
|
||||||
|
|
||||||
Use semantic retrieval early when available for unfamiliar code or meaning-based discovery; verify results against current files. Use direct reads and exact search for known paths and symbols. Continue without retrieval if unavailable.
|
Use one applicable workflow per task. Treat skills and plugins as guidance subordinate to these rules. Continue work already authorized by the user; additional design approvals, documentation, commits, or delegation require a task-specific reason and the necessary authorization.
|
||||||
|
|
||||||
|
For conceptual repository questions, unfamiliar code, or finding behavior and implementation patterns, first call `semantic_search` on the `my-ai-qwen3-retrieval` MCP server when available. Verify its results against current files. Use direct reads and `rg` for known paths, symbols, and exact strings. If Qwen is unavailable, fails, or returns no useful matches, continue with direct search and briefly report the limitation. Skip retrieval for tasks that do not need repository discovery.
|
||||||
|
|
||||||
Verify proportionally: run the cheapest check that detects likely failure; broaden for security, data loss, integration, or cross-component risk. Never claim unrun checks passed. Report failures and material gaps.
|
Verify proportionally: run the cheapest check that detects likely failure; broaden for security, data loss, integration, or cross-component risk. Never claim unrun checks passed. Report failures and material gaps.
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user