fix(retrieval): run search as local CLI
CI / linux (push) Successful in 6s

This commit is contained in:
fraujulian committed 2026-09-17 19:27:11 +02:00
1 parent bb5ff39f03
commit 9b312cd840
8 files changed
+90 -111

No files matched your search

-52
View File
@@ -8,30 +8,14 @@ import subprocess
import sys
NAME = 'my-ai-qwen3-retrieval'
FILES = ('server.py', 'requirements.txt')
CUDA_INDEX = 'https://download.pytorch.org/whl/cu126'
TOOL_TIMEOUT_SECONDS = 1800
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def command(client, arguments, home, dry_run):
if dry_run:
print('DRYRUN ' + ' '.join(arguments))
return True
executable = shutil.which(client)
if executable is None:
raise ValueError(f'{client} CLI is required for semantic retrieval.')
result = subprocess.run([executable, *arguments[1:]], stdin=subprocess.DEVNULL, capture_output=True, text=True,
timeout=180, env=dict(os.environ, HOME=str(home), USERPROFILE=str(home), CODEX_HOME=str(home / '.codex'), CLAUDE_CONFIG_DIR=str(home / '.claude')))
if result.returncode:
raise ValueError(f'Semantic retrieval command failed: {client} {" ".join(arguments[1:])}\n{result.stdout}{result.stderr}')
return True
def python_path(target):
return target / ('Scripts/python.exe' if os.name == 'nt' else 'bin/python')
@@ -57,26 +41,6 @@ def ensure_runtime(target, dry_run):
return python
def set_codex_tool_timeout(path, seconds):
lines = path.read_text(encoding='utf-8').splitlines(keepends=True)
section = f'[mcp_servers.{NAME}]'
start = next((number for number, line in enumerate(lines) if line.strip() == section), None)
if start is None:
raise ValueError('Codex semantic retrieval configuration is missing.')
end = next((number for number in range(start + 1, len(lines)) if lines[number].lstrip().startswith('[')), len(lines))
for number in range(start + 1, end):
if lines[number].lstrip().startswith('tool_timeout_sec'):
newline = '\r\n' if lines[number].endswith('\r\n') else '\n'
lines[number] = f'tool_timeout_sec = {seconds}{newline}'
break
else:
while end > start + 1 and not lines[end - 1].strip():
end -= 1
newline = '\r\n' if lines and lines[0].endswith('\r\n') else '\n'
lines.insert(end, f'tool_timeout_sec = {seconds}{newline}')
path.write_text(''.join(lines), encoding='utf-8')
def load_state(path):
if not path.exists():
return {'version': 1, 'clients': [], 'files': {}}
@@ -110,8 +74,6 @@ def sync(root, home, clients, enabled, dry_run, update, summary=False):
if not summary:
print('DRYRUN disable semantic retrieval: ' + (', '.join(owned) if owned else 'nothing owned'))
return
for client in owned:
command(client, [client, 'mcp', 'remove', NAME] + (['--scope', 'user'] if client == 'claude' else []), home, False)
state['clients'] = [client for client in state['clients'] if client not in clients]
if state['clients']:
save_state(state_path, state, False)
@@ -135,23 +97,9 @@ def sync(root, home, clients, enabled, dry_run, update, summary=False):
shutil.copy2(source / name, target / name)
state['files'][name] = digest(target / name)
python = ensure_runtime(target, False)
existing_clients = set(state['clients'])
state['clients'] = list(dict.fromkeys(state['clients']))
save_state(state_path, state, False)
arguments = ['--', str(python), str(target / 'server.py'), '--root', '.', '--data-dir', str(target / 'data'), '--model-cache', str(target / 'model-cache')]
for client in clients:
command_args = [client, 'mcp', 'add']
if client == 'claude':
command_args += [NAME, '--scope', 'user', '-e', f'HF_HOME={target / "model-cache"}']
else:
command_args += ['--env', f'HF_HOME={target / "model-cache"}']
command_args += [NAME]
command_args += arguments
if update and client in existing_clients:
command(client, [client, 'mcp', 'remove', NAME] + (['--scope', 'user'] if client == 'claude' else []), home, False)
command(client, command_args, home, False)
if client == 'codex':
set_codex_tool_timeout(home / '.codex' / 'config.toml', TOOL_TIMEOUT_SECONDS)
if client not in state['clients']:
state['clients'].append(client)
save_state(state_path, state, False)
+10 -1
View File
@@ -27,6 +27,15 @@ def embedding():
class RetrievalServerTests(unittest.TestCase):
def test_runtime_python_uses_adjacent_virtualenv(self):
with TemporaryDirectory() as temporary:
script = Path(temporary) / 'semantic-retrieval' / 'server.py'
runtime = script.parent / '.venv' / 'Scripts' / 'python.exe'
runtime.parent.mkdir(parents=True)
runtime.touch()
self.assertEqual(MODULE.runtime_python(script, 'C:/Python/python.exe', 'nt'), runtime.resolve())
def test_preferred_device_uses_cuda_with_cpu_fallback(self):
with patch.dict('sys.modules', torch=SimpleNamespace(cuda=SimpleNamespace(is_available=lambda: True))):
self.assertEqual(MODULE.preferred_device(), 'cuda')
@@ -89,7 +98,7 @@ class RetrievalServerTests(unittest.TestCase):
encoded.clear()
with ThreadPoolExecutor(max_workers=1) as executor:
self.assertEqual(executor.submit(indexes[0].search, 'query', 5).result()[0]['text'], 'alpha')
self.assertEqual(encoded, ['query'])
self.assertEqual(encoded, [])
timestamp = file.stat()
file.write_text('bravo', encoding='utf-8')
os.utime(file, ns=(timestamp.st_atime_ns, timestamp.st_mtime_ns))
+8 -34
View File
@@ -16,15 +16,13 @@ SPEC.loader.exec_module(MODULE)
class SemanticRetrievalTests(unittest.TestCase):
def test_codex_registration_sets_tool_timeout(self):
def test_sync_does_not_register_mcp(self):
with tempfile.TemporaryDirectory() as directory:
config = Path(directory) / 'config.toml'
config.write_text('[mcp_servers.my-ai-qwen3-retrieval]\ncommand = "python"\n\n[mcp_servers.my-ai-qwen3-retrieval.env]\nHF_HOME = "cache"\n', encoding='utf-8')
MODULE.set_codex_tool_timeout(config, 1800)
self.assertEqual(config.read_text(encoding='utf-8'),
'[mcp_servers.my-ai-qwen3-retrieval]\ncommand = "python"\ntool_timeout_sec = 1800\n\n[mcp_servers.my-ai-qwen3-retrieval.env]\nHF_HOME = "cache"\n')
home = Path(directory) / 'home'
output = io.StringIO()
with redirect_stdout(output):
MODULE.sync(ROOT, home, ['codex', 'claude'], True, True, False)
self.assertNotIn('mcp', output.getvalue().lower())
def test_runtime_prefers_cuda_packages_for_nvidia_gpu(self):
with tempfile.TemporaryDirectory() as directory:
@@ -60,24 +58,7 @@ class SemanticRetrievalTests(unittest.TestCase):
self.assertEqual(output.getvalue(), '')
self.assertFalse(home.exists())
def test_claude_registration_places_name_before_environment(self):
with tempfile.TemporaryDirectory() as directory:
home = Path(directory) / 'home'
calls = []
original = MODULE.command
original_runtime = MODULE.ensure_runtime
MODULE.command = lambda client, arguments, home, dry_run: calls.append((client, arguments))
MODULE.ensure_runtime = lambda target, dry_run: target / '.venv' / 'bin' / 'python'
try:
MODULE.sync(ROOT, home, ['claude'], True, False, False)
finally:
MODULE.command = original
MODULE.ensure_runtime = original_runtime
add = calls[0][1]
self.assertEqual(add[:5], ['claude', 'mcp', 'add', MODULE.NAME, '--scope'])
self.assertIn('-e', add)
def test_disable_removes_owned_runtime_and_registration(self):
def test_disable_removes_owned_runtime_without_cli_registration(self):
with tempfile.TemporaryDirectory() as directory:
home = Path(directory) / 'home'
target = home / '.my-ai-configuration/semantic-retrieval'
@@ -87,14 +68,7 @@ class SemanticRetrievalTests(unittest.TestCase):
state_path = target.parent / 'semantic-retrieval.json'
state_path.write_text(json.dumps({'version': 1, 'clients': ['codex'], 'files': {
name: hashlib.sha256((target / name).read_bytes()).hexdigest() for name in MODULE.FILES}}))
calls = []
original = MODULE.command
MODULE.command = lambda client, arguments, home, dry_run: calls.append((client, arguments))
try:
MODULE.sync(ROOT, home, ['codex'], False, False, False)
finally:
MODULE.command = original
self.assertEqual(calls[0][1], ['codex', 'mcp', 'remove', MODULE.NAME])
MODULE.sync(ROOT, home, ['codex'], False, False, False)
self.assertFalse(target.exists())
self.assertFalse(state_path.exists())