feat(retrieval): add a background daemon to avoid reloading models per query
Each server.py search/rebuild invocation was a fresh process that reloaded the embedding and reranker models from disk before answering a single query. Add an optional daemon (server.py daemon) that keeps both models resident in memory and serves repeated queries, across repository roots, over a local socket/pipe. search and rebuild now try the daemon first, spawn one on a cache miss (guarded by an exclusive lock file to avoid duplicate spawns), and fall back to the original one-shot in-process worker if the daemon cannot be reached or started, so behavior is unchanged when the daemon is unavailable. The daemon exits on its own after 30 minutes of inactivity. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
208213c1c7
commit
9ad793289d
2 files changed
+164
-14
No files matched your search
@@ -200,6 +200,17 @@ class RetrievalServerTests(unittest.TestCase):
|
||||
finally:
|
||||
index.db.close()
|
||||
|
||||
def test_daemon_address_is_deterministic_and_platform_specific(self):
|
||||
with TemporaryDirectory() as temporary:
|
||||
data_dir = Path(temporary)
|
||||
first = MODULE.daemon_address(data_dir)
|
||||
second = MODULE.daemon_address(data_dir)
|
||||
self.assertEqual(first, second)
|
||||
if os.name == 'nt':
|
||||
self.assertTrue(first.startswith(r'\\.\pipe\ai-config-retrieval-'))
|
||||
else:
|
||||
self.assertTrue(first.startswith(str(data_dir)))
|
||||
|
||||
def test_shared_encoder_and_reranker_are_cached_module_singletons(self):
|
||||
with patch.object(MODULE, '_shared_encoder', None), patch.object(MODULE, '_shared_reranker', None), \
|
||||
patch.object(MODULE, 'preferred_device', lambda: 'cpu'):
|
||||
|
||||
Reference in new issue
Block a user