From cba9a7a2f460cfa6d7f57e87bca5837f652436b6 Mon Sep 17 00:00:00 2001 From: Julian lechner Date: Thu, 8 Oct 2026 11:16:52 +0200 Subject: [PATCH] feat(qmd): add token-protected search daemon with idle exit Co-Authored-By: Claude Opus 5.5 --- scripts/tests/test-qmd-daemon.mjs | 75 ++++++++++++++++ shared/qmd/qmd-daemon.mjs | 138 ++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 scripts/tests/test-qmd-daemon.mjs create mode 100644 shared/qmd/qmd-daemon.mjs diff --git a/scripts/tests/test-qmd-daemon.mjs b/scripts/tests/test-qmd-daemon.mjs new file mode 100644 index 0000000..accbb29 --- /dev/null +++ b/scripts/tests/test-qmd-daemon.mjs @@ -0,0 +1,75 @@ +// scripts/tests/test-qmd-daemon.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer } from '../../shared/qmd/qmd-daemon.mjs'; +import { collectionName } from '../../shared/qmd/qmd-lib.mjs'; + +function fakeStore() { + const calls = []; + return { + calls, + listCollections: async () => [{ name: 'notes', includeByDefault: true }], + getDefaultCollectionNames: async () => ['notes', 'repo-aaaaaaaaaaaa'], + addCollection: async (name, opts) => calls.push(['add', name, opts]), + update: async (opts) => { calls.push(['update', opts]); return { indexed: 1 }; }, + embed: async (opts) => { calls.push(['embed', opts]); return { chunksEmbedded: 1 }; }, + search: async (opts) => { calls.push(['search', opts]); return [{ displayPath: `${collectionName('/r')}/src/a.ts`, body: 'x\nretry here', bestChunkPos: 2, bestChunk: 'retry here', score: 0.91, file: 'qmd://repo/src/a.ts' }]; }, + }; +} + +async function start(overrides = {}) { + const store = fakeStore(); + let clock = 0; + const idle = []; + const daemon = createServer({ + store, token: 't0k', idleMs: 1000, now: () => clock, onIdle: () => idle.push(clock), + extractSnippet: (body) => ({ line: 2, snippet: 'retry here' }), + git: () => 'fingerprint-1', excludeCollection: async () => {}, recordCollection: () => {}, ...overrides, + }); + await new Promise((done) => daemon.server.listen(0, '127.0.0.1', done)); + const port = daemon.server.address().port; + const call = (path, body, token = 't0k') => fetch(`http://127.0.0.1:${port}${path}`, { + method: body === undefined ? 'GET' : 'POST', headers: { 'x-qmd-token': token, 'content-type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body) }); + return { daemon, store, call, setClock: (v) => { clock = v; }, idle, close: () => daemon.server.close() }; +} + +test('rejects requests without the token', async () => { + const s = await start(); + assert.equal((await s.call('/health', undefined, 'wrong')).status, 403); + s.close(); +}); + +test('health identifies the service', async () => { + const s = await start(); + assert.equal((await (await s.call('/health')).json()).service, 'ai-config-qmd'); + s.close(); +}); + +test('search adds the repo collection once, refreshes on change, and filters other repos', async () => { + const s = await start(); + const body = await (await s.call('/search', { repo: '/r', query: 'where are retries', limit: 3 })).json(); + assert.deepEqual(body.results, [{ path: 'src/a.ts', line: 2, score: 0.91, snippet: 'retry here' }]); + const search = s.store.calls.find((c) => c[0] === 'search')[1]; + assert.equal(search.query, 'where are retries'); + assert.ok(search.collections.includes('notes')); + assert.ok(!search.collections.includes('repo-aaaaaaaaaaaa')); + assert.equal(search.collections.filter((n) => n.startsWith('repo-')).length, 1); + assert.equal(s.store.calls.filter((c) => c[0] === 'add').length, 1); + await s.call('/search', { repo: '/r', query: 'again', limit: 3 }); + assert.equal(s.store.calls.filter((c) => c[0] === 'update').length, 1, 'unchanged fingerprint skips update'); + s.close(); +}); + +test('rejects a missing query', async () => { + const s = await start(); + assert.equal((await s.call('/search', { repo: '/r' })).status, 400); + s.close(); +}); + +test('idle check calls onIdle only after the idle window', async () => { + const s = await start(); + s.setClock(500); s.daemon.checkIdle(); assert.deepEqual(s.idle, []); + s.setClock(1500); s.daemon.checkIdle(); assert.deepEqual(s.idle, [1500]); + s.close(); +}); diff --git a/shared/qmd/qmd-daemon.mjs b/shared/qmd/qmd-daemon.mjs new file mode 100644 index 0000000..04958e1 --- /dev/null +++ b/shared/qmd/qmd-daemon.mjs @@ -0,0 +1,138 @@ +// shared/qmd/qmd-daemon.mjs — keeps QMD's models loaded for fast searches; exits when idle. +import { execFileSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { createServer as createHttpServer } from 'node:http'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { MASK, collectionName, idleMinutes, importQmd, qmdConfigPath, qmdDbPath, readJson, runQmdCli, stateDir, writePrivateJson } from './qmd-lib.mjs'; + +const MAX_BODY = 64 * 1024; + +function gitFingerprint(repo) { + try { + return execFileSync('git', ['-C', repo, 'status', '--porcelain=v1', '-uall'], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }) + + execFileSync('git', ['-C', repo, 'rev-parse', 'HEAD'], { encoding: 'utf8', windowsHide: true }); + } catch { + return String(Date.now()); + } +} + +export function createServer({ store, token, idleMs, now = Date.now, onIdle, extractSnippet, git = gitFingerprint, + excludeCollection, recordCollection }) { + let lastRequest = now(); + const known = new Set(); + const fingerprints = new Map(); + + async function refresh(repo) { + const name = collectionName(repo); + if (!known.has(name)) { + const existing = (await store.listCollections()).some((c) => c.name === name); + if (!existing) { + await store.addCollection(name, { path: repo, pattern: MASK }); + await excludeCollection(name); + recordCollection(name); + } + known.add(name); + } + const fingerprint = git(repo); + if (fingerprints.get(name) === fingerprint) return { collection: name, updated: false }; + await store.update({ collections: [name] }); + await store.embed({ collection: name, chunkStrategy: 'auto' }); + fingerprints.set(name, fingerprint); + return { collection: name, updated: true }; + } + + async function search({ repo, query, limit }) { + const { collection } = await refresh(repo); + const defaults = (await store.getDefaultCollectionNames()).filter((n) => !n.startsWith('repo-')); + const results = await store.search({ query, collections: [collection, ...defaults], limit: limit || 5, chunkStrategy: 'auto' }); + return results.map((r) => { + const { line, snippet } = extractSnippet(r.body, query, 300, r.bestChunkPos, r.bestChunk.length); + // displayPath is /: repo hits become repo-relative, others stay addressable as qmd:// URIs. + const path = r.displayPath.startsWith(`${collection}/`) ? r.displayPath.slice(collection.length + 1) : `qmd://${r.displayPath}`; + return { path, line, score: Math.round(r.score * 100) / 100, snippet }; + }); + } + + const server = createHttpServer(async (request, response) => { + const reply = (status, value) => { + response.writeHead(status, { 'content-type': 'application/json' }); + response.end(JSON.stringify(value)); + }; + if (request.headers['x-qmd-token'] !== token) return reply(403, { error: 'forbidden' }); + lastRequest = now(); + try { + if (request.method === 'GET' && request.url === '/health') return reply(200, { status: 'ok', service: 'ai-config-qmd', pid: process.pid }); + let raw = ''; + for await (const chunk of request) { + raw += chunk; + if (raw.length > MAX_BODY) return reply(413, { error: 'body too large' }); + } + const body = raw ? JSON.parse(raw) : {}; + if (request.method === 'POST' && request.url === '/stop') { + reply(200, { stopping: true }); + return onIdle(); + } + if (typeof body.repo !== 'string' || !body.repo) return reply(400, { error: 'repo is required' }); + if (request.method === 'POST' && request.url === '/refresh') return reply(200, await refresh(body.repo)); + if (request.method === 'POST' && request.url === '/search') { + if (typeof body.query !== 'string' || !body.query.trim()) return reply(400, { error: 'query is required' }); + return reply(200, { results: await search(body) }); + } + return reply(404, { error: 'not found' }); + } catch (error) { + return reply(500, { error: String(error?.message || error) }); + } finally { + lastRequest = now(); + } + }); + + const checkIdle = () => { if (now() - lastRequest >= idleMs) onIdle(); }; + return { server, refresh, search, checkIdle }; +} + +async function main() { + const dir = stateDir(); + const daemonFile = join(dir, 'daemon.json'); + const current = readJson(daemonFile, null); + if (current) { + try { + const health = await fetch(`http://127.0.0.1:${current.port}/health`, { headers: { 'x-qmd-token': current.token }, signal: AbortSignal.timeout(2000) }); + if (health.ok && (await health.json()).service === 'ai-config-qmd') return; + } catch { /* stale file: start a new daemon */ } + } + const { createStore, extractSnippet } = await importQmd(); + const store = await createStore({ dbPath: qmdDbPath(), configPath: qmdConfigPath() }); + const collectionsFile = join(dir, 'collections.json'); + const token = randomBytes(32).toString('hex'); + let stopping = false; + const stop = async () => { + if (stopping) return; + stopping = true; + const latest = readJson(daemonFile, null); + if (latest?.pid === process.pid) writePrivateJson(daemonFile, { stopped: true }); + await store.close(); + process.exit(0); + }; + const daemon = createServer({ + store, token, idleMs: idleMinutes() * 60_000, onIdle: stop, extractSnippet, + excludeCollection: async (name) => { + try { runQmdCli(['collection', 'exclude', name], { windowsHide: true, stdio: 'ignore' }); } catch { /* indexing still works without the exclusion */ } + }, + recordCollection: (name) => { + const names = new Set(readJson(collectionsFile, [])); + names.add(name); + writePrivateJson(collectionsFile, [...names].sort()); + }, + }); + daemon.server.listen(0, '127.0.0.1', () => { + writePrivateJson(daemonFile, { port: daemon.server.address().port, token, pid: process.pid }); + }); + setInterval(daemon.checkIdle, 60_000).unref(); + process.on('SIGTERM', stop); + process.on('SIGINT', stop); +} + +if (import.meta.url === pathToFileURL(process.argv[1] || '').href) { + main().catch((error) => { console.error(`qmd-daemon: ${error.message}`); process.exit(1); }); +}