diff --git a/scripts/tests/test-qmd-daemon.mjs b/scripts/tests/test-qmd-daemon.mjs index 52366c8..e2f8df1 100644 --- a/scripts/tests/test-qmd-daemon.mjs +++ b/scripts/tests/test-qmd-daemon.mjs @@ -5,7 +5,7 @@ import { execFileSync } from 'node:child_process'; import { mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { createServer, gitFingerprint, keepModelsLoaded } from '../../shared/qmd/qmd-daemon.mjs'; +import { createServer, gitFingerprint, ignoreGlobs, keepModelsLoaded } from '../../shared/qmd/qmd-daemon.mjs'; import { collectionName } from '../../shared/qmd/qmd-lib.mjs'; const defaultHits = () => [{ displayPath: `${collectionName('/r')}/src/a.ts`, body: 'x\nretry here', bestChunkPos: 2, bestChunk: 'retry here', score: 0.91 }]; @@ -30,7 +30,8 @@ async function start(t, overrides = {}, { updateGate, hits } = {}) { const daemon = createServer({ store, token: 't0k', idleMs: 1000, now: () => clock, onIdle: () => idle.push(clock), extractSnippet: () => ({ line: 2, snippet: 'retry here' }), - git: () => 'fingerprint-1', excludeCollection: async () => {}, recordCollection: () => {}, ...overrides, + git: () => 'fingerprint-1', excludeCollection: async () => {}, recordCollection: () => {}, + ignored: () => [], persistIgnore: async () => {}, ...overrides, }); await new Promise((done) => daemon.server.listen(0, '127.0.0.1', done)); t.after(() => { daemon.server.closeAllConnections(); daemon.server.close(); }); @@ -174,3 +175,29 @@ test('keepModelsLoaded disables QMD unloading models after 5 idle minutes', () = keepModelsLoaded({ internal: { llm } }); assert.deepEqual(llm, { inactivityTimeoutMs: 0, disposeModelsOnInactivity: false }); }); + +test('ignoreGlobs turns git ignored paths into escaped QMD ignore globs', () => { + const escape = (path) => path.replace(/[()[\]]/g, '\\$&'); + assert.deepEqual(ignoreGlobs('dist/\0a b.ts\0out (1)/\0x[1].md\0', escape), ['dist/**', 'a b.ts', 'out \\(1\\)/**', 'x\\[1\\].md']); + assert.deepEqual(ignoreGlobs('', escape), []); +}); + +test('git ignored paths reach addCollection and are persisted only when they change', async (t) => { + let ignore = ['dist/**']; + let fingerprint = 'one'; + const persisted = []; + const s = await start(t, { ignored: () => ignore, git: () => fingerprint, persistIgnore: async (name, list) => persisted.push([name, list]) }); + await s.call('/search', { repo: '/r', query: 'q' }); + assert.deepEqual(s.store.calls.find((c) => c[0] === 'add')[2].ignore, ['dist/**']); + assert.deepEqual(persisted, [[collectionName('/r'), ['dist/**']]]); + const order = s.store.calls.map((c) => c[0]); + assert.ok(order.indexOf('add') < order.indexOf('update')); + fingerprint = 'two'; + await s.call('/search', { repo: '/r', query: 'q' }); + assert.equal(persisted.length, 1, 'an unchanged list is not written again'); + ignore = ['dist/**', 'tmp/**']; + fingerprint = 'three'; + await s.call('/search', { repo: '/r', query: 'q' }); + assert.deepEqual(persisted[1], [collectionName('/r'), ['dist/**', 'tmp/**']]); + assert.equal(s.store.calls.filter((c) => c[0] === 'update').length, 3); +}); diff --git a/scripts/tests/test-qmd-lib.mjs b/scripts/tests/test-qmd-lib.mjs index 29f6045..6f91273 100644 --- a/scripts/tests/test-qmd-lib.mjs +++ b/scripts/tests/test-qmd-lib.mjs @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import * as YAML from './fixtures/yaml-shim.mjs'; import { collectionName, deviceEnv, idleMinutes, qmdConfigPath, writePrivateJson, writeTextAtomic, readJson, runQmdCli, MODELS } from '../../shared/qmd/qmd-lib.mjs'; -import { setModels, unsetModels, previousModels, restoreModels } from '../../shared/qmd/qmd-config.mjs'; +import { setModels, unsetModels, previousModels, restoreModels, setCollectionIgnore } from '../../shared/qmd/qmd-config.mjs'; test('collection name is stable, prefixed, and path-safe', () => { const a = collectionName('/tmp/My Repo/ΓΌ'); @@ -102,3 +102,14 @@ test('writeTextAtomic replaces the file through a temp file in the same director assert.equal(readFileSync(file, 'utf8'), 'b: 2\n'); assert.deepEqual(readdirSync(dir), ['index.yml']); }); + +test('setCollectionIgnore writes the ignore list of one collection and keeps the rest', () => { + const original = '# mine\ncollections:\n notes:\n path: /n # keep\n repo-aaaaaaaaaaaa:\n path: /r\n includeByDefault: false\n'; + const withIgnore = setCollectionIgnore(original, YAML, 'repo-aaaaaaaaaaaa', ['dist/**', 'a.ts']); + assert.match(withIgnore, /# mine/); + assert.match(withIgnore, /path: \/n # keep/); + assert.deepEqual(YAML.default.parse(withIgnore).collections['repo-aaaaaaaaaaaa'], { path: '/r', includeByDefault: false, ignore: ['dist/**', 'a.ts'] }); + const cleared = setCollectionIgnore(withIgnore, YAML, 'repo-aaaaaaaaaaaa', []); + assert.deepEqual(YAML.default.parse(cleared).collections['repo-aaaaaaaaaaaa'], { path: '/r', includeByDefault: false }); + assert.equal(setCollectionIgnore(original, YAML, 'repo-bbbbbbbbbbbb', ['x']), null, 'an unknown collection is not created'); +}); diff --git a/shared/qmd/qmd-config.mjs b/shared/qmd/qmd-config.mjs index e069107..437b860 100644 --- a/shared/qmd/qmd-config.mjs +++ b/shared/qmd/qmd-config.mjs @@ -1,4 +1,4 @@ -// shared/qmd/qmd-config.mjs β€” edits only the models: block of QMD's index.yml. +// shared/qmd/qmd-config.mjs β€” edits only the models: block and setup-owned collection ignore lists in QMD's index.yml. import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -33,6 +33,16 @@ export function unsetModels(text, YAML) { return document.contents === null || document.contents.items?.length === 0 ? '' : document.toString(); } +// Returns the new text, or null when index.yml has no such collection (QMD's write-through always creates it). +export function setCollectionIgnore(text, YAML, name, ignore) { + const document = YAML.parseDocument(text || ''); + if (document.errors.length) throw new Error(`index.yml cannot be parsed: ${document.errors[0].message}`); + if (!document.hasIn(['collections', name])) return null; + if (ignore.length) document.setIn(['collections', name, 'ignore'], document.createNode(ignore)); + else document.deleteIn(['collections', name, 'ignore']); + return document.toString(); +} + async function main(action, argument) { if (!['set-models', 'unset-models', 'restore-models'].includes(action)) throw new Error('usage: qmd-config.mjs set-models|unset-models|restore-models '); const YAML = await import(pathToFileURL(join(qmdPackageDir(), 'node_modules', 'yaml', 'dist', 'index.js')).href); diff --git a/shared/qmd/qmd-daemon.mjs b/shared/qmd/qmd-daemon.mjs index 3576fd3..fd13a2b 100644 --- a/shared/qmd/qmd-daemon.mjs +++ b/shared/qmd/qmd-daemon.mjs @@ -1,11 +1,12 @@ // shared/qmd/qmd-daemon.mjs β€” keeps QMD's models loaded for fast searches; exits when idle. import { execFileSync } from 'node:child_process'; import { randomBytes, timingSafeEqual } from 'node:crypto'; -import { statSync } from 'node:fs'; +import { existsSync, readFileSync, statSync } from 'node:fs'; import { createServer as createHttpServer } from 'node:http'; import { join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { MASK, collectionName, idleMinutes, importQmd, qmdConfigPath, qmdDbPath, readJson, runQmdCli, stateDir, writePrivateJson } from './qmd-lib.mjs'; +import { setCollectionIgnore } from './qmd-config.mjs'; +import { MASK, collectionName, idleMinutes, importQmd, qmdConfigPath, qmdDbPath, readJson, runQmdCli, stateDir, writePrivateJson, writeTextAtomic } from './qmd-lib.mjs'; const MAX_BODY = 64 * 1024; @@ -27,6 +28,20 @@ export function gitFingerprint(repo) { } } +// git ls-files -z --directory output: ignored directories end with '/' and become 'dir/**'; escape quotes glob syntax in names. +export function ignoreGlobs(output, escape) { + return output.split('\0').filter(Boolean).map((path) => (path.endsWith('/') ? `${escape(path.slice(0, -1))}/**` : escape(path))); +} + +export function gitIgnored(repo, escape) { + try { + return ignoreGlobs(execFileSync('git', ['-C', repo, 'ls-files', '-z', '--others', '--ignored', '--exclude-standard', '--directory'], + { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }), escape); + } catch { + return []; + } +} + // QMD 2.8.3 createStore unloads models after 5 idle minutes (store.internal.llm); the daemon's own idle exit replaces that. export function keepModelsLoaded(store) { const llm = store.internal?.llm; @@ -44,11 +59,12 @@ function tokenMatches(given, expected) { } export function createServer({ store, token, idleMs, now = Date.now, onIdle, extractSnippet, git = gitFingerprint, - excludeCollection, recordCollection }) { + excludeCollection, recordCollection, ignored, persistIgnore }) { let lastRequest = now(); let active = 0; const known = new Set(); const fingerprints = new Map(); + const ignoreKeys = new Map(); const inFlight = new Map(); @@ -59,9 +75,14 @@ export function createServer({ store, token, idleMs, now = Date.now, onIdle, ext } async function doRefresh(repo, name) { + const fingerprint = git(repo); + let ignore; if (!known.has(name)) { const existing = (await store.listCollections()).some((c) => c.name === name); - if (!existing) await store.addCollection(name, { path: resolve(repo), pattern: MASK }); + if (!existing) { + ignore = ignored(repo); + await store.addCollection(name, { path: resolve(repo), pattern: MASK, ignore }); + } try { await excludeCollection(name); recordCollection(name); @@ -70,8 +91,17 @@ export function createServer({ store, token, idleMs, now = Date.now, onIdle, ext console.error(`qmd-daemon: could not exclude ${name} from default search: ${error.message}`); } } - const fingerprint = git(repo); if (fingerprints.get(name) === fingerprint) return { collection: name, updated: false }; + ignore ??= ignored(repo); + const ignoreKey = JSON.stringify(ignore); + if (ignoreKeys.get(name) !== ignoreKey) { + try { + await persistIgnore(name, ignore); + ignoreKeys.set(name, ignoreKey); + } catch (error) { + console.error(`qmd-daemon: could not save the git ignore list of ${name}: ${error.message}`); + } + } await store.update({ collections: [name] }); await store.embed({ collection: name, chunkStrategy: 'auto' }); fingerprints.set(name, fingerprint); @@ -145,6 +175,12 @@ async function main() { const { createStore, extractSnippet } = await importQmd(); const store = await createStore({ dbPath: qmdDbPath(), configPath: qmdConfigPath() }); keepModelsLoaded(store); + // QMD keeps index.yml authoritative: createStore and the CLI re-sync it into SQLite, and its addCollection + // write-through drops `ignore`, so the list is written into index.yml and re-synced the way QMD does it. + const fastGlob = (await importQmd(join('node_modules', 'fast-glob', 'out', 'index.js'))).default; + const YAML = await importQmd(join('node_modules', 'yaml', 'dist', 'index.js')); + const { loadConfig } = await importQmd(join('dist', 'collections.js')); + const { syncConfigToDb } = await importQmd(join('dist', 'store.js')); const collectionsFile = join(dir, 'collections.json'); const token = randomBytes(32).toString('hex'); let stopping = false; @@ -160,6 +196,15 @@ async function main() { }; const daemon = createServer({ store, token, idleMs: idleMinutes() * 60_000, onIdle: stop, extractSnippet, + ignored: (repo) => gitIgnored(repo, fastGlob.escapePath), + persistIgnore: async (name, ignore) => { + const path = qmdConfigPath(); + const text = existsSync(path) ? readFileSync(path, 'utf8') : ''; + const next = setCollectionIgnore(text, YAML, name, ignore); + if (next === null) return; // never re-sync without the collection: the sync would delete it from SQLite + if (next !== text) writeTextAtomic(path, next); + syncConfigToDb(store.internal.db, loadConfig()); + }, excludeCollection: async (name) => { runQmdCli(['collection', 'exclude', name], { windowsHide: true, stdio: 'ignore' }); }, recordCollection: (name) => { const names = new Set(readJson(collectionsFile, []));