fix(qmd): skip git-ignored paths in repository collections

The daemon passes `git ls-files --others --ignored --directory` as
escaped ignore globs to addCollection. QMD's YAML write-through drops
`ignore` and index.yml wins on every re-sync, so the list is also
written into index.yml and re-synced whenever it changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Julian lechnerandClaude Opus 5.5 committed 2026-10-08 12:45:27 +02:00
1 parent abface2293
commit 691ad6a69f
4 files changed
+102 -9

No files matched your search

+50 -5
View File
@@ -1,11 +1,12 @@
// shared/qmd/qmd-daemon.mjs — keeps QMD's models loaded for fast searches; exits when idle.
import { execFileSync } from 'node:child_process';
import { randomBytes, timingSafeEqual } from 'node:crypto';
import { statSync } from 'node:fs';
import { existsSync, readFileSync, statSync } from 'node:fs';
import { createServer as createHttpServer } from 'node:http';
import { join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import { MASK, collectionName, idleMinutes, importQmd, qmdConfigPath, qmdDbPath, readJson, runQmdCli, stateDir, writePrivateJson } from './qmd-lib.mjs';
import { setCollectionIgnore } from './qmd-config.mjs';
import { MASK, collectionName, idleMinutes, importQmd, qmdConfigPath, qmdDbPath, readJson, runQmdCli, stateDir, writePrivateJson, writeTextAtomic } from './qmd-lib.mjs';
const MAX_BODY = 64 * 1024;
@@ -27,6 +28,20 @@ export function gitFingerprint(repo) {
}
}
// git ls-files -z --directory output: ignored directories end with '/' and become 'dir/**'; escape quotes glob syntax in names.
export function ignoreGlobs(output, escape) {
return output.split('\0').filter(Boolean).map((path) => (path.endsWith('/') ? `${escape(path.slice(0, -1))}/**` : escape(path)));
}
export function gitIgnored(repo, escape) {
try {
return ignoreGlobs(execFileSync('git', ['-C', repo, 'ls-files', '-z', '--others', '--ignored', '--exclude-standard', '--directory'],
{ encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, windowsHide: true }), escape);
} catch {
return [];
}
}
// QMD 2.8.3 createStore unloads models after 5 idle minutes (store.internal.llm); the daemon's own idle exit replaces that.
export function keepModelsLoaded(store) {
const llm = store.internal?.llm;
@@ -44,11 +59,12 @@ function tokenMatches(given, expected) {
}
export function createServer({ store, token, idleMs, now = Date.now, onIdle, extractSnippet, git = gitFingerprint,
excludeCollection, recordCollection }) {
excludeCollection, recordCollection, ignored, persistIgnore }) {
let lastRequest = now();
let active = 0;
const known = new Set();
const fingerprints = new Map();
const ignoreKeys = new Map();
const inFlight = new Map();
@@ -59,9 +75,14 @@ export function createServer({ store, token, idleMs, now = Date.now, onIdle, ext
}
async function doRefresh(repo, name) {
const fingerprint = git(repo);
let ignore;
if (!known.has(name)) {
const existing = (await store.listCollections()).some((c) => c.name === name);
if (!existing) await store.addCollection(name, { path: resolve(repo), pattern: MASK });
if (!existing) {
ignore = ignored(repo);
await store.addCollection(name, { path: resolve(repo), pattern: MASK, ignore });
}
try {
await excludeCollection(name);
recordCollection(name);
@@ -70,8 +91,17 @@ export function createServer({ store, token, idleMs, now = Date.now, onIdle, ext
console.error(`qmd-daemon: could not exclude ${name} from default search: ${error.message}`);
}
}
const fingerprint = git(repo);
if (fingerprints.get(name) === fingerprint) return { collection: name, updated: false };
ignore ??= ignored(repo);
const ignoreKey = JSON.stringify(ignore);
if (ignoreKeys.get(name) !== ignoreKey) {
try {
await persistIgnore(name, ignore);
ignoreKeys.set(name, ignoreKey);
} catch (error) {
console.error(`qmd-daemon: could not save the git ignore list of ${name}: ${error.message}`);
}
}
await store.update({ collections: [name] });
await store.embed({ collection: name, chunkStrategy: 'auto' });
fingerprints.set(name, fingerprint);
@@ -145,6 +175,12 @@ async function main() {
const { createStore, extractSnippet } = await importQmd();
const store = await createStore({ dbPath: qmdDbPath(), configPath: qmdConfigPath() });
keepModelsLoaded(store);
// QMD keeps index.yml authoritative: createStore and the CLI re-sync it into SQLite, and its addCollection
// write-through drops `ignore`, so the list is written into index.yml and re-synced the way QMD does it.
const fastGlob = (await importQmd(join('node_modules', 'fast-glob', 'out', 'index.js'))).default;
const YAML = await importQmd(join('node_modules', 'yaml', 'dist', 'index.js'));
const { loadConfig } = await importQmd(join('dist', 'collections.js'));
const { syncConfigToDb } = await importQmd(join('dist', 'store.js'));
const collectionsFile = join(dir, 'collections.json');
const token = randomBytes(32).toString('hex');
let stopping = false;
@@ -160,6 +196,15 @@ async function main() {
};
const daemon = createServer({
store, token, idleMs: idleMinutes() * 60_000, onIdle: stop, extractSnippet,
ignored: (repo) => gitIgnored(repo, fastGlob.escapePath),
persistIgnore: async (name, ignore) => {
const path = qmdConfigPath();
const text = existsSync(path) ? readFileSync(path, 'utf8') : '';
const next = setCollectionIgnore(text, YAML, name, ignore);
if (next === null) return; // never re-sync without the collection: the sync would delete it from SQLite
if (next !== text) writeTextAtomic(path, next);
syncConfigToDb(store.internal.db, loadConfig());
},
excludeCollection: async (name) => { runQmdCli(['collection', 'exclude', name], { windowsHide: true, stdio: 'ignore' }); },
recordCollection: (name) => {
const names = new Set(readJson(collectionsFile, []));