From 1ad18807043e102ca1664f326e126da967014bca Mon Sep 17 00:00:00 2001 From: Julian lechner Date: Thu, 8 Oct 2026 11:15:08 +0200 Subject: [PATCH] feat(qmd): add shared Node helpers and index.yml model block editing Co-Authored-By: Claude Opus 5.5 --- scripts/tests/fixtures/yaml-shim.mjs | 7 +++ scripts/tests/test-qmd-lib.mjs | 77 ++++++++++++++++++++++++++ shared/qmd/qmd-config.mjs | 34 ++++++++++++ shared/qmd/qmd-lib.mjs | 83 ++++++++++++++++++++++++++++ 4 files changed, 201 insertions(+) create mode 100644 scripts/tests/fixtures/yaml-shim.mjs create mode 100644 scripts/tests/test-qmd-lib.mjs create mode 100644 shared/qmd/qmd-config.mjs create mode 100644 shared/qmd/qmd-lib.mjs diff --git a/scripts/tests/fixtures/yaml-shim.mjs b/scripts/tests/fixtures/yaml-shim.mjs new file mode 100644 index 0000000..d6075a5 --- /dev/null +++ b/scripts/tests/fixtures/yaml-shim.mjs @@ -0,0 +1,7 @@ +// Loads QMD's bundled yaml package for tests. +import { qmdPackageDir } from '../../../shared/qmd/qmd-lib.mjs'; +import { pathToFileURL } from 'node:url'; +import { join } from 'node:path'; +const YAML = await import(pathToFileURL(join(qmdPackageDir(), 'node_modules', 'yaml', 'dist', 'index.js')).href); +export const parseDocument = YAML.parseDocument; +export default YAML; diff --git a/scripts/tests/test-qmd-lib.mjs b/scripts/tests/test-qmd-lib.mjs new file mode 100644 index 0000000..bea60f1 --- /dev/null +++ b/scripts/tests/test-qmd-lib.mjs @@ -0,0 +1,77 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import * as YAML from './fixtures/yaml-shim.mjs'; +import { collectionName, deviceEnv, idleMinutes, qmdConfigPath, writePrivateJson, readJson, runQmdCli, MODELS } from '../../shared/qmd/qmd-lib.mjs'; +import { setModels, unsetModels } from '../../shared/qmd/qmd-config.mjs'; + +test('collection name is stable, prefixed, and path-safe', () => { + const a = collectionName('/tmp/My Repo/ü'); + assert.match(a, /^repo-[0-9a-f]{12}$/); + assert.equal(a, collectionName('/tmp/My Repo/ü')); + assert.notEqual(a, collectionName('/tmp/My Repo/u')); +}); + +test('device env forces CPU or enables GPU auto', () => { + assert.equal(deviceEnv('cpu', { QMD_LLAMA_GPU: 'cuda' }).QMD_FORCE_CPU, '1'); + assert.equal(deviceEnv('cpu', { QMD_LLAMA_GPU: 'cuda' }).QMD_LLAMA_GPU, undefined); + assert.equal(deviceEnv('gpu', { QMD_FORCE_CPU: '1' }).QMD_LLAMA_GPU, 'auto'); + assert.equal(deviceEnv('gpu', { QMD_FORCE_CPU: '1' }).QMD_FORCE_CPU, undefined); +}); + +test('device env never narrows GPUs or threads', () => { + for (const device of ['gpu', 'cpu']) { + const env = deviceEnv(device, {}); + for (const key of ['CUDA_VISIBLE_DEVICES', 'GGML_VK_VISIBLE_DEVICES', 'HIP_VISIBLE_DEVICES', 'OMP_NUM_THREADS']) assert.equal(env[key], undefined); + } +}); + +test('idle minutes default to 30 and reject nonsense', () => { + assert.equal(idleMinutes({}), 30); + assert.equal(idleMinutes({ QMD_IDLE_MINUTES: '5' }), 5); + assert.equal(idleMinutes({ QMD_IDLE_MINUTES: '-1' }), 30); +}); + +test('config path honors QMD_CONFIG_DIR then XDG_CONFIG_HOME', () => { + assert.equal(qmdConfigPath({ QMD_CONFIG_DIR: '/c' }, '/h'), join('/c', 'index.yml')); + assert.equal(qmdConfigPath({ XDG_CONFIG_HOME: '/x' }, '/h'), join('/x', 'qmd', 'index.yml')); + assert.equal(qmdConfigPath({}, '/h'), join('/h', '.config', 'qmd', 'index.yml')); +}); + +test('private json round-trips', () => { + const file = join(mkdtempSync(join(tmpdir(), 'qmd-lib-')), 'a', 'b.json'); + writePrivateJson(file, { x: 1 }); + assert.deepEqual(readJson(file, null), { x: 1 }); + assert.equal(readJson(file + '.missing', 'fallback'), 'fallback'); +}); + +test('setModels keeps collections and comments, unsetModels removes only models', () => { + const original = '# mine\ncollections:\n notes:\n path: /n # keep\n'; + const withModels = setModels(original, YAML); + assert.match(withModels, /# mine/); + assert.match(withModels, /path: \/n # keep/); + assert.match(withModels, new RegExp(MODELS.embed.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))); + const restored = unsetModels(withModels, YAML); + assert.doesNotMatch(restored, /models:/); + assert.match(restored, /path: \/n # keep/); +}); + +test('setModels on an empty file creates only the models block', () => { + const text = setModels('', YAML); + assert.match(text, /^models:/m); +}); + +test('runQmdCli runs the CLI entry of QMD_PACKAGE_DIR without a shell', () => { + const dir = mkdtempSync(join(tmpdir(), 'qmd-pkg-')); + mkdirSync(join(dir, 'dist', 'cli'), { recursive: true }); + writeFileSync(join(dir, 'dist', 'cli', 'qmd.js'), 'console.log(JSON.stringify(process.argv.slice(2)));\n'); + const previous = process.env.QMD_PACKAGE_DIR; + process.env.QMD_PACKAGE_DIR = dir; + try { + assert.deepEqual(JSON.parse(runQmdCli(['search', 'a "b" & c'])), ['search', 'a "b" & c']); + } finally { + if (previous === undefined) delete process.env.QMD_PACKAGE_DIR; else process.env.QMD_PACKAGE_DIR = previous; + } +}); diff --git a/shared/qmd/qmd-config.mjs b/shared/qmd/qmd-config.mjs new file mode 100644 index 0000000..4e836ea --- /dev/null +++ b/shared/qmd/qmd-config.mjs @@ -0,0 +1,34 @@ +// shared/qmd/qmd-config.mjs — edits only the models: block of QMD's index.yml. +import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { MODELS, qmdConfigPath, qmdPackageDir } from './qmd-lib.mjs'; + +export function setModels(text, YAML) { + const document = YAML.parseDocument(text || ''); + if (document.errors.length) throw new Error(`index.yml cannot be parsed: ${document.errors[0].message}`); + if (document.contents === null) document.contents = document.createNode({}); + document.set('models', document.createNode({ ...MODELS })); + return document.toString(); +} + +export function unsetModels(text, YAML) { + const document = YAML.parseDocument(text || ''); + if (document.errors.length) throw new Error(`index.yml cannot be parsed: ${document.errors[0].message}`); + if (document.contents !== null) document.delete('models'); + return document.contents === null || document.contents.items?.length === 0 ? '' : document.toString(); +} + +async function main(action) { + if (!['set-models', 'unset-models'].includes(action)) throw new Error('usage: qmd-config.mjs set-models|unset-models'); + const YAML = await import(pathToFileURL(join(qmdPackageDir(), 'node_modules', 'yaml', 'dist', 'index.js')).href); + const path = qmdConfigPath(); + const current = existsSync(path) ? readFileSync(path, 'utf8') : ''; + const next = action === 'set-models' ? setModels(current, YAML) : unsetModels(current, YAML); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, next); +} + +if (import.meta.url === pathToFileURL(process.argv[1] || '').href) { + main(process.argv[2]).catch((error) => { console.error(`qmd-config: ${error.message}`); process.exit(1); }); +} diff --git a/shared/qmd/qmd-lib.mjs b/shared/qmd/qmd-lib.mjs new file mode 100644 index 0000000..8aba9da --- /dev/null +++ b/shared/qmd/qmd-lib.mjs @@ -0,0 +1,83 @@ +// shared/qmd/qmd-lib.mjs +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +export const MODELS = { + embed: 'hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf', + rerank: 'hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf', + generate: 'hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf', +}; +export const MASK = '**/*.{md,py,cs,ts,tsx,js,jsx,mjs,go,rs,java,ps1,sh,json,yml,yaml,xml,sql}'; + +export function collectionName(repoPath) { + return 'repo-' + createHash('sha256').update(resolve(repoPath), 'utf8').digest('hex').slice(0, 12); +} + +export const stateDir = (home = homedir()) => join(home, '.my-ai-configuration', 'qmd'); +export const setupStatePath = (home = homedir()) => join(home, '.my-ai-configuration', 'qmd.json'); + +export function qmdConfigPath(env = process.env, home = homedir()) { + const dir = env.QMD_CONFIG_DIR || (env.XDG_CONFIG_HOME ? join(env.XDG_CONFIG_HOME, 'qmd') : join(home, '.config', 'qmd')); + return join(dir, 'index.yml'); +} + +export function qmdDbPath(env = process.env, home = homedir()) { + return join(env.XDG_CACHE_HOME || join(home, '.cache'), 'qmd', 'index.sqlite'); +} + +export function deviceEnv(device, env = process.env) { + const result = { ...env }; + if (device === 'cpu') { + result.QMD_FORCE_CPU = '1'; + delete result.QMD_LLAMA_GPU; + } else { + result.QMD_LLAMA_GPU = 'auto'; + delete result.QMD_FORCE_CPU; + } + return result; +} + +export function idleMinutes(env = process.env) { + const value = Number(env.QMD_IDLE_MINUTES); + return Number.isFinite(value) && value > 0 ? value : 30; +} + +export function readJson(path, fallback) { + try { + return JSON.parse(readFileSync(path, 'utf8')); + } catch { + return fallback; + } +} + +// The state directory lives in the user profile; mode 0600 restricts the token on POSIX. +export function writePrivateJson(path, value) { + mkdirSync(dirname(path), { recursive: true }); + const temporary = `${path}.${process.pid}.tmp`; + writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 }); + renameSync(temporary, path); +} + +let cachedPackageDir; + +export function qmdPackageDir() { + if (process.env.QMD_PACKAGE_DIR) return process.env.QMD_PACKAGE_DIR; + if (!cachedPackageDir) { + const root = execFileSync('npm', ['root', '--global'], { encoding: 'utf8', shell: process.platform === 'win32', windowsHide: true }).trim(); + cachedPackageDir = join(root, '@tobilu', 'qmd'); + } + return cachedPackageDir; +} + +export function importQmd(subpath = join('dist', 'index.js')) { + return import(pathToFileURL(join(qmdPackageDir(), subpath)).href); +} + +// Runs the QMD CLI through node without a shell, so free-text arguments are never re-parsed. +export function runQmdCli(args, options = {}) { + return execFileSync(process.execPath, [join(qmdPackageDir(), 'dist', 'cli', 'qmd.js'), ...args], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, ...options }); +}