feat(qmd): add shared Node helpers and index.yml model block editing
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Julian lechnerandClaude Opus 5.5 committed
1 parent
c7c7c57621
commit
1ad1880704
4 files changed
+201
No files matched your search
Vendored
+7
@@ -0,0 +1,7 @@
|
||||
// Loads QMD's bundled yaml package for tests.
|
||||
import { qmdPackageDir } from '../../../shared/qmd/qmd-lib.mjs';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { join } from 'node:path';
|
||||
const YAML = await import(pathToFileURL(join(qmdPackageDir(), 'node_modules', 'yaml', 'dist', 'index.js')).href);
|
||||
export const parseDocument = YAML.parseDocument;
|
||||
export default YAML;
|
||||
@@ -0,0 +1,77 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import * as YAML from './fixtures/yaml-shim.mjs';
|
||||
import { collectionName, deviceEnv, idleMinutes, qmdConfigPath, writePrivateJson, readJson, runQmdCli, MODELS } from '../../shared/qmd/qmd-lib.mjs';
|
||||
import { setModels, unsetModels } from '../../shared/qmd/qmd-config.mjs';
|
||||
|
||||
test('collection name is stable, prefixed, and path-safe', () => {
|
||||
const a = collectionName('/tmp/My Repo/ü');
|
||||
assert.match(a, /^repo-[0-9a-f]{12}$/);
|
||||
assert.equal(a, collectionName('/tmp/My Repo/ü'));
|
||||
assert.notEqual(a, collectionName('/tmp/My Repo/u'));
|
||||
});
|
||||
|
||||
test('device env forces CPU or enables GPU auto', () => {
|
||||
assert.equal(deviceEnv('cpu', { QMD_LLAMA_GPU: 'cuda' }).QMD_FORCE_CPU, '1');
|
||||
assert.equal(deviceEnv('cpu', { QMD_LLAMA_GPU: 'cuda' }).QMD_LLAMA_GPU, undefined);
|
||||
assert.equal(deviceEnv('gpu', { QMD_FORCE_CPU: '1' }).QMD_LLAMA_GPU, 'auto');
|
||||
assert.equal(deviceEnv('gpu', { QMD_FORCE_CPU: '1' }).QMD_FORCE_CPU, undefined);
|
||||
});
|
||||
|
||||
test('device env never narrows GPUs or threads', () => {
|
||||
for (const device of ['gpu', 'cpu']) {
|
||||
const env = deviceEnv(device, {});
|
||||
for (const key of ['CUDA_VISIBLE_DEVICES', 'GGML_VK_VISIBLE_DEVICES', 'HIP_VISIBLE_DEVICES', 'OMP_NUM_THREADS']) assert.equal(env[key], undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test('idle minutes default to 30 and reject nonsense', () => {
|
||||
assert.equal(idleMinutes({}), 30);
|
||||
assert.equal(idleMinutes({ QMD_IDLE_MINUTES: '5' }), 5);
|
||||
assert.equal(idleMinutes({ QMD_IDLE_MINUTES: '-1' }), 30);
|
||||
});
|
||||
|
||||
test('config path honors QMD_CONFIG_DIR then XDG_CONFIG_HOME', () => {
|
||||
assert.equal(qmdConfigPath({ QMD_CONFIG_DIR: '/c' }, '/h'), join('/c', 'index.yml'));
|
||||
assert.equal(qmdConfigPath({ XDG_CONFIG_HOME: '/x' }, '/h'), join('/x', 'qmd', 'index.yml'));
|
||||
assert.equal(qmdConfigPath({}, '/h'), join('/h', '.config', 'qmd', 'index.yml'));
|
||||
});
|
||||
|
||||
test('private json round-trips', () => {
|
||||
const file = join(mkdtempSync(join(tmpdir(), 'qmd-lib-')), 'a', 'b.json');
|
||||
writePrivateJson(file, { x: 1 });
|
||||
assert.deepEqual(readJson(file, null), { x: 1 });
|
||||
assert.equal(readJson(file + '.missing', 'fallback'), 'fallback');
|
||||
});
|
||||
|
||||
test('setModels keeps collections and comments, unsetModels removes only models', () => {
|
||||
const original = '# mine\ncollections:\n notes:\n path: /n # keep\n';
|
||||
const withModels = setModels(original, YAML);
|
||||
assert.match(withModels, /# mine/);
|
||||
assert.match(withModels, /path: \/n # keep/);
|
||||
assert.match(withModels, new RegExp(MODELS.embed.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
|
||||
const restored = unsetModels(withModels, YAML);
|
||||
assert.doesNotMatch(restored, /models:/);
|
||||
assert.match(restored, /path: \/n # keep/);
|
||||
});
|
||||
|
||||
test('setModels on an empty file creates only the models block', () => {
|
||||
const text = setModels('', YAML);
|
||||
assert.match(text, /^models:/m);
|
||||
});
|
||||
|
||||
test('runQmdCli runs the CLI entry of QMD_PACKAGE_DIR without a shell', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'qmd-pkg-'));
|
||||
mkdirSync(join(dir, 'dist', 'cli'), { recursive: true });
|
||||
writeFileSync(join(dir, 'dist', 'cli', 'qmd.js'), 'console.log(JSON.stringify(process.argv.slice(2)));\n');
|
||||
const previous = process.env.QMD_PACKAGE_DIR;
|
||||
process.env.QMD_PACKAGE_DIR = dir;
|
||||
try {
|
||||
assert.deepEqual(JSON.parse(runQmdCli(['search', 'a "b" & c'])), ['search', 'a "b" & c']);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.QMD_PACKAGE_DIR; else process.env.QMD_PACKAGE_DIR = previous;
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
// shared/qmd/qmd-config.mjs — edits only the models: block of QMD's index.yml.
|
||||
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { MODELS, qmdConfigPath, qmdPackageDir } from './qmd-lib.mjs';
|
||||
|
||||
export function setModels(text, YAML) {
|
||||
const document = YAML.parseDocument(text || '');
|
||||
if (document.errors.length) throw new Error(`index.yml cannot be parsed: ${document.errors[0].message}`);
|
||||
if (document.contents === null) document.contents = document.createNode({});
|
||||
document.set('models', document.createNode({ ...MODELS }));
|
||||
return document.toString();
|
||||
}
|
||||
|
||||
export function unsetModels(text, YAML) {
|
||||
const document = YAML.parseDocument(text || '');
|
||||
if (document.errors.length) throw new Error(`index.yml cannot be parsed: ${document.errors[0].message}`);
|
||||
if (document.contents !== null) document.delete('models');
|
||||
return document.contents === null || document.contents.items?.length === 0 ? '' : document.toString();
|
||||
}
|
||||
|
||||
async function main(action) {
|
||||
if (!['set-models', 'unset-models'].includes(action)) throw new Error('usage: qmd-config.mjs set-models|unset-models');
|
||||
const YAML = await import(pathToFileURL(join(qmdPackageDir(), 'node_modules', 'yaml', 'dist', 'index.js')).href);
|
||||
const path = qmdConfigPath();
|
||||
const current = existsSync(path) ? readFileSync(path, 'utf8') : '';
|
||||
const next = action === 'set-models' ? setModels(current, YAML) : unsetModels(current, YAML);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, next);
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] || '').href) {
|
||||
main(process.argv[2]).catch((error) => { console.error(`qmd-config: ${error.message}`); process.exit(1); });
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// shared/qmd/qmd-lib.mjs
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
export const MODELS = {
|
||||
embed: 'hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf',
|
||||
rerank: 'hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf',
|
||||
generate: 'hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf',
|
||||
};
|
||||
export const MASK = '**/*.{md,py,cs,ts,tsx,js,jsx,mjs,go,rs,java,ps1,sh,json,yml,yaml,xml,sql}';
|
||||
|
||||
export function collectionName(repoPath) {
|
||||
return 'repo-' + createHash('sha256').update(resolve(repoPath), 'utf8').digest('hex').slice(0, 12);
|
||||
}
|
||||
|
||||
export const stateDir = (home = homedir()) => join(home, '.my-ai-configuration', 'qmd');
|
||||
export const setupStatePath = (home = homedir()) => join(home, '.my-ai-configuration', 'qmd.json');
|
||||
|
||||
export function qmdConfigPath(env = process.env, home = homedir()) {
|
||||
const dir = env.QMD_CONFIG_DIR || (env.XDG_CONFIG_HOME ? join(env.XDG_CONFIG_HOME, 'qmd') : join(home, '.config', 'qmd'));
|
||||
return join(dir, 'index.yml');
|
||||
}
|
||||
|
||||
export function qmdDbPath(env = process.env, home = homedir()) {
|
||||
return join(env.XDG_CACHE_HOME || join(home, '.cache'), 'qmd', 'index.sqlite');
|
||||
}
|
||||
|
||||
export function deviceEnv(device, env = process.env) {
|
||||
const result = { ...env };
|
||||
if (device === 'cpu') {
|
||||
result.QMD_FORCE_CPU = '1';
|
||||
delete result.QMD_LLAMA_GPU;
|
||||
} else {
|
||||
result.QMD_LLAMA_GPU = 'auto';
|
||||
delete result.QMD_FORCE_CPU;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function idleMinutes(env = process.env) {
|
||||
const value = Number(env.QMD_IDLE_MINUTES);
|
||||
return Number.isFinite(value) && value > 0 ? value : 30;
|
||||
}
|
||||
|
||||
export function readJson(path, fallback) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf8'));
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
// The state directory lives in the user profile; mode 0600 restricts the token on POSIX.
|
||||
export function writePrivateJson(path, value) {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
const temporary = `${path}.${process.pid}.tmp`;
|
||||
writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 });
|
||||
renameSync(temporary, path);
|
||||
}
|
||||
|
||||
let cachedPackageDir;
|
||||
|
||||
export function qmdPackageDir() {
|
||||
if (process.env.QMD_PACKAGE_DIR) return process.env.QMD_PACKAGE_DIR;
|
||||
if (!cachedPackageDir) {
|
||||
const root = execFileSync('npm', ['root', '--global'], { encoding: 'utf8', shell: process.platform === 'win32', windowsHide: true }).trim();
|
||||
cachedPackageDir = join(root, '@tobilu', 'qmd');
|
||||
}
|
||||
return cachedPackageDir;
|
||||
}
|
||||
|
||||
export function importQmd(subpath = join('dist', 'index.js')) {
|
||||
return import(pathToFileURL(join(qmdPackageDir(), subpath)).href);
|
||||
}
|
||||
|
||||
// Runs the QMD CLI through node without a shell, so free-text arguments are never re-parsed.
|
||||
export function runQmdCli(args, options = {}) {
|
||||
return execFileSync(process.execPath, [join(qmdPackageDir(), 'dist', 'cli', 'qmd.js'), ...args], { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024, ...options });
|
||||
}
|
||||
Reference in new issue
Block a user