From 5f21e4fb7c20544ebd49072f1f0bfe5e78e28225 Mon Sep 17 00:00:00 2001 From: Julian lechner Date: Thu, 8 Oct 2026 12:17:49 +0200 Subject: [PATCH] ci(qmd): replace the Qwen retrieval job with a QMD integration job Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/ci.yml | 29 +++++++++++++------------- scripts/benchmarks/ab.py | 19 ++++++++--------- scripts/tests/test-qmd-integration.mjs | 24 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 25 deletions(-) create mode 100644 scripts/tests/test-qmd-integration.mjs diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 6736596..f601568 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -172,29 +172,28 @@ jobs: AI_CONFIG_REQUIRE_CLIENTS: '1' run: timeout 900 python scripts/tests/test-client-smoke.py -v - retrieval: - name: Qwen retrieval integration + qmd: + name: QMD local search integration runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 needs: [python] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v22.16.0 with: - python-version: '3.12' - cache: pip - cache-dependency-path: shared/retrieval/requirements.txt - - name: Install retrieval dependencies - run: python -m pip install -r shared/retrieval/requirements.txt + node-version: '24' + - name: Install QMD + run: npm install --global @tobilu/qmd@2.8.3 - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: .cache/huggingface - key: qwen-retrieval-${{ runner.os }}-${{ hashFiles('shared/retrieval/requirements.txt') }} - - name: Run retrieval tests with real models + path: .cache/qmd-models + key: qmd-models-${{ runner.os }}-2.8.3-qwen3 + - name: Run QMD unit tests + run: node --test scripts/tests/test-qmd-lib.mjs scripts/tests/test-qmd-daemon.mjs scripts/tests/test-qmd-search.mjs scripts/tests/test-qmd-warm.mjs scripts/tests/test-qmd-benchmark.mjs + - name: Run QMD integration test with real models env: - QWEN_TEST_MODEL_CACHE: ${{ github.workspace }}/.cache/huggingface - HF_HUB_DISABLE_XET: '1' - run: python scripts/tests/test-retrieval-server.py + QMD_TEST_CACHE: ${{ github.workspace }}/.cache/qmd-models + run: node --test scripts/tests/test-qmd-integration.mjs security: name: CI configuration security checks diff --git a/scripts/benchmarks/ab.py b/scripts/benchmarks/ab.py index 7a5bf31..c782e41 100644 --- a/scripts/benchmarks/ab.py +++ b/scripts/benchmarks/ab.py @@ -84,8 +84,8 @@ def claude_flags(arm, args): if arm == 'baseline': flags += ['--setting-sources', '', '--strict-mcp-config'] if arm == 'setup-no-search': - flags += ['--disallowedTools', 'Skill(semantic-search)', 'Bash(*semantic-retrieval*)', 'PowerShell(*semantic-retrieval*)', - '--append-system-prompt', 'The semantic-search skill and the semantic retrieval CLI are unavailable in this session.'] + flags += ['--disallowedTools', 'Skill(semantic-search)', 'Bash(*qmd-search*)', 'PowerShell(*qmd-search*)', + '--append-system-prompt', 'The semantic-search skill and the QMD search wrapper are unavailable in this session.'] return flags @@ -276,7 +276,7 @@ def preflight(args, clients, arms): return provenance, problems -RETRIEVAL = Path.home() / '.my-ai-configuration/semantic-retrieval' +QMD_SCRIPTS = Path.home() / '.my-ai-configuration/qmd' FILLER_TOPICS = [ ('statement', 'download', 'Render a statement download for the export queue.'), ('trial', 'banner', 'Show the trial banner text in the marketing header.'), @@ -303,11 +303,10 @@ def generate_filler(work, count): def warm_index(work): - """Build the semantic index before the session, as in a repository that was searched before.""" - server = RETRIEVAL / 'server.py' - if server.is_file(): - subprocess.run([sys.executable, str(server), 'rebuild', '--root', str(work), '--data-dir', str(RETRIEVAL / 'data'), - '--model-cache', str(RETRIEVAL / 'model-cache')], capture_output=True, timeout=1800) + """Index the fixture and start the QMD daemon before the session, as in a repository that was searched before.""" + script = QMD_SCRIPTS / 'qmd-warm.mjs' + if script.is_file(): + subprocess.run(['node', str(script), '--root', str(work)], capture_output=True, timeout=1800) def check(work, task): @@ -377,7 +376,7 @@ def run_one(job, args): 'agents': sum(turn['agents'] for turn in turns), 'skills': [skill for turn in turns for skill in turn['skills']], 'commands': [command for turn in turns for command in turn['commands']], 'models': sorted({model for turn in turns for model in turn['models']}), - 'searches': sum('semantic-retrieval' in command for turn in turns for command in turn['commands']) + 'searches': sum('qmd-search' in command for turn in turns for command in turn['commands']) + sum(skill == 'semantic-search' for turn in turns for skill in turn['skills']), 'timed_out': any(turn['timed_out'] for turn in turns), 'workflow_kind': spec['workflow'].get('kind'), 'workflow_violations': workflow_violations(spec['workflow'], turns), 'check_output': '' if passed else output, @@ -414,7 +413,7 @@ def main(): parser.add_argument('--clients', nargs='+', choices=('claude', 'codex'), default=['claude', 'codex']) parser.add_argument('--arms', nargs='+', choices=('setup', 'baseline', 'setup-no-search'), default=['setup', 'baseline'], help='setup-no-search is the setup with the semantic-search skill and CLI blocked (Claude only)') - parser.add_argument('--warm-index', action='store_true', help='build the semantic index before setup-arm sessions, outside the timing') + parser.add_argument('--warm-index', action='store_true', help='start the QMD daemon and index the fixture before setup-arm sessions, outside the timing') parser.add_argument('--tasks', nargs='+', help='task directory names or patterns such as 11-* (default: all)') parser.add_argument('--repetitions', type=int, default=3) parser.add_argument('--first-repetition', type=int, default=1, help='number of the first repetition, to rerun selected repetitions') diff --git a/scripts/tests/test-qmd-integration.mjs b/scripts/tests/test-qmd-integration.mjs new file mode 100644 index 0000000..cdd8537 --- /dev/null +++ b/scripts/tests/test-qmd-integration.mjs @@ -0,0 +1,24 @@ +// Real QMD, real models; run only in the qmd CI job or manually. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +const search = resolve('shared/qmd/qmd-search.mjs'); + +test('wrapper finds the retry logic through the daemon and stops it', { timeout: 1_800_000 }, () => { + const home = mkdtempSync(join(tmpdir(), 'qmd-it-home-')); + const repo = join(home, 'repo'); + mkdirSync(join(repo, 'src'), { recursive: true }); + writeFileSync(join(repo, 'src', 'upload.ts'), 'export async function sendWithRetry(send) {\n for (let attempt = 1; attempt <= 3; attempt++) {\n try { return await send(); } catch (e) { if (!e.transient || attempt === 3) throw e; }\n }\n}\n'); + writeFileSync(join(repo, 'src', 'price.ts'), 'export const formatCents = (c) => (c / 100).toFixed(2);\n'); + execFileSync('git', ['init', '-q'], { cwd: repo }); + execFileSync('git', ['add', '-A'], { cwd: repo }); + const env = { ...process.env, HOME: home, USERPROFILE: home, XDG_CONFIG_HOME: join(home, '.config'), XDG_CACHE_HOME: process.env.QMD_TEST_CACHE || join(home, '.cache'), QMD_FORCE_CPU: '1' }; + execFileSync('node', ['shared/qmd/qmd-config.mjs', 'set-models'], { env }); + const results = JSON.parse(execFileSync('node', [search, '--root', repo, '--query', 'Where are failed uploads retried after temporary errors?'], { env, encoding: 'utf8', timeout: 1_700_000 })); + assert.equal(results[0].path.replace(/\/g, '/'), 'src/upload.ts'); + execFileSync('node', [search, '--stop'], { env }); +});