ci(qmd): replace the Qwen retrieval job with a QMD integration job
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c31af49b70
commit
5f21e4fb7c
3 files changed
+47
-25
No files matched your search
+14
-15
@@ -172,29 +172,28 @@ jobs:
|
|||||||
AI_CONFIG_REQUIRE_CLIENTS: '1'
|
AI_CONFIG_REQUIRE_CLIENTS: '1'
|
||||||
run: timeout 900 python scripts/tests/test-client-smoke.py -v
|
run: timeout 900 python scripts/tests/test-client-smoke.py -v
|
||||||
|
|
||||||
retrieval:
|
qmd:
|
||||||
name: Qwen retrieval integration
|
name: QMD local search integration
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 30
|
timeout-minutes: 45
|
||||||
needs: [python]
|
needs: [python]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
- uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v22.16.0
|
||||||
with:
|
with:
|
||||||
python-version: '3.12'
|
node-version: '24'
|
||||||
cache: pip
|
- name: Install QMD
|
||||||
cache-dependency-path: shared/retrieval/requirements.txt
|
run: npm install --global @tobilu/qmd@2.8.3
|
||||||
- name: Install retrieval dependencies
|
|
||||||
run: python -m pip install -r shared/retrieval/requirements.txt
|
|
||||||
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
- uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||||
with:
|
with:
|
||||||
path: .cache/huggingface
|
path: .cache/qmd-models
|
||||||
key: qwen-retrieval-${{ runner.os }}-${{ hashFiles('shared/retrieval/requirements.txt') }}
|
key: qmd-models-${{ runner.os }}-2.8.3-qwen3
|
||||||
- name: Run retrieval tests with real models
|
- name: Run QMD unit tests
|
||||||
|
run: node --test scripts/tests/test-qmd-lib.mjs scripts/tests/test-qmd-daemon.mjs scripts/tests/test-qmd-search.mjs scripts/tests/test-qmd-warm.mjs scripts/tests/test-qmd-benchmark.mjs
|
||||||
|
- name: Run QMD integration test with real models
|
||||||
env:
|
env:
|
||||||
QWEN_TEST_MODEL_CACHE: ${{ github.workspace }}/.cache/huggingface
|
QMD_TEST_CACHE: ${{ github.workspace }}/.cache/qmd-models
|
||||||
HF_HUB_DISABLE_XET: '1'
|
run: node --test scripts/tests/test-qmd-integration.mjs
|
||||||
run: python scripts/tests/test-retrieval-server.py
|
|
||||||
|
|
||||||
security:
|
security:
|
||||||
name: CI configuration security checks
|
name: CI configuration security checks
|
||||||
|
|||||||
@@ -84,8 +84,8 @@ def claude_flags(arm, args):
|
|||||||
if arm == 'baseline':
|
if arm == 'baseline':
|
||||||
flags += ['--setting-sources', '', '--strict-mcp-config']
|
flags += ['--setting-sources', '', '--strict-mcp-config']
|
||||||
if arm == 'setup-no-search':
|
if arm == 'setup-no-search':
|
||||||
flags += ['--disallowedTools', 'Skill(semantic-search)', 'Bash(*semantic-retrieval*)', 'PowerShell(*semantic-retrieval*)',
|
flags += ['--disallowedTools', 'Skill(semantic-search)', 'Bash(*qmd-search*)', 'PowerShell(*qmd-search*)',
|
||||||
'--append-system-prompt', 'The semantic-search skill and the semantic retrieval CLI are unavailable in this session.']
|
'--append-system-prompt', 'The semantic-search skill and the QMD search wrapper are unavailable in this session.']
|
||||||
return flags
|
return flags
|
||||||
|
|
||||||
|
|
||||||
@@ -276,7 +276,7 @@ def preflight(args, clients, arms):
|
|||||||
return provenance, problems
|
return provenance, problems
|
||||||
|
|
||||||
|
|
||||||
RETRIEVAL = Path.home() / '.my-ai-configuration/semantic-retrieval'
|
QMD_SCRIPTS = Path.home() / '.my-ai-configuration/qmd'
|
||||||
FILLER_TOPICS = [
|
FILLER_TOPICS = [
|
||||||
('statement', 'download', 'Render a statement download for the export queue.'),
|
('statement', 'download', 'Render a statement download for the export queue.'),
|
||||||
('trial', 'banner', 'Show the trial banner text in the marketing header.'),
|
('trial', 'banner', 'Show the trial banner text in the marketing header.'),
|
||||||
@@ -303,11 +303,10 @@ def generate_filler(work, count):
|
|||||||
|
|
||||||
|
|
||||||
def warm_index(work):
|
def warm_index(work):
|
||||||
"""Build the semantic index before the session, as in a repository that was searched before."""
|
"""Index the fixture and start the QMD daemon before the session, as in a repository that was searched before."""
|
||||||
server = RETRIEVAL / 'server.py'
|
script = QMD_SCRIPTS / 'qmd-warm.mjs'
|
||||||
if server.is_file():
|
if script.is_file():
|
||||||
subprocess.run([sys.executable, str(server), 'rebuild', '--root', str(work), '--data-dir', str(RETRIEVAL / 'data'),
|
subprocess.run(['node', str(script), '--root', str(work)], capture_output=True, timeout=1800)
|
||||||
'--model-cache', str(RETRIEVAL / 'model-cache')], capture_output=True, timeout=1800)
|
|
||||||
|
|
||||||
|
|
||||||
def check(work, task):
|
def check(work, task):
|
||||||
@@ -377,7 +376,7 @@ def run_one(job, args):
|
|||||||
'agents': sum(turn['agents'] for turn in turns), 'skills': [skill for turn in turns for skill in turn['skills']],
|
'agents': sum(turn['agents'] for turn in turns), 'skills': [skill for turn in turns for skill in turn['skills']],
|
||||||
'commands': [command for turn in turns for command in turn['commands']],
|
'commands': [command for turn in turns for command in turn['commands']],
|
||||||
'models': sorted({model for turn in turns for model in turn['models']}),
|
'models': sorted({model for turn in turns for model in turn['models']}),
|
||||||
'searches': sum('semantic-retrieval' in command for turn in turns for command in turn['commands'])
|
'searches': sum('qmd-search' in command for turn in turns for command in turn['commands'])
|
||||||
+ sum(skill == 'semantic-search' for turn in turns for skill in turn['skills']),
|
+ sum(skill == 'semantic-search' for turn in turns for skill in turn['skills']),
|
||||||
'timed_out': any(turn['timed_out'] for turn in turns), 'workflow_kind': spec['workflow'].get('kind'),
|
'timed_out': any(turn['timed_out'] for turn in turns), 'workflow_kind': spec['workflow'].get('kind'),
|
||||||
'workflow_violations': workflow_violations(spec['workflow'], turns), 'check_output': '' if passed else output,
|
'workflow_violations': workflow_violations(spec['workflow'], turns), 'check_output': '' if passed else output,
|
||||||
@@ -414,7 +413,7 @@ def main():
|
|||||||
parser.add_argument('--clients', nargs='+', choices=('claude', 'codex'), default=['claude', 'codex'])
|
parser.add_argument('--clients', nargs='+', choices=('claude', 'codex'), default=['claude', 'codex'])
|
||||||
parser.add_argument('--arms', nargs='+', choices=('setup', 'baseline', 'setup-no-search'), default=['setup', 'baseline'],
|
parser.add_argument('--arms', nargs='+', choices=('setup', 'baseline', 'setup-no-search'), default=['setup', 'baseline'],
|
||||||
help='setup-no-search is the setup with the semantic-search skill and CLI blocked (Claude only)')
|
help='setup-no-search is the setup with the semantic-search skill and CLI blocked (Claude only)')
|
||||||
parser.add_argument('--warm-index', action='store_true', help='build the semantic index before setup-arm sessions, outside the timing')
|
parser.add_argument('--warm-index', action='store_true', help='start the QMD daemon and index the fixture before setup-arm sessions, outside the timing')
|
||||||
parser.add_argument('--tasks', nargs='+', help='task directory names or patterns such as 11-* (default: all)')
|
parser.add_argument('--tasks', nargs='+', help='task directory names or patterns such as 11-* (default: all)')
|
||||||
parser.add_argument('--repetitions', type=int, default=3)
|
parser.add_argument('--repetitions', type=int, default=3)
|
||||||
parser.add_argument('--first-repetition', type=int, default=1, help='number of the first repetition, to rerun selected repetitions')
|
parser.add_argument('--first-repetition', type=int, default=1, help='number of the first repetition, to rerun selected repetitions')
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
// Real QMD, real models; run only in the qmd CI job or manually.
|
||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join, resolve } from 'node:path';
|
||||||
|
|
||||||
|
const search = resolve('shared/qmd/qmd-search.mjs');
|
||||||
|
|
||||||
|
test('wrapper finds the retry logic through the daemon and stops it', { timeout: 1_800_000 }, () => {
|
||||||
|
const home = mkdtempSync(join(tmpdir(), 'qmd-it-home-'));
|
||||||
|
const repo = join(home, 'repo');
|
||||||
|
mkdirSync(join(repo, 'src'), { recursive: true });
|
||||||
|
writeFileSync(join(repo, 'src', 'upload.ts'), 'export async function sendWithRetry(send) {\n for (let attempt = 1; attempt <= 3; attempt++) {\n try { return await send(); } catch (e) { if (!e.transient || attempt === 3) throw e; }\n }\n}\n');
|
||||||
|
writeFileSync(join(repo, 'src', 'price.ts'), 'export const formatCents = (c) => (c / 100).toFixed(2);\n');
|
||||||
|
execFileSync('git', ['init', '-q'], { cwd: repo });
|
||||||
|
execFileSync('git', ['add', '-A'], { cwd: repo });
|
||||||
|
const env = { ...process.env, HOME: home, USERPROFILE: home, XDG_CONFIG_HOME: join(home, '.config'), XDG_CACHE_HOME: process.env.QMD_TEST_CACHE || join(home, '.cache'), QMD_FORCE_CPU: '1' };
|
||||||
|
execFileSync('node', ['shared/qmd/qmd-config.mjs', 'set-models'], { env });
|
||||||
|
const results = JSON.parse(execFileSync('node', [search, '--root', repo, '--query', 'Where are failed uploads retried after temporary errors?'], { env, encoding: 'utf8', timeout: 1_700_000 }));
|
||||||
|
assert.equal(results[0].path.replace(/\/g, '/'), 'src/upload.ts');
|
||||||
|
execFileSync('node', [search, '--stop'], { env });
|
||||||
|
});
|
||||||
Reference in new issue
Block a user