diff --git a/scripts/tests/test-client-smoke.py b/scripts/tests/test-client-smoke.py new file mode 100644 index 0000000..9eb00f7 --- /dev/null +++ b/scripts/tests/test-client-smoke.py @@ -0,0 +1,165 @@ +"""Behavioral client smoke tests against an isolated installation. + +Each test installs a generated package into a temporary home that already holds +foreign settings, then asks the real client CLI what it discovered. Claude runs +with an invalid API key and is stopped at its init event; Codex is queried +through `codex app-server`. No model request completes. +""" +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parent.parent.parent +SHELL = 'powershell' if os.name == 'nt' else 'bash' +MARKER_SCRIPT = "import sys; open(sys.argv[1], 'w').write('ok')" + + +def frontmatter_names(directory, pattern): + names = set() + for path in directory.rglob(pattern): + match = re.search(r'(?m)^name\s*[:=]\s*"?([^"\r\n]+)"?\s*$', path.read_text(encoding='utf-8-sig')) + if match: + names.add(match.group(1).strip()) + return names + + +def marker_command(marker): + return f'"{sys.executable}" -c "{MARKER_SCRIPT}" "{marker}"' + + +def clean_env(**values): + env = {key: value for key, value in os.environ.items() + if not key.startswith(('BUN_INSPECT', 'CLAUDE', 'ANTHROPIC', 'CODEX', 'OPENAI'))} + env.update(values) + return env + + +def install(client, home): + if SHELL == 'powershell': + script = ('$ErrorActionPreference = "Stop"; ' + f'. "{ROOT}/scripts/lib/manifest.ps1"; . "{ROOT}/scripts/lib/install-targets.ps1"; ' + '$command = "powershell -NoProfile -ExecutionPolicy Bypass -File"; ' + f'foreach ($item in (Get-InstallTargets -Generated "{ROOT}/generated" -HomePath "{home}" -Shell PowerShell -Client {client})) {{ ' + 'Sync-ManagedDestination -Source $item.Source -Destination $item.Destination -Stamp smoke ' + '-AiConfigRoot $item.Destination.Replace("\\","/") -ShellCommand $command -PowerShellCommand $command -Summary | Out-Null }') + args = ['powershell', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', script] + else: + script = (f'set -e; . "{ROOT}/scripts/lib/manifest.sh"; . "{ROOT}/scripts/lib/install-targets.sh"; ' + f'get_install_targets "{ROOT}/generated" "{home}" bash {client.lower()} | ' + 'while IFS="|" read -r source destination; do ' + 'sync_managed_destination "$source" "$destination" smoke "$destination" bash "pwsh -NoProfile -File" false true >/dev/null; done') + args = ['bash', '-c', script] + result = subprocess.run(args, capture_output=True, text=True) + if result.returncode: + raise RuntimeError(f"install failed: {result.stdout}{result.stderr}") + + +class ClientSmokeTests(unittest.TestCase): + def setUp(self): + self.base = Path(tempfile.mkdtemp(prefix='ai-config-client-smoke-')) + self.home = self.base / 'home' + self.work = self.base / 'work' + self.work.mkdir(parents=True) + subprocess.run(['git', 'init', '-q'], cwd=self.work, check=True) + self.marker = self.base / 'foreign-hook.txt' + + def tearDown(self): + shutil.rmtree(self.base, ignore_errors=True) + + @unittest.skipUnless(shutil.which('claude'), 'Claude CLI unavailable') + def test_claude_discovers_package_and_keeps_foreign_settings(self): + package = ROOT / f'generated/claude-{SHELL}' + config = self.home / '.claude' + config.mkdir(parents=True) + foreign_hook = {'hooks': [{'type': 'command', 'command': marker_command(self.marker), 'timeout': 10}]} + (config / 'settings.json').write_text(json.dumps({'foreignSetting': 'keep', 'hooks': {'SessionStart': [foreign_hook]}})) + install('Claude', self.home) + + process = subprocess.Popen([shutil.which('claude'), '-p', 'smoke', '--output-format', 'stream-json', '--verbose', '--max-turns', '1'], + cwd=self.work, env=clean_env(CLAUDE_CONFIG_DIR=str(config), ANTHROPIC_API_KEY='sk-ant-invalid-smoke'), + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding='utf-8') + events = [] + try: + for line in process.stdout: + events.append(json.loads(line)) + if events[-1].get('subtype') == 'init': + break + finally: + process.kill() + for stream in (process.stdin, process.stdout): + if stream: + stream.close() + process.wait(timeout=30) + init = next((event for event in events if event.get('subtype') == 'init'), None) + self.assertIsNotNone(init, 'Claude emitted no init event') + + with self.subTest('skills'): + self.assertEqual(sorted(frontmatter_names(package / 'skills', 'SKILL.md') - set(init['skills'])), []) + with self.subTest('agents'): + self.assertEqual(sorted(frontmatter_names(package / 'agents', '*.md') - set(init['agents'])), []) + with self.subTest('hooks'): + failed = [(event.get('hook_name'), event.get('exit_code'), event.get('stderr', '')[:200]) + for event in events if event.get('subtype') == 'hook_response' and event.get('exit_code') != 0] + self.assertEqual(failed, []) + self.assertTrue(self.marker.exists(), 'foreign SessionStart hook did not run') + with self.subTest('foreign settings'): + self.assertEqual(json.loads((config / 'settings.json').read_text(encoding='utf-8-sig')).get('foreignSetting'), 'keep') + + @unittest.skipUnless(shutil.which('codex'), 'Codex CLI unavailable') + def test_codex_discovers_package_and_keeps_foreign_settings(self): + package = ROOT / f'generated/codex-{SHELL}' + codex_home = self.home / '.codex' + codex_home.mkdir(parents=True) + command = json.dumps(marker_command(self.marker)) + (codex_home / 'config.toml').write_text( + '[mcp_servers.smoke_demo]\ncommand = "smoke-demo"\nenabled = false\n\n' + f'[[hooks.SessionStart]]\n[[hooks.SessionStart.hooks]]\ntype = "command"\ncommand = {command}\ntimeout = 10\n') + install('Codex', self.home) + + process = subprocess.Popen([shutil.which('codex'), 'app-server'], cwd=self.work, + env=clean_env(CODEX_HOME=str(codex_home), OPENAI_API_KEY='sk-invalid-smoke'), + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, encoding='utf-8') + + def call(request_id, method, params): + process.stdin.write(json.dumps({'jsonrpc': '2.0', 'id': request_id, 'method': method, 'params': params}) + '\n') + process.stdin.flush() + while True: + message = json.loads(process.stdout.readline()) + if message.get('id') == request_id: + self.assertNotIn('error', message, method) + return message['result'] + + try: + call(1, 'initialize', {'clientInfo': {'name': 'ai-config-smoke', 'version': '1'}}) + process.stdin.write(json.dumps({'jsonrpc': '2.0', 'method': 'initialized'}) + '\n') + # Codex resolves ~/.agents/skills from the real Windows profile, so register the isolated root explicitly. + skills_root = self.home / '.agents/skills' + call(2, 'skills/extraRoots/set', {'extraRoots': [str(skills_root)]}) + skills = call(3, 'skills/list', {'cwds': [str(self.work)], 'forceReload': True})['data'][0]['skills'] + hooks = call(4, 'hooks/list', {'cwds': [str(self.work)]})['data'][0]['hooks'] + config = call(5, 'config/read', {})['config'] + finally: + process.kill() + for stream in (process.stdin, process.stdout): + if stream: + stream.close() + process.wait(timeout=30) + + isolated = {skill['name'] for skill in skills if Path(skill['path']).resolve().is_relative_to(skills_root.resolve())} + self.assertEqual(sorted(frontmatter_names(package / 'skills', 'SKILL.md') - isolated), [], 'skills not discovered') + events = {(hook['eventName'], hook['enabled']) for hook in hooks if hook.get('source') == 'user'} + self.assertIn(('stop', True), events, 'managed Stop hook not registered') + self.assertIn(('sessionStart', True), events, 'foreign SessionStart hook lost') + self.assertEqual(config['mcp_servers']['smoke_demo']['command'], 'smoke-demo') + self.assertTrue((codex_home / 'agents').is_dir() and frontmatter_names(codex_home / 'agents', '*.toml') == frontmatter_names(package / 'agents', '*.toml')) + + +if __name__ == '__main__': + unittest.main()