From 9ff03305374592ba58f932c60e5da0073874e989 Mon Sep 17 00:00:00 2001 From: Julian lechner Date: Thu, 1 Oct 2026 13:59:11 +0200 Subject: [PATCH] test(config): verify agent limits and permissions take effect Check that Codex keeps the V1 multi-agent implementation that honors max_depth, and, with AI_CONFIG_LIVE_TESTS=1, that Claude withholds the Agent tool from subagents, refuses subagents above the concurrency limit, and blocks git reset --hard in every shell tool. Co-Authored-By: Claude Opus 5.5 --- scripts/tests/test-config-effects.py | 148 +++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 scripts/tests/test-config-effects.py diff --git a/scripts/tests/test-config-effects.py b/scripts/tests/test-config-effects.py new file mode 100644 index 0000000..f85cada --- /dev/null +++ b/scripts/tests/test-config-effects.py @@ -0,0 +1,148 @@ +"""Check that generated agent limits and permissions take effect, not only that the clients accept them. + +Static checks (always, when the CLI is installed): the generated Codex package keeps the V1 multi-agent +implementation that honors agents.max_depth, and Codex reads the configured limits back. +Live checks (only with AI_CONFIG_LIVE_TESTS=1, they start short Haiku sessions): Claude, with only the generated +settings env applied, withholds the Agent tool from subagents and refuses an over-limit burst of parallel subagents; +with only the generated deny rules applied, blocks a destructive git command in every shell tool. +""" +import json +import os +from pathlib import Path +import queue +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import tomllib +import unittest + +ROOT = Path(__file__).resolve().parent.parent.parent +SHELL = 'powershell' if os.name == 'nt' else 'bash' +LIVE = os.environ.get('AI_CONFIG_LIVE_TESTS') == '1' +TIMEOUT = 600 + + +def clean_env(**values): + env = {key: value for key, value in os.environ.items() if not key.startswith(('BUN_INSPECT', 'CLAUDECODE', 'CLAUDE_CODE_ENTRYPOINT', 'CODEX'))} + env.update(values) + return env + + +def stop(process): + if os.name == 'nt': + subprocess.run(['taskkill', '/T', '/F', '/PID', str(process.pid)], capture_output=True, timeout=30) + else: + process.kill() + process.wait(timeout=30) + + +def stream_events(command, cwd, env): + """Collect JSON lines until the process exits or TIMEOUT passes.""" + process = subprocess.Popen(command, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, encoding='utf-8', errors='replace') + lines = queue.Queue() + threading.Thread(target=lambda: ([lines.put(line) for line in process.stdout], lines.put(None)), daemon=True).start() + events, deadline = [], time.monotonic() + TIMEOUT + try: + while True: + line = lines.get(timeout=max(1, deadline - time.monotonic())) + if line is None: + return events + try: + events.append(json.loads(line)) + except ValueError: + pass + except queue.Empty: + raise AssertionError(f'client did not finish within {TIMEOUT}s') from None + finally: + if process.poll() is None: + stop(process) + + +class CodexAgentLimitTests(unittest.TestCase): + @unittest.skipUnless(shutil.which('codex'), 'Codex CLI unavailable') + def test_generated_limits_use_the_implementation_that_honors_them(self): + generated = (ROOT / f'generated/codex-{SHELL}/config.toml').read_text(encoding='utf-8-sig') + config = tomllib.loads(generated.replace('__AI_CONFIG_ROOT__', '/x').replace('__HOOK_COMMAND__', 'x') + .replace('__POWERSHELL_HOOK_COMMAND__', 'x').replace('__POWERSHELL_COMMAND__', 'x')) + self.assertEqual(config['agents']['max_depth'], 1) + with tempfile.TemporaryDirectory() as home: + (Path(home) / 'config.toml').write_text(generated.replace('__AI_CONFIG_ROOT__', Path(home).as_posix()), encoding='utf-8') + listing = subprocess.run([shutil.which('codex'), 'features', 'list'], env=clean_env(CODEX_HOME=home), + capture_output=True, text=True, timeout=120).stdout + states = {line.split()[0]: line.split()[-1] for line in listing.splitlines() if len(line.split()) >= 2} + # Codex documents agents.max_depth as a limit of the V1 multi-agent implementation; V2 ignores it. + self.assertEqual(states.get('multi_agent'), 'true', listing) + self.assertEqual(states.get('multi_agent_v2'), 'false', 'multi_agent_v2 is enabled, so agents.max_depth has no effect') + + +class ClaudeAgentLimitTests(unittest.TestCase): + """Live checks with forced test agents, so the result does not depend on how the model paraphrases a task.""" + + AGENTS = { + 'spawner': {'description': 'Test agent that starts another agent.', 'tools': ['Agent'], + 'prompt': 'Your first action: call the Agent tool with subagent_type "general-purpose", description "nested", ' + 'prompt "Reply OK", and run_in_background false. Then output the exact tool result text verbatim.'}, + 'sleeper': {'description': 'Test agent that replies once.', 'tools': [], 'prompt': 'Reply with the word OK. Do not use tools.'}, + } + + def setUp(self): + if not (LIVE and shutil.which('claude')): + self.skipTest('set AI_CONFIG_LIVE_TESTS=1 with the Claude CLI installed') + settings = json.loads((ROOT / f'generated/claude-{SHELL}/settings.json').read_text(encoding='utf-8-sig')) + self.env = {key: value for key, value in settings['env'].items() if not value.startswith('__')} + + def run_claude(self, prompt, env): + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as work: + subprocess.run(['git', 'init', '-q'], cwd=work, check=True, timeout=60) + events = stream_events([shutil.which('claude'), '-p', prompt, '--output-format', 'stream-json', '--verbose', '--model', 'haiku', + '--setting-sources', '', '--strict-mcp-config', '--settings', json.dumps({'env': env}), + '--agents', json.dumps(self.AGENTS), '--permission-mode', 'auto', '--allowedTools', 'Agent'], work, clean_env()) + return json.dumps(events) + + def test_spawn_depth_removes_the_agent_tool_from_subagents(self): + self.assertEqual(self.env.get('CLAUDE_CODE_MAX_SUBAGENT_SPAWN_DEPTH'), '1') + prompt = ('Call the Agent tool exactly once with subagent_type "spawner", description "spawn test", prompt "go", and ' + 'run_in_background false. Wait for its result, then quote it verbatim.') + limited = self.run_claude(prompt, self.env) + self.assertTrue('matched no tools in this session [Agent]' in limited or 'Subagent nesting limit reached' in limited, + 'a subagent still had the Agent tool') + # Control: without the limit the same agent reaches its nested agent, so the check above can fail. + unlimited = self.run_claude(prompt, {key: value for key, value in self.env.items() if key != 'CLAUDE_CODE_MAX_SUBAGENT_SPAWN_DEPTH'}) + self.assertNotIn('matched no tools in this session [Agent]', unlimited) + + def test_concurrency_limit_refuses_the_extra_parallel_subagent(self): + limit = int(self.env['CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS']) + prompt = (f'In ONE single message, call the Agent tool {limit + 1} times in parallel with subagent_type "sleeper" and prompt "go". ' + 'Then quote every tool result and error verbatim.') + self.assertIn(f'You can run {limit} subagents at once', self.run_claude(prompt, self.env)) + + +class ClaudePermissionTests(unittest.TestCase): + def test_deny_rules_block_destructive_git_in_every_shell_tool(self): + if not (LIVE and shutil.which('claude')): + self.skipTest('set AI_CONFIG_LIVE_TESTS=1 with the Claude CLI installed') + deny = json.loads((ROOT / f'generated/claude-{SHELL}/settings.json').read_text(encoding='utf-8-sig'))['permissions']['deny'] + for tool in ('Bash', 'PowerShell') if os.name == 'nt' else ('Bash',): + with self.subTest(tool), tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as work: + for command in (['init', '-q'], ['add', '-A'], ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-qm', 'c', '--allow-empty']): + subprocess.run(['git', *command], cwd=work, check=True, timeout=60) + Path(work, 'tracked.txt').write_text('v1') + subprocess.run(['git', 'add', '-A'], cwd=work, check=True, timeout=60) + subprocess.run(['git', '-c', 'user.name=t', '-c', 'user.email=t@example.invalid', 'commit', '-qm', 'v1'], cwd=work, check=True, timeout=60) + Path(work, 'tracked.txt').write_text('local edit') + # bypassPermissions isolates the deny rules: nothing else may stop the command. + events = stream_events([shutil.which('claude'), '-p', f'Use the {tool} tool to run exactly this command and nothing else: git reset --hard', + '--output-format', 'stream-json', '--verbose', '--model', 'haiku', '--setting-sources', '', '--strict-mcp-config', + '--settings', json.dumps({'permissions': {'deny': deny}}), '--permission-mode', 'bypassPermissions'], work, clean_env()) + attempts = [block for event in events if event.get('type') == 'assistant' for block in event.get('message', {}).get('content', []) + if isinstance(block, dict) and block.get('type') == 'tool_use' and block.get('name') == tool] + self.assertTrue(attempts, f'the model never tried the {tool} tool, so the deny rule was not exercised') + self.assertEqual(Path(work, 'tracked.txt').read_text(), 'local edit', f'git reset --hard ran through the {tool} tool') + + +if __name__ == '__main__': + unittest.main()