feat(qwen): split chunks at symbols and sections

This commit is contained in:
Julian lechner committed 2026-09-17 15:53:48 +02:00
1 parent 685dbe61f4
commit b76a993c75
2 files changed
+92 -2

No files matched your search

+24
View File
@@ -175,6 +175,30 @@ class RetrievalServerTests(unittest.TestCase):
reconstructed = windows[0] + [word for window in windows[1:] for word in window[64:]]
self.assertEqual(reconstructed, words)
def test_python_symbols_include_decorators_and_restore_parent(self):
source = 'class Service:\n @cached\n def load(self):\n return 1\n\nvalue = 2\n\nasync def refresh():\n return value\n'
result = list(MODULE.sections(source, 'service.py'))
self.assertEqual([symbol for symbol, _ in result], ['Service', 'Service.load', '<module>', 'refresh'])
self.assertIn('@cached', result[1][1])
self.assertEqual(''.join(content for _, content in result), source)
broken = 'def incomplete(\n' + 'word ' * 450
self.assertEqual(len(list(MODULE.chunks(broken, 'broken.py', tokenize))), 2)
def test_markdown_sections_ignore_fenced_headings(self):
source = '# Setup\nIntro\n## Install\n```sh\n# not a heading\n```\nSteps\n## Run\nStart\n'
result = list(MODULE.chunks(source, 'guide.md', tokenize))
self.assertEqual([symbol for symbol, _ in result], ['Setup', 'Setup / Install', 'Setup / Run'])
self.assertIn('# not a heading', result[1][1])
def test_common_declarations_and_config_sections(self):
for path, source, symbols in [
('service.ts', 'export function load() {\n return 1;\n}\nexport const save = () => 2;\n', ['load', 'save']),
('service.cs', 'public class Service\n{\n public void Load()\n {\n }\n}\n', ['Service', 'Load']),
('setup.ps1', 'function Get-Result {\n return 1\n}\n', ['Get-Result']),
('config.toml', '[server]\nport = 1\n[client]\nport = 2\n', ['server', 'client']),
]:
self.assertEqual([symbol for symbol, _ in MODULE.sections(source, path)], symbols)
def test_old_cache_is_preserved_but_not_reused(self):
with TemporaryDirectory() as temporary:
root = Path(temporary)