feat(qwen): split chunks at symbols and sections
This commit is contained in:
1 parent
685dbe61f4
commit
b76a993c75
2 files changed
+92
-2
No files matched your search
@@ -175,6 +175,30 @@ class RetrievalServerTests(unittest.TestCase):
|
||||
reconstructed = windows[0] + [word for window in windows[1:] for word in window[64:]]
|
||||
self.assertEqual(reconstructed, words)
|
||||
|
||||
def test_python_symbols_include_decorators_and_restore_parent(self):
|
||||
source = 'class Service:\n @cached\n def load(self):\n return 1\n\nvalue = 2\n\nasync def refresh():\n return value\n'
|
||||
result = list(MODULE.sections(source, 'service.py'))
|
||||
self.assertEqual([symbol for symbol, _ in result], ['Service', 'Service.load', '<module>', 'refresh'])
|
||||
self.assertIn('@cached', result[1][1])
|
||||
self.assertEqual(''.join(content for _, content in result), source)
|
||||
broken = 'def incomplete(\n' + 'word ' * 450
|
||||
self.assertEqual(len(list(MODULE.chunks(broken, 'broken.py', tokenize))), 2)
|
||||
|
||||
def test_markdown_sections_ignore_fenced_headings(self):
|
||||
source = '# Setup\nIntro\n## Install\n```sh\n# not a heading\n```\nSteps\n## Run\nStart\n'
|
||||
result = list(MODULE.chunks(source, 'guide.md', tokenize))
|
||||
self.assertEqual([symbol for symbol, _ in result], ['Setup', 'Setup / Install', 'Setup / Run'])
|
||||
self.assertIn('# not a heading', result[1][1])
|
||||
|
||||
def test_common_declarations_and_config_sections(self):
|
||||
for path, source, symbols in [
|
||||
('service.ts', 'export function load() {\n return 1;\n}\nexport const save = () => 2;\n', ['load', 'save']),
|
||||
('service.cs', 'public class Service\n{\n public void Load()\n {\n }\n}\n', ['Service', 'Load']),
|
||||
('setup.ps1', 'function Get-Result {\n return 1\n}\n', ['Get-Result']),
|
||||
('config.toml', '[server]\nport = 1\n[client]\nport = 2\n', ['server', 'client']),
|
||||
]:
|
||||
self.assertEqual([symbol for symbol, _ in MODULE.sections(source, path)], symbols)
|
||||
|
||||
def test_old_cache_is_preserved_but_not_reused(self):
|
||||
with TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
|
||||
Reference in new issue
Block a user