feat(retrieval): make the file-scan limit configurable and warn on it

Repositories with more than 2000 indexable files silently lost the
remainder during indexing. Make the cap configurable via
SEMANTIC_RETRIEVAL_MAX_FILES (default unchanged) and print an explicit
stderr warning naming the repository when it is reached, instead of
truncating without any signal.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

scripts/test-retrieval-server.py

@@ -160,7 +160,7 @@ class RetrievalServerTests(unittest.TestCase):
             finally:
                 index.db.close()
 
-    def test_scan_prunes_ignored_directories_and_stops_at_limit(self):
+    def test_scan_prunes_ignored_directories_and_warns_at_limit(self):
         with TemporaryDirectory() as temporary:
             root = Path(temporary)
             index = MODULE.Index(root, root / 'data')
@@ -172,8 +172,31 @@ class RetrievalServerTests(unittest.TestCase):
             try:
                 self.assertEqual(index.files(), [(root / 'visible.md').resolve()])
                 filenames = ['visible.md'] * 2001
-                with patch.object(MODULE.os, 'walk', return_value=iter([(str(root), [], filenames)])):
+                with patch.object(MODULE.os, 'walk', return_value=iter([(str(root), [], filenames)])), \
+                     patch.object(MODULE.sys, 'stderr') as stderr:
                     self.assertEqual(len(index.files()), 2000)
+                self.assertIn('stopped scanning after 2000', stderr.write.call_args_list[0].args[0])
+            finally:
+                index.db.close()
+
+    def test_max_files_is_configurable_via_environment(self):
+        with TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            index = MODULE.Index(root, root / 'data')
+            try:
+                with patch.dict(os.environ, {MODULE.MAX_FILES_ENV: '3'}):
+                    self.assertEqual(MODULE.read_max_files(), 3)
+                with patch.dict(os.environ, {MODULE.MAX_FILES_ENV: 'not-a-number'}):
+                    with self.assertRaises(ValueError):
+                        MODULE.read_max_files()
+                with patch.dict(os.environ, {}, clear=True):
+                    self.assertEqual(MODULE.read_max_files(), MODULE.DEFAULT_MAX_FILES)
+                (root / 'visible.md').write_text('visible', encoding='utf-8')
+                index.max_files = 3
+                filenames = ['visible.md'] * 5
+                with patch.object(MODULE.os, 'walk', return_value=iter([(str(root), [], filenames)])), \
+                     patch.object(MODULE.sys, 'stderr'):
+                    self.assertEqual(len(index.files()), 3)
             finally:
                 index.db.close()
 

shared/retrieval/server.py

@@ -31,6 +31,8 @@ RRF_K = 60
 INDEX_VERSION = 2
 RETRIEVAL_INSTRUCTION = "Given a codebase question, retrieve relevant code and documentation that answer the question."
 TEXT_EXTENSIONS = {".c", ".cpp", ".cs", ".go", ".java", ".js", ".json", ".md", ".py", ".ps1", ".rs", ".sh", ".toml", ".ts", ".tsx", ".txt", ".yaml", ".yml"}
+DEFAULT_MAX_FILES = 2000
+MAX_FILES_ENV = "SEMANTIC_RETRIEVAL_MAX_FILES"
 
 
 def runtime_python(script=__file__, executable=sys.executable, system=os.name):
@@ -148,6 +150,19 @@ def preferred_device():
     return "cuda" if torch.cuda.is_available() else "cpu"
 
 
+def read_max_files():
+    value = os.environ.get(MAX_FILES_ENV)
+    if value is None:
+        return DEFAULT_MAX_FILES
+    try:
+        parsed = int(value)
+    except ValueError:
+        parsed = 0
+    if parsed <= 0:
+        raise ValueError(f"{MAX_FILES_ENV} must be a positive integer, got {value!r}")
+    return parsed
+
+
 def reciprocal_rank_fusion(*rankings):
     scores = {}
     for ranking in rankings:
@@ -174,6 +189,7 @@ class Index:
         self.lock = threading.RLock()
         self.model = None
         self.reranker_model = None
+        self.max_files = read_max_files()
 
     def files(self):
         ignored = {".git", ".venv", "generated", "node_modules", ".my-ai-configuration"}
@@ -188,7 +204,10 @@ class Index:
                 if path.is_symlink() or path.suffix.lower() not in TEXT_EXTENSIONS or not path.is_file() or path.stat().st_size > 524288:
                     continue
                 result.append(path)
-                if len(result) == 2000:
+                if len(result) == self.max_files:
+                    print(f"Semantic retrieval: stopped scanning after {self.max_files} indexable files; "
+                          f"some files under {self.root} were not indexed. Set {MAX_FILES_ENV} to raise this limit.",
+                          file=sys.stderr)
                     return result
         return result