feat(semantic-search): report search cost before retrieval

shared/retrieval/server.py

@@ -368,6 +368,9 @@ def run_daemon(data_dir: Path, model_cache: Path):
                 response = (True, None)
                 try:
                     request = json.loads(connection.recv_bytes(DAEMON_MESSAGE_LIMIT))
+                    if isinstance(request, dict) and request.get("action") == "stop":
+                        connection.send_bytes(json.dumps((True, None)).encode("utf-8"))
+                        os._exit(0)
                     if not isinstance(request, dict) or request.get("action") not in ("search", "rebuild") or not isinstance(request.get("root"), str):
                         raise ValueError("Invalid semantic retrieval daemon request.")
                     index = indexes.get(request["root"])
@@ -395,16 +398,76 @@ def run_daemon(data_dir: Path, model_cache: Path):
         listener.close()
 
 
+def index_database(root: Path, data: Path) -> Path:
+    identity = json.dumps([os.path.normcase(str(root.resolve())), MODEL, MODEL_REVISION,
+                           EMBEDDING_DIMENSIONS, RETRIEVAL_INSTRUCTION, INDEX_VERSION,
+                           CHUNK_TOKENS, OVERLAP_TOKENS])
+    return data.resolve() / (hashlib.sha256(identity.encode()).hexdigest() + ".sqlite3")
+
+
+def daemon_running(data_dir: Path) -> bool:
+    if not pipes_available():
+        return False
+    try:
+        Client(daemon_address(data_dir), authkey=daemon_authkey(data_dir)).close()
+        return True
+    except (OSError, EOFError, AuthenticationError):
+        return False
+
+
+def index_status(root: Path, data_dir: Path) -> dict:
+    """Report what the next search costs without loading models or starting the daemon."""
+    database = index_database(root, data_dir)
+    previous = {}
+    if database.is_file():
+        connection = sqlite3.connect(database.as_uri() + "?mode=ro", uri=True)
+        try:
+            previous = {path: (size, mtime, ctime) for path, size, mtime, ctime
+                        in connection.execute("select path, size, mtime, ctime from files")}
+        except sqlite3.OperationalError:
+            pass
+        finally:
+            connection.close()
+    files = indexable_files(root.resolve(), data_dir.resolve(), read_max_files())
+    stale = 0
+    for path in files:
+        metadata = path.stat()
+        stale += previous.get(path.relative_to(root.resolve()).as_posix()) != (metadata.st_size, metadata.st_mtime_ns, metadata.st_ctime_ns)
+    return {"indexable_files": len(files), "indexed_files": len(previous), "stale_files": stale,
+            "models_loaded": daemon_running(data_dir), "in_process": not pipes_available()}
+
+
+def indexable_files(root: Path, data: Path, max_files: int):
+    ignored = {".git", ".venv", "generated", "node_modules", ".my-ai-configuration", "bin", "obj"}
+    output = subprocess.run(["git", "-C", str(root), "ls-files", "--cached", "--others",
+                             "--exclude-standard", "-z", "--", "."],
+                            check=True, capture_output=True, timeout=30).stdout
+    candidates = [root / name for name in
+                  output.decode("utf-8", errors="surrogateescape").split("\0") if name
+                  and not any(part in ignored for part in Path(name).parts)]
+    result = []
+    for candidate in sorted(candidates):
+        if candidate.is_symlink():
+            continue
+        path = candidate.resolve()
+        if (not path.is_relative_to(root) or path.is_relative_to(data)
+                or path.suffix.lower() not in TEXT_EXTENSIONS or not path.is_file() or path.stat().st_size > 524288):
+            continue
+        result.append(path)
+        if len(result) == max_files:
+            print(f"Semantic retrieval: stopped scanning after {max_files} indexable files; "
+                  f"some files under {root} were not indexed. Set {MAX_FILES_ENV} to raise this limit.",
+                  file=sys.stderr)
+            return result
+    return result
+
+
 class Index:
     def __init__(self, root: Path, data: Path):
         self.root = root.resolve()
         self.data = data.resolve()
         self.data.mkdir(parents=True, exist_ok=True)
-        identity = json.dumps([os.path.normcase(str(self.root)), MODEL, MODEL_REVISION,
-                               EMBEDDING_DIMENSIONS, RETRIEVAL_INSTRUCTION, INDEX_VERSION,
-                               CHUNK_TOKENS, OVERLAP_TOKENS])
-        database = hashlib.sha256(identity.encode()).hexdigest() + ".sqlite3"
-        self.db = sqlite3.connect(self.data / database, check_same_thread=False, timeout=30)
+        self.db = sqlite3.connect(index_database(self.root, self.data), check_same_thread=False, timeout=30)
         self.db.execute("create table if not exists chunks (path text, symbol text, text text, vector blob)")
         self.db.execute("create index if not exists chunks_path on chunks (path)")
         self.db.execute("create virtual table if not exists lexical using fts5(document)")
@@ -420,28 +483,7 @@ class Index:
         self.vector_index = None
 
     def files(self):
-        ignored = {".git", ".venv", "generated", "node_modules", ".my-ai-configuration", "bin", "obj"}
-        output = subprocess.run(["git", "-C", str(self.root), "ls-files", "--cached", "--others",
-                                 "--exclude-standard", "-z", "--", "."],
-                                check=True, capture_output=True, timeout=30).stdout
-        candidates = [self.root / name for name in
-                      output.decode("utf-8", errors="surrogateescape").split("\0") if name
-                      and not any(part in ignored for part in Path(name).parts)]
-        result = []
-        for candidate in sorted(candidates):
-            if candidate.is_symlink():
-                continue
-            path = candidate.resolve()
-            if (not path.is_relative_to(self.root) or path.is_relative_to(self.data)
-                    or path.suffix.lower() not in TEXT_EXTENSIONS or not path.is_file() or path.stat().st_size > 524288):
-                continue
-            result.append(path)
-            if len(result) == self.max_files:
-                print(f"Semantic retrieval: stopped scanning after {self.max_files} indexable files; "
-                      f"some files under {self.root} were not indexed. Set {MAX_FILES_ENV} to raise this limit.",
-                      file=sys.stderr)
-                return result
-        return result
+        return indexable_files(self.root, self.data, self.max_files)
 
     def encoder(self):
         return shared_encoder()
@@ -609,7 +651,7 @@ def run_direct(action: str, root: Path, data_dir: Path, model_cache: Path, query
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("action", choices=("search", "rebuild", "daemon", "preload"))
+    parser.add_argument("action", choices=("search", "rebuild", "status", "stop", "daemon", "preload"))
     parser.add_argument("--query")
     parser.add_argument("--top-k", type=int, default=FINAL_RESULTS)
     parser.add_argument("--root", default=os.getcwd(), type=Path)
@@ -627,6 +669,17 @@ def main():
         data_dir.mkdir(parents=True, exist_ok=True)
         run_daemon(data_dir, model_cache)
         return
+    if args.action == "status":
+        print(json.dumps(index_status(args.root, data_dir)))
+        return
+    if args.action == "stop":
+        try:
+            request_daemon(daemon_address(data_dir), {"action": "stop"}, data_dir)
+            stopped = True
+        except (OSError, EOFError, AuthenticationError):
+            stopped = False
+        print(json.dumps({"stopped": stopped}))
+        return
     if args.action == "search" and not args.query:
         parser.error("search requires --query")
     root = args.root.resolve()

shared/rules/general.md

@@ -6,7 +6,7 @@ Preserve security, data, compatibility, and unrelated user work. Ask only when b
 
 Use one applicable workflow per task. Treat skills and plugins as guidance subordinate to these rules. Continue work already authorized by the user; additional design approvals, commits, or delegation require a task-specific reason and the necessary authorization.
 
-Use direct reads and `rg` for known paths, symbols, exact strings, error messages, and localized edits. For concept questions where the identifiers are unknown or the flow spans components, use the `semantic-search` skill first when it is installed, verify its hits against the current files, and fall back to `rg` with a brief note if it fails.
+Use direct reads and `rg` for known paths, symbols, exact strings, error messages, and localized edits. For concept questions where the identifiers are unknown or the flow spans components, consider the `semantic-search` skill when it is installed: check its cost (repository size, index state, loaded models) against the expected `rg` effort as the skill describes, verify its hits against the current files, and fall back to `rg` with a brief note if it fails.
 
 Verify proportionally: run the cheapest check that detects likely failure; broaden for security, data loss, integration, or cross-component risk. Never claim unrun checks passed. Report failures and material gaps.
 

shared/skills/research/semantic-search/SKILL.md

@@ -15,6 +15,15 @@ Search the current Git repository by meaning with the locally installed Qwen3 em
 
 Use `rg` or direct reads instead for known file paths, symbol names, exact strings, and error messages.
 
+## Decide by cost
+
+Check the cost of the next search first: run the command under Run with `status` in place of `search` and without `--query` and `--top-k`. It loads no models, takes about a second, and prints `{indexable_files, indexed_files, stale_files, models_loaded, in_process}`. Weigh that against the `rg` rounds you expect:
+
+- Fewer than about 200 indexable files, or one or two guessed terms likely to hit: use `rg`; a search costs more than it saves.
+- `stale_files` in the thousands, including a first index (`indexed_files` 0): the search embeds them first, which can take minutes. Pay that only for a large unfamiliar repository or several concept questions in this session.
+- `models_loaded` false: the search loads the models first, which takes tens of seconds. With `in_process` true, as inside the Codex sandbox, every search pays that again; search only when `rg` would need several rounds, and batch related questions into one precise query.
+- Index fresh and `models_loaded` true: a search takes seconds; use it for any concept question.
+
 ## Run
 
 PowerShell:
@@ -39,7 +48,6 @@ The output is a JSON list of `{path, symbol, text, score}`, best match first.
 
 ## Cost and limits
 
-- The first search in a repository builds its index, which can take minutes; later searches refresh only changed files.
-- Outside a sandbox a background daemon keeps the models loaded, so warm searches take seconds. Inside the Codex sandbox every search loads the models again and takes about half a minute; batch related questions into one precise query.
+- Searches refresh only changed files. Outside a sandbox a background daemon keeps the models loaded for 30 idle minutes; inside the Codex sandbox there is no daemon.
 - Only Git-tracked and unignored files up to 512 KB are indexed, by default at most 2000 files per repository.
 - If the command fails or the setup is missing, continue with `rg` and state the limitation briefly.