semcode-mcp 0.1.0__tar.gz → 0.1.1__tar.gz

{semcode_mcp-0.1.0 → semcode_mcp-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: semcode-mcp
-Version: 0.1.0
+Version: 0.1.1
 Summary: Semantic code search MCP server: Tree-sitter AST chunking + voyage-code-3 embeddings + hybrid retrieval (sqlite-vec + FTS5 + RRF) + Voyage rerank-2.5 + call graph expansion
 Project-URL: Homepage, https://github.com/huawang1258/semantic-code-mcp
 Project-URL: Repository, https://github.com/huawang1258/semantic-code-mcp

{semcode_mcp-0.1.0 → semcode_mcp-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "semcode-mcp"
-version = "0.1.0"
+version = "0.1.1"
 description = "Semantic code search MCP server: Tree-sitter AST chunking + voyage-code-3 embeddings + hybrid retrieval (sqlite-vec + FTS5 + RRF) + Voyage rerank-2.5 + call graph expansion"
 readme = "README.md"
 license = "MIT"

{semcode_mcp-0.1.0 → semcode_mcp-0.1.1}/src/semantic_code_mcp/embedder.py

@@ -81,14 +81,25 @@ class Embedder:
 
     def embed_query(self, text: str) -> list[float]:
         """检索用：embedding 单条查询（带 LRU 缓存）。"""
-        if text in self._query_cache:
-            self._query_cache.move_to_end(text)
-            return self._query_cache[text]
-        result = self._embed([text], "query")[0]
-        self._query_cache[text] = result
-        if len(self._query_cache) > self._query_cache_max:
-            self._query_cache.popitem(last=False)
-        return result
+        return self.embed_queries([text])[0]
+
+    def embed_queries(self, texts: list[str]) -> list[list[float]]:
+        """检索用：批量 embedding 多条查询，与输入一一对应。
+
+        复合查询主查询 + 子查询合并成一次请求（省 N-1 次网络往返）；
+        命中缓存的不重发，只有未命中子集上行。
+        """
+        missing = [t for t in dict.fromkeys(texts) if t not in self._query_cache]
+        if missing:
+            for t, vec in zip(missing, self._embed(missing, "query")):
+                self._query_cache[t] = vec
+                if len(self._query_cache) > self._query_cache_max:
+                    self._query_cache.popitem(last=False)
+        out: list[list[float]] = []
+        for t in texts:
+            self._query_cache.move_to_end(t)
+            out.append(self._query_cache[t])
+        return out
 
     def _embed(self, texts: list[str], input_type: str) -> list[list[float]]:
         results: list[list[float]] = []
@@ -183,6 +194,10 @@ class LocalEmbedder:
             self._query_cache.popitem(last=False)
         return result
 
+    def embed_queries(self, texts: list[str]) -> list[list[float]]:
+        """与 Embedder.embed_queries 同接口；本地推理无网络往返，逐条等价。"""
+        return [self.embed_query(t) for t in texts]
+
 
 def create_embedder(**kwargs):
     """按 SCM_EMBED_BACKEND 选择后端：voyage（默认）/ local。"""

{semcode_mcp-0.1.0 → semcode_mcp-0.1.1}/src/semantic_code_mcp/retriever.py

@@ -17,6 +17,7 @@ import fnmatch
 import os
 import re
 import time
+from concurrent.futures import ThreadPoolExecutor
 
 from .embedder import Embedder
 from .expander import create_expander
@@ -141,12 +142,19 @@ _COMPOUND_SUB_WEIGHT = 0.7
 _COMPOUND_PROBE_TOP = 10
 # 子查询 rerank 分数混合：探针块用整句 rerank 分天然吃亏（"清理 100 天"
 # 在整句里只占小半权重），单独用子查询文本再 rerank 一次，
-# 最终分取 max(整句分, 子查询分 × 此系数)
-_COMPOUND_SUB_RERANK_MIX = 0.9
-# 子查询分参与混合/保底的最低门槛：实测"清理"这种宽泛短词的字面命中
-# （clearCache/clear 工具方法）全部 ≤0.14，真业务清理实现 ≥0.22，
-# 0.2 是天然分界线；低于门槛的子查询信号不参与任何提分
-_COMPOUND_SUB_MIN_SCORE = 0.2
+# 混合分 = 整句 top 分 × 子查询相对强度 × 此系数（子意图最佳代表
+# ≈ 整句最佳的 95 折；Voyage 分数域窄，0.9 的提拔量不足以越过
+# 中间名次的密集分数带，0.95 实测 K1 探针 rank5→rank4）
+_COMPOUND_SUB_RERANK_MIX = 0.95
+# 子查询分参与混合/保底的最低门槛，按 rerank 后端分档标定（分数分布不同，
+# 绝对阈值跨后端不通用）：
+# - cohere（rerank-v3.5）分布宽：实测"清理"这种宽泛短词的字面命中
+#   （clearCache/clear 工具方法）全部 ≤0.14，真业务清理实现 ≥0.22，0.2 是分界线
+# - voyage（rerank-2.5）分布窄且整体偏高：无关块 <0.5，字面噪声（0.63）与
+#   真业务（0.645）重叠不可分——0.55 只拦"彻底无关"，字面噪声由
+#   "子意图已有代表则不动"（6.5）与 must_contain 探针兜底
+_COMPOUND_SUB_MIN_SCORE = {"voyage": 0.55, "cohere": 0.2}
+_COMPOUND_SUB_MIN_DEFAULT = 0.2
 
 
 class Retriever:
@@ -355,7 +363,10 @@ class Retriever:
         把调用者/被调用者作为关联结果附加（带 relation 字段）。
         """
         # 1. 向量召回 + 2. BM25 召回（原查询）
-        q_emb = self.embedder.embed_query(query)
+        # 复合查询提前拆分，主查询 + 子查询合批一次 embed（省 N-1 次往返）
+        subs = self._compound_subqueries(query)
+        embs = self.embedder.embed_queries([query, *subs])
+        q_emb, sub_embs = embs[0], embs[1:]
         rank_lists = [
             [cid for cid, _ in self.store.search_vector(q_emb, top_k)],
             [cid for cid, _ in self.store.search_fts(query, top_k)],
@@ -386,10 +397,9 @@ class Retriever:
                 weights.append(weights[0])
         # 2.9 复合意图拆分（"生成+清理"）：单次向量召回对双动作查询天然偏科，
         #     各子查询独立召回后低权重并入 RRF，两个意图都能拿到席位
-        subs = self._compound_subqueries(query)
+        min_sub = _COMPOUND_SUB_MIN_SCORE.get(self.rerank_backend, _COMPOUND_SUB_MIN_DEFAULT)
         sub_probes: list[list[int]] = []  # 每个子查询自己的前 N 名（保底探针/分数混合用）
-        for sub in subs:
-            s_emb = self.embedder.embed_query(sub)
+        for sub, s_emb in zip(subs, sub_embs):
             sub_lists = [[cid for cid, _ in self.store.search_vector(s_emb, top_k)]]
             sub_fts = (
                 self.store.search_fts_trigram(sub, top_k)
@@ -438,22 +448,29 @@ class Retriever:
             #     子查询文本打分才能反映它对那半意图的真实相关性
             if subs and sub_probes:
                 by_id = {c.get("id"): c for c in candidates}
-                for sub, probe in zip(subs, sub_probes):
-                    sub_cands = [by_id[cid] for cid in probe if cid in by_id]
-                    if not sub_cands:
-                        continue
-                    try:
-                        scored = self._rerank(sub, sub_cands, len(sub_cands))
-                    except Exception:
-                        continue
+                # 混合量纲锚定整句 top 分：Voyage 分数域窄（整句/子查询分都挤在
+                # 0.5-0.8，裸子查询分 ×0.9 永远赢不了整句分，混合失效）；
+                # 改为整句 top × 子查询相对强度，两后端语义一致：
+                # 子查询的最佳代表 ≈ 整句最佳的 _COMPOUND_SUB_RERANK_MIX 折
+                full_top = max((c.get("score", 0.0) for c in candidates), default=0.0)
+                tasks = [
+                    (sub, [by_id[cid] for cid in probe if cid in by_id])
+                    for sub, probe in zip(subs, sub_probes)
+                ]
+                tasks = [(s, c) for s, c in tasks if c]
+                scored_lists = self._rerank_many(tasks)
+                for (sub, _), scored in zip(tasks, scored_lists):
+                    sub_top = max((s["score"] for s in scored), default=0.0)
                     for s in scored:
                         c = by_id.get(s.get("id"))
                         if c is None:
                             continue
                         c["_sub_score"] = max(c.get("_sub_score", 0.0), s["score"])
-                        if s["score"] >= _COMPOUND_SUB_MIN_SCORE:
+                        if s["score"] >= min_sub and sub_top > 0:
                             c["score"] = max(
-                                c["score"], s["score"] * _COMPOUND_SUB_RERANK_MIX
+                                c["score"],
+                                full_top * (s["score"] / sub_top)
+                                * _COMPOUND_SUB_RERANK_MIX,
                             )
         # 4.5 调用链意图：结构化命中注入高分（并入统一打分管线，测试调用方会被后续降权拆开）
         intent_hits = self._caller_intent_hits(query)
@@ -548,7 +565,7 @@ class Retriever:
                     (
                         by_id[cid] for cid in probe
                         if cid in by_id and cid not in rescued_ids
-                        and by_id[cid].get("_sub_score", 0.0) >= _COMPOUND_SUB_MIN_SCORE
+                        and by_id[cid].get("_sub_score", 0.0) >= min_sub
                         and not (path_filter and not self._path_match(
                             by_id[cid].get("file_path", ""), path_filter))
                     ),
@@ -657,6 +674,22 @@ class Retriever:
                 scores[doc_id] = scores.get(doc_id, 0.0) + w / (k + rank + 1)
         return sorted(scores.items(), key=lambda x: x[1], reverse=True)
 
+    def _rerank_many(self, tasks: list[tuple[str, list[dict]]]) -> list[list[dict]]:
+        """批量 rerank（子查询分数混合用），失败项降级为空列表。
+
+        多个子查询并行发送（纯网络 IO，省一次往返延迟）；
+        评测限速场景（SCM_RERANK_MIN_INTERVAL>0）保持串行，不破坏全局请求间隔。"""
+        def one(sub: str, cands: list[dict]) -> list[dict]:
+            try:
+                return self._rerank(sub, cands, len(cands))
+            except Exception:
+                return []
+
+        if len(tasks) > 1 and _rerank_min_interval() <= 0:
+            with ThreadPoolExecutor(max_workers=len(tasks)) as ex:
+                return list(ex.map(lambda t: one(*t), tasks))
+        return [one(s, c) for s, c in tasks]
+
     def _rerank(self, query: str, candidates: list[dict], top_n: int) -> list[dict]:
         docs = [self._doc_text(c) for c in candidates]
         min_interval = _rerank_min_interval()