jarvis-ai-assistant 0.3.20__py3-none-any.whl → 0.3.21__py3-none-any.whl

jarvis/jarvis_rag/retriever.py

@@ -1,6 +1,8 @@
 import os
 import pickle
-from typing import Any, Dict, List, cast
+import json
+import hashlib
+from typing import Any, Dict, List, Optional, cast
 
 import chromadb
 from langchain.docstore.document import Document
@@ -48,6 +50,10 @@ class ChromaRetriever:
         # BM25索引设置
         self.bm25_index_path = os.path.join(self.db_path, f"{collection_name}_bm25.pkl")
         self._load_or_initialize_bm25()
+        # 清单文件用于检测源文件的变更/删除
+        self.manifest_path = os.path.join(
+            self.db_path, f"{collection_name}_manifest.json"
+        )
 
     def _load_or_initialize_bm25(self):
         """从磁盘加载BM25索引或初始化一个新索引。"""
@@ -59,7 +65,9 @@ class ChromaRetriever:
                 self.bm25_index = BM25Okapi(self.bm25_corpus)
             PrettyOutput.print("BM25 索引加载成功。", OutputType.SUCCESS)
         else:
-            PrettyOutput.print("未找到 BM25 索引，将初始化一个新的。", OutputType.WARNING)
+            PrettyOutput.print(
+                "未找到 BM25 索引，将初始化一个新的。", OutputType.WARNING
+            )
             self.bm25_corpus = []
             self.bm25_index = None
 
@@ -71,6 +79,138 @@ class ChromaRetriever:
                 pickle.dump({"corpus": self.bm25_corpus, "index": self.bm25_index}, f)
             PrettyOutput.print("BM25 索引保存成功。", OutputType.SUCCESS)
 
+    def _load_manifest(self) -> Dict[str, Dict[str, Any]]:
+        """加载已索引文件清单，用于变更检测。"""
+        if os.path.exists(self.manifest_path):
+            try:
+                with open(self.manifest_path, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+                if isinstance(data, dict):
+                    return data  # type: ignore[return-value]
+            except Exception:
+                pass
+        return {}
+
+    def _save_manifest(self, manifest: Dict[str, Dict[str, Any]]) -> None:
+        """保存已索引文件清单。"""
+        try:
+            with open(self.manifest_path, "w", encoding="utf-8") as f:
+                json.dump(manifest, f, ensure_ascii=False, indent=2)
+        except Exception as e:
+            PrettyOutput.print(f"保存索引清单失败: {e}", OutputType.WARNING)
+
+    def _compute_md5(
+        self, file_path: str, chunk_size: int = 1024 * 1024
+    ) -> Optional[str]:
+        """流式计算文件的MD5，避免占用过多内存。失败时返回None。"""
+        try:
+            md5 = hashlib.md5()
+            with open(file_path, "rb") as f:
+                while True:
+                    data = f.read(chunk_size)
+                    if not data:
+                        break
+                    md5.update(data)
+            return md5.hexdigest()
+        except Exception:
+            return None
+
+    def _update_manifest_with_sources(self, sources: List[str]) -> None:
+        """根据本次新增文档的来源，更新索引清单（记录mtime与size）。"""
+        manifest = self._load_manifest()
+        updated = 0
+        for src in set(sources):
+            try:
+                if isinstance(src, str) and os.path.exists(src):
+                    st = os.stat(src)
+                    entry: Dict[str, Any] = {
+                        "mtime": float(st.st_mtime),
+                        "size": int(st.st_size),
+                    }
+                    md5sum = self._compute_md5(src)
+                    if md5sum:
+                        entry["md5"] = md5sum
+                    manifest[src] = entry  # type: ignore[dict-item]
+                    updated += 1
+            except Exception:
+                continue
+        if updated > 0:
+            self._save_manifest(manifest)
+            PrettyOutput.print(
+                f"已更新索引清单，记录 {updated} 个源文件状态。", OutputType.INFO
+            )
+
+    def _detect_changed_or_deleted(self) -> Dict[str, List[str]]:
+        """检测已记录的源文件是否发生变化或被删除。"""
+        manifest = self._load_manifest()
+        changed: List[str] = []
+        deleted: List[str] = []
+        for src, info in manifest.items():
+            try:
+                if not os.path.exists(src):
+                    deleted.append(src)
+                    continue
+                st = os.stat(src)
+                size_changed = int(info.get("size", -1)) != int(st.st_size)
+                if size_changed:
+                    changed.append(src)
+                    continue
+                md5_old = info.get("md5")
+                if md5_old:
+                    # 仅在mtime变化时计算md5以降低开销
+                    mtime_changed = (
+                        abs(float(info.get("mtime", 0.0)) - float(st.st_mtime)) >= 1e-6
+                    )
+                    if mtime_changed:
+                        md5_new = self._compute_md5(src)
+                        if not md5_new or md5_new != md5_old:
+                            changed.append(src)
+                else:
+                    # 没有记录md5，回退使用mtime判断
+                    mtime_changed = (
+                        abs(float(info.get("mtime", 0.0)) - float(st.st_mtime)) >= 1e-6
+                    )
+                    if mtime_changed:
+                        changed.append(src)
+            except Exception:
+                # 无法读取文件状态，视为发生变化
+                changed.append(src)
+        return {"changed": changed, "deleted": deleted}
+
+    def _warn_if_sources_changed(self) -> None:
+        """如发现已索引文件变化或删除，给出提醒。"""
+        result = self._detect_changed_or_deleted()
+        changed = result["changed"]
+        deleted = result["deleted"]
+        if not changed and not deleted:
+            return
+        if changed:
+            PrettyOutput.print(
+                f"检测到 {len(changed)} 个已索引文件发生变化，建议重新索引以保证检索准确性。",
+                OutputType.WARNING,
+            )
+            for p in changed[:5]:
+                PrettyOutput.print(f"  变更: {p}", OutputType.WARNING)
+            if len(changed) > 5:
+                PrettyOutput.print(
+                    f"  ... 以及另外 {len(changed) - 5} 个文件", OutputType.WARNING
+                )
+        if deleted:
+            PrettyOutput.print(
+                f"检测到 {len(deleted)} 个已索引文件已被删除，建议清理并重新索引。",
+                OutputType.WARNING,
+            )
+            for p in deleted[:5]:
+                PrettyOutput.print(f"  删除: {p}", OutputType.WARNING)
+            if len(deleted) > 5:
+                PrettyOutput.print(
+                    f"  ... 以及另外 {len(deleted) - 5} 个文件", OutputType.WARNING
+                )
+        PrettyOutput.print(
+            "提示：请使用 'jarvis-rag add <路径>' 重新索引相关文件，以更新向量库与BM25索引。",
+            OutputType.INFO,
+        )
+
     def add_documents(
         self, documents: List[Document], chunk_size=1000, chunk_overlap=100
     ):
@@ -114,6 +254,13 @@ class ChromaRetriever:
         self.bm25_corpus.extend(tokenized_chunks)
         self.bm25_index = BM25Okapi(self.bm25_corpus)
         self._save_bm25_index()
+        # 更新索引清单（用于检测源文件变更/删除）
+        source_list = [
+            md.get("source")
+            for md in metadatas
+            if md and isinstance(md.get("source"), str)
+        ]
+        self._update_manifest_with_sources(cast(List[str], source_list))
 
     def retrieve(
         self, query: str, n_results: int = 5, use_bm25: bool = True
@@ -122,6 +269,8 @@ class ChromaRetriever:
         使用向量搜索和BM25执行混合检索，然后使用倒数排序融合（RRF）
         对结果进行融合。
         """
+        # 在检索前检查源文件变更/删除并提醒
+        self._warn_if_sources_changed()
         # 1. 向量搜索 (ChromaDB)
         query_embedding = self.embedding_manager.embed_query(query)
         vector_results = self.collection.query(

jarvis/jarvis_smart_shell/main.py

@@ -125,7 +125,9 @@ def install_jss_completion(
 ) -> None:
     """为指定的shell安装'命令未找到'处理器，实现自然语言命令建议"""
     if shell not in ("fish", "bash", "zsh"):
-        PrettyOutput.print(f"错误: 不支持的shell类型: {shell}, 仅支持fish, bash, zsh", OutputType.ERROR)
+        PrettyOutput.print(
+            f"错误: 不支持的shell类型: {shell}, 仅支持fish, bash, zsh", OutputType.ERROR
+        )
         raise typer.Exit(code=1)
 
     if shell == "fish":
@@ -142,7 +144,10 @@ def install_jss_completion(
             content = f.read()
 
         if start_marker in content:
-            PrettyOutput.print("JSS fish completion 已安装，请执行: source ~/.config/fish/config.fish", OutputType.SUCCESS)
+            PrettyOutput.print(
+                "JSS fish completion 已安装，请执行: source ~/.config/fish/config.fish",
+                OutputType.SUCCESS,
+            )
             return
 
         with open(config_file, "a") as f:
@@ -162,7 +167,10 @@ end
 {end_marker}
 """
             )
-        PrettyOutput.print("JSS fish completion 已安装，请执行: source ~/.config/fish/config.fish", OutputType.SUCCESS)
+        PrettyOutput.print(
+            "JSS fish completion 已安装，请执行: source ~/.config/fish/config.fish",
+            OutputType.SUCCESS,
+        )
     elif shell == "bash":
         config_file = _get_bash_config_file()
         start_marker, end_marker = _get_bash_markers()
@@ -177,7 +185,10 @@ end
             content = f.read()
 
         if start_marker in content:
-            PrettyOutput.print("JSS bash completion 已安装，请执行: source ~/.bashrc", OutputType.SUCCESS)
+            PrettyOutput.print(
+                "JSS bash completion 已安装，请执行: source ~/.bashrc",
+                OutputType.SUCCESS,
+            )
             return
         else:
             with open(config_file, "a") as f:
@@ -220,7 +231,10 @@ command_not_found_handle() {{
 {end_marker}
 """
                 )
-            PrettyOutput.print("JSS bash completion 已安装，请执行: source ~/.bashrc", OutputType.SUCCESS)
+            PrettyOutput.print(
+                "JSS bash completion 已安装，请执行: source ~/.bashrc",
+                OutputType.SUCCESS,
+            )
     elif shell == "zsh":
         config_file = _get_zsh_config_file()
         start_marker, end_marker = _get_zsh_markers()
@@ -235,7 +249,9 @@ command_not_found_handle() {{
             content = f.read()
 
         if start_marker in content:
-            PrettyOutput.print("JSS zsh completion 已安装，请执行: source ~/.zshrc", OutputType.SUCCESS)
+            PrettyOutput.print(
+                "JSS zsh completion 已安装，请执行: source ~/.zshrc", OutputType.SUCCESS
+            )
             return
 
         with open(config_file, "a") as f:
@@ -282,7 +298,9 @@ command_not_found_handler() {{
 {end_marker}
 """
             )
-        PrettyOutput.print("JSS zsh completion 已安装，请执行: source ~/.zshrc", OutputType.SUCCESS)
+        PrettyOutput.print(
+            "JSS zsh completion 已安装，请执行: source ~/.zshrc", OutputType.SUCCESS
+        )
         return
 
         with open(config_file, "a") as f:
@@ -325,7 +343,9 @@ command_not_found_handle() {{
 {end_marker}
 """
             )
-        PrettyOutput.print("JSS bash completion 已安装，请执行: source ~/.bashrc", OutputType.SUCCESS)
+        PrettyOutput.print(
+            "JSS bash completion 已安装，请执行: source ~/.bashrc", OutputType.SUCCESS
+        )
 
 
 @app.command("uninstall")
@@ -334,7 +354,9 @@ def uninstall_jss_completion(
 ) -> None:
     """卸载JSS shell'命令未找到'处理器"""
     if shell not in ("fish", "bash", "zsh"):
-        PrettyOutput.print(f"错误: 不支持的shell类型: {shell}, 仅支持fish, bash, zsh", OutputType.ERROR)
+        PrettyOutput.print(
+            f"错误: 不支持的shell类型: {shell}, 仅支持fish, bash, zsh", OutputType.ERROR
+        )
         raise typer.Exit(code=1)
 
     if shell == "fish":
@@ -342,14 +364,18 @@ def uninstall_jss_completion(
         start_marker, end_marker = _get_markers()
 
         if not os.path.exists(config_file):
-            PrettyOutput.print("未找到 JSS fish completion 配置，无需卸载", OutputType.INFO)
+            PrettyOutput.print(
+                "未找到 JSS fish completion 配置，无需卸载", OutputType.INFO
+            )
             return
 
         with open(config_file, "r") as f:
             content = f.read()
 
         if start_marker not in content:
-            PrettyOutput.print("未找到 JSS fish completion 配置，无需卸载", OutputType.INFO)
+            PrettyOutput.print(
+                "未找到 JSS fish completion 配置，无需卸载", OutputType.INFO
+            )
             return
 
         new_content = content.split(start_marker)[0] + content.split(end_marker)[-1]
@@ -357,20 +383,27 @@ def uninstall_jss_completion(
         with open(config_file, "w") as f:
             f.write(new_content)
 
-        PrettyOutput.print("JSS fish completion 已卸载，请执行: source ~/.config/fish/config.fish", OutputType.SUCCESS)
+        PrettyOutput.print(
+            "JSS fish completion 已卸载，请执行: source ~/.config/fish/config.fish",
+            OutputType.SUCCESS,
+        )
     elif shell == "bash":
         config_file = _get_bash_config_file()
         start_marker, end_marker = _get_bash_markers()
 
         if not os.path.exists(config_file):
-            PrettyOutput.print("未找到 JSS bash completion 配置，无需卸载", OutputType.INFO)
+            PrettyOutput.print(
+                "未找到 JSS bash completion 配置，无需卸载", OutputType.INFO
+            )
             return
 
         with open(config_file, "r") as f:
             content = f.read()
 
         if start_marker not in content:
-            PrettyOutput.print("未找到 JSS bash completion 配置，无需卸载", OutputType.INFO)
+            PrettyOutput.print(
+                "未找到 JSS bash completion 配置，无需卸载", OutputType.INFO
+            )
             return
 
         new_content = content.split(start_marker)[0] + content.split(end_marker)[-1]
@@ -378,20 +411,26 @@ def uninstall_jss_completion(
         with open(config_file, "w") as f:
             f.write(new_content)
 
-        PrettyOutput.print("JSS bash completion 已卸载，请执行: source ~/.bashrc", OutputType.SUCCESS)
+        PrettyOutput.print(
+            "JSS bash completion 已卸载，请执行: source ~/.bashrc", OutputType.SUCCESS
+        )
     elif shell == "zsh":
         config_file = _get_zsh_config_file()
         start_marker, end_marker = _get_zsh_markers()
 
         if not os.path.exists(config_file):
-            PrettyOutput.print("未找到 JSS zsh completion 配置，无需卸载", OutputType.INFO)
+            PrettyOutput.print(
+                "未找到 JSS zsh completion 配置，无需卸载", OutputType.INFO
+            )
             return
 
         with open(config_file, "r") as f:
             content = f.read()
 
         if start_marker not in content:
-            PrettyOutput.print("未找到 JSS zsh completion 配置，无需卸载", OutputType.INFO)
+            PrettyOutput.print(
+                "未找到 JSS zsh completion 配置，无需卸载", OutputType.INFO
+            )
             return
 
         new_content = content.split(start_marker)[0] + content.split(end_marker)[-1]
@@ -399,7 +438,9 @@ def uninstall_jss_completion(
         with open(config_file, "w") as f:
             f.write(new_content)
 
-        PrettyOutput.print("JSS zsh completion 已卸载，请执行: source ~/.zshrc", OutputType.SUCCESS)
+        PrettyOutput.print(
+            "JSS zsh completion 已卸载，请执行: source ~/.zshrc", OutputType.SUCCESS
+        )
 
 
 def process_request(request: str) -> Optional[str]:

jarvis/jarvis_stats/cli.py

@@ -208,7 +208,7 @@ def list():
             # 获取数据点数和标签
             records = stats._get_storage().get_metrics(metric, start_time, end_time)
             count = len(records)
-            
+
             # 收集所有唯一的标签
             all_tags = {}
             for record in records:
@@ -217,7 +217,7 @@ def list():
                     if k not in all_tags:
                         all_tags[k] = set()
                     all_tags[k].add(v)
-            
+
             # 格式化标签显示
             tag_str = ""
             if all_tags:
@@ -292,7 +292,9 @@ def export(
 
     if output == "json":
         # JSON格式输出
-        PrettyOutput.print(json.dumps(data, indent=2, ensure_ascii=False), OutputType.CODE, lang="json")
+        PrettyOutput.print(
+            json.dumps(data, indent=2, ensure_ascii=False), OutputType.CODE, lang="json"
+        )
     else:
         # CSV格式输出
         records = data.get("records", [])
@@ -316,30 +318,30 @@ def remove(
         # 显示指标信息供用户确认
         stats = StatsManager(_get_stats_dir())
         metrics = stats.list_metrics()
-        
+
         if metric not in metrics:
             rprint(f"[red]错误：指标 '{metric}' 不存在[/red]")
             return
-            
+
         # 获取指标的基本信息
         info = stats._get_storage().get_metric_info(metric)
         if info:
             unit = info.get("unit", "-")
             last_updated = info.get("last_updated", "-")
-            
+
             rprint(f"\n[yellow]准备删除指标:[/yellow]")
             rprint(f"  名称: {metric}")
             rprint(f"  单位: {unit}")
             rprint(f"  最后更新: {last_updated}")
-            
+
         confirm = typer.confirm(f"\n确定要删除指标 '{metric}' 及其所有数据吗？")
         if not confirm:
             rprint("[yellow]已取消操作[/yellow]")
             return
-    
+
     stats = StatsManager(_get_stats_dir())
     success = stats.remove_metric(metric)
-    
+
     if success:
         rprint(f"[green]✓[/green] 已成功删除指标: {metric}")
     else:

jarvis/jarvis_stats/stats.py

@@ -116,7 +116,9 @@ class StatsManager:
         info = storage.get_metric_info(metric_name)
         if not info or not info.get("group"):
             try:
-                grp = storage.resolve_metric_group(metric_name)  # 触发一次分组解析与回填
+                grp = storage.resolve_metric_group(
+                    metric_name
+                )  # 触发一次分组解析与回填
                 if grp:
                     info = storage.get_metric_info(metric_name)
             except Exception:
@@ -313,10 +315,10 @@ class StatsManager:
     def remove_metric(metric_name: str) -> bool:
         """
         删除指定的指标及其所有数据
-        
+
         Args:
             metric_name: 要删除的指标名称
-            
+
         Returns:
             True 如果成功删除，False 如果指标不存在
         """
@@ -359,15 +361,15 @@ class StatsManager:
         for metric in metrics:
             # 获取该指标的记录
             records = storage.get_metrics(metric, start_time, end_time, tags)
-            
+
             # 如果指定了标签过滤，但没有匹配的记录，跳过该指标
             if tags and len(records) == 0:
                 continue
-            
+
             info = storage.get_metric_info(metric)
             unit = "-"
             last_updated = "-"
-            
+
             if info:
                 unit = info.get("unit", "-")
                 last_updated = info.get("last_updated", "-")
@@ -440,7 +442,9 @@ class StatsManager:
         )
 
         if not aggregated:
-            PrettyOutput.print(f"没有找到指标 '{metric_name}' 的数据", OutputType.WARNING)
+            PrettyOutput.print(
+                f"没有找到指标 '{metric_name}' 的数据", OutputType.WARNING
+            )
             return
 
         # 获取指标信息
@@ -550,7 +554,9 @@ class StatsManager:
         )
 
         if not aggregated:
-            PrettyOutput.print(f"没有找到指标 '{metric_name}' 的数据", OutputType.WARNING)
+            PrettyOutput.print(
+                f"没有找到指标 '{metric_name}' 的数据", OutputType.WARNING
+            )
             return
 
         # 获取指标信息

jarvis/jarvis_stats/storage.py

@@ -347,15 +347,24 @@ class StatsStorage:
                 info = meta["metrics"].get(metric_name)
                 now_iso = datetime.now().isoformat()
                 if info is None:
-                    info = {"unit": None, "created_at": now_iso, "last_updated": now_iso}
+                    info = {
+                        "unit": None,
+                        "created_at": now_iso,
+                        "last_updated": now_iso,
+                    }
                     meta["metrics"][metric_name] = info
                 if not info.get("group"):
                     inferred_group = None
                     if group_counts:
-                        inferred_group = max(group_counts.items(), key=lambda kv: kv[1])[0]
+                        inferred_group = max(
+                            group_counts.items(), key=lambda kv: kv[1]
+                        )[0]
                     # 名称启发式作为补充
                     if not inferred_group:
-                        if metric_name.startswith("code_lines_") or "commit" in metric_name:
+                        if (
+                            metric_name.startswith("code_lines_")
+                            or "commit" in metric_name
+                        ):
                             inferred_group = "code_agent"
                     if inferred_group:
                         info["group"] = inferred_group
@@ -381,7 +390,9 @@ class StatsStorage:
         try:
             # 优先从元数据读取
             meta = self._load_json(self.meta_file)
-            metrics_meta = meta.get("metrics", {}) if isinstance(meta.get("metrics"), dict) else {}
+            metrics_meta = (
+                meta.get("metrics", {}) if isinstance(meta.get("metrics"), dict) else {}
+            )
             info = metrics_meta.get(metric_name)
             if info and isinstance(info, dict):
                 grp = info.get("group")
@@ -420,7 +431,11 @@ class StatsStorage:
                     metrics_meta = meta["metrics"]
                 if info is None:
                     now_iso = datetime.now().isoformat()
-                    info = {"unit": None, "created_at": now_iso, "last_updated": now_iso}
+                    info = {
+                        "unit": None,
+                        "created_at": now_iso,
+                        "last_updated": now_iso,
+                    }
                     metrics_meta[metric_name] = info
                 info["group"] = inferred_group
                 self._save_json(self.meta_file, meta)
@@ -461,7 +476,9 @@ class StatsStorage:
             pass
 
         # 合并三个来源的指标并返回排序后的列表
-        all_metrics = metrics_from_meta.union(metrics_from_data).union(metrics_from_totals)
+        all_metrics = metrics_from_meta.union(metrics_from_data).union(
+            metrics_from_totals
+        )
         return sorted(list(all_metrics))
 
     def aggregate_metrics(