sirchmunk 0.0.1.post1__py3-none-any.whl → 0.0.2__py3-none-any.whl

sirchmunk/utils/tokenizer_util.py

@@ -27,9 +27,44 @@ class TokenizerUtil:
         """
         if not content.strip():
             return []
-
         return self.tokenizer.encode(content.strip())
 
+    def decode(self, token_ids: List[int]) -> str:
+        """Decode a list of token IDs back into a natural text string.
+
+        Args:
+            token_ids: List of token IDs to decode.
+
+        Returns:
+            Decoded text string.
+        """
+        if not token_ids:
+            return ""
+        return self.tokenizer.decode(token_ids, skip_special_tokens=True)
+
+    def segment(self, content: str) -> List[str]:
+        """Tokenize text into a list of token strings suitable for BM25-like algorithms indexing/retrieval.
+
+        This method returns the actual sub-word tokens as strings (e.g., ["▁Hello", "▁world"]),
+        preserving token boundaries. These tokens can be directly used as terms in BM25.
+
+        Args:
+            content: Input text string.
+
+        Returns:
+            List of token strings (not IDs), ready for BM25-style processing.
+        """
+        if not content.strip():
+            return []
+
+        token_ids = self.encode(content)
+        # Decode each token ID individually to get its string representation
+        token_strings = [
+            self.tokenizer.decode([tid], skip_special_tokens=True)
+            for tid in token_ids
+        ]
+        return token_strings
+
     def count_tokens(self, contents: Union[str, List[str]]) -> Union[int, List[int]]:
         """
         Batch count tokens for multiple texts.

sirchmunk/version.py

@@ -1 +1 @@
-__version__ = "0.0.1.post1"
+__version__ = "0.0.2"

{sirchmunk-0.0.1.post1.dist-info → sirchmunk-0.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sirchmunk
-Version: 0.0.1.post1
+Version: 0.0.2
 Summary: Sirchmunk: From raw data to self-evolving real-time intelligence.
 Author: ModelScope Team
 Author-email: contact@modelscope.cn
@@ -34,6 +34,8 @@ Requires-Dist: tqdm
 Requires-Dist: rapidfuzz
 Requires-Dist: duckdb
 Requires-Dist: kreuzberg>=4.0.0rc1
+Requires-Dist: sentence-transformers
+Requires-Dist: modelscope
 Provides-Extra: web
 Requires-Dist: fastapi>=0.100.0; extra == "web"
 Requires-Dist: uvicorn[standard]>=0.24.0; extra == "web"
@@ -103,6 +105,7 @@ Dynamic: license-file
 [![ripgrep-all](https://img.shields.io/badge/ripgrep--all-Search-E67E22?style=flat-square&logo=rust&logoColor=white)](https://github.com/phiresky/ripgrep-all)
 [![OpenAI](https://img.shields.io/badge/OpenAI-API-412991?style=flat-square&logo=openai&logoColor=white)](https://github.com/openai/openai-python)
 [![Kreuzberg](https://img.shields.io/badge/Kreuzberg-Text_Extraction-4CAF50?style=flat-square)](https://github.com/kreuzberg-dev/kreuzberg)
+[![MCP](https://img.shields.io/badge/MCP-Python_SDK-8B5CF6?style=flat-square&logo=python&logoColor=white)](https://github.com/modelcontextprotocol/python-sdk)
 
 
 [**Quick Start**](#-quick-start) · [**Key Features**](#-key-features) · [**Web UI**](#-web-ui) · [**How it Works**](#-how-it-works) · [**FAQ**](#-faq)
@@ -228,6 +231,12 @@ It serves as a unified intelligent hub for AI agents, delivering deep insights a
 
 ## 🎉 News
 
+* 🚀 **Feb 5, 2026**: Release **v0.0.2** — MCP Support, CLI Commands & Knowledge Persistence!
+  - **MCP Integration**: Full [Model Context Protocol](https://modelcontextprotocol.io) support, works seamlessly with Claude Desktop and Cursor IDE.
+  - **CLI Commands**: New `sirchmunk` CLI with `init`, `config`, `serve`, and `search` commands.
+  - **KnowledgeCluster Persistence**: DuckDB-powered storage with Parquet export for efficient knowledge management.
+  - **Knowledge Reuse**: Semantic similarity-based cluster retrieval for faster searches via embedding vectors.
+
 * 🎉🎉 Jan 22, 2026: Introducing **Sirchmunk**: Initial Release v0.0.1 Now Available!
 
 
@@ -273,9 +282,9 @@ llm = OpenAIChat(
 
 async def main():
     
-    agent_search = AgenticSearch(llm=llm)
+    searcher = AgenticSearch(llm=llm)
     
-    result: str = await agent_search.search(
+    result: str = await searcher.search(
         query="How does transformer attention work?",
         search_paths=["/path/to/documents"],
     )
@@ -286,11 +295,117 @@ asyncio.run(main())
 ```
 
 **⚠️ Notes:**
-- Upon initialization, AgenticSearch automatically checks if ripgrep-all and ripgrep are installed. If they are missing, it will attempt to install them automatically. If the automatic installation fails, please install them manually.
+- Upon initialization, `AgenticSearch` automatically checks if `ripgrep-all` and `ripgrep` are installed. If they are missing, it will attempt to install them automatically. If the automatic installation fails, please install them manually.
   - References: https://github.com/BurntSushi/ripgrep | https://github.com/phiresky/ripgrep-all
 - Replace `"your-api-key"`, `"your-base-url"`, `"your-model-name"` and `/path/to/documents` with your actual values.
 
 
+### Command Line Interface
+
+Sirchmunk provides a powerful CLI for server management and search operations.
+
+#### Installation
+
+```bash
+pip install "sirchmunk[web]"
+
+# or install via UV
+uv pip install "sirchmunk[web]"
+```
+
+
+#### Initialize
+
+```bash
+# Initialize Sirchmunk with default settings (Default work path: `~/.sirchmunk/`)
+sirchmunk init
+
+# Alternatively, initialize with custom work path
+sirchmunk init --work-path /path/to/workspace
+```
+
+#### Configure
+
+```bash
+# Show current configuration
+sirchmunk config
+
+# Regenerate configuration file if needed (Default config file: ~/.sirchmunk/.env)
+sirchmunk config --generate
+```
+
+#### Start API Server
+
+```bash
+# Start server with default settings
+sirchmunk serve
+
+# Custom host and port
+sirchmunk serve --host 0.0.0.0 --port 8000
+
+# Development mode with auto-reload
+sirchmunk serve --reload
+```
+
+#### Search
+
+```bash
+# Search in current directory
+sirchmunk search "How does authentication work?"
+
+# Search in specific paths
+sirchmunk search "find all API endpoints" ./src ./docs
+
+# Quick filename search
+sirchmunk search "config" --mode FILENAME_ONLY
+
+# Output as JSON
+sirchmunk search "database schema" --output json
+
+# Use API server (requires running server)
+sirchmunk search "query" --api --api-url http://localhost:8584
+```
+
+#### Available Commands
+
+| Command | Description |
+|---------|-------------|
+| `sirchmunk init` | Initialize working directory and configuration |
+| `sirchmunk config` | Show or generate configuration |
+| `sirchmunk serve` | Start the API server |
+| `sirchmunk search` | Perform search queries |
+| `sirchmunk version` | Show version information |
+
+---
+
+## 🔌 MCP Server
+
+Sirchmunk provides a [Model Context Protocol (MCP)](https://modelcontextprotocol.io) server that exposes its intelligent search capabilities as MCP tools. This enables seamless integration with AI assistants like **Claude Desktop** and **Cursor IDE**.
+
+### Quick Start
+
+```bash
+# Install MCP package
+pip install sirchmunk-mcp
+
+# Initialize and configure
+sirchmunk-mcp init
+sirchmunk-mcp config --generate
+
+# Edit ~/.sirchmunk/.mcp_env with your LLM API key
+
+# Test with MCP Inspector
+npx @modelcontextprotocol/inspector sirchmunk-mcp serve
+```
+
+### Features
+
+- **Multi-Mode Search**: DEEP mode for comprehensive analysis, FILENAME_ONLY for fast file discovery
+- **Knowledge Cluster Management**: Automatic extraction, storage, and reuse of knowledge
+- **Standard MCP Protocol**: Works with stdio and Streamable HTTP transports
+
+📖 **For detailed documentation, see [Sirchmunk MCP README](src/sirchmunk_mcp/README.md)**.
+
 ---
 
 ## 🖥️ Web UI
@@ -364,10 +479,10 @@ python scripts/stop_web.py
 
 ### Data Storage
 
-All persistent data is stored in the configured `WORK_PATH` (default: `~/.sirchmunk/`):
+All persistent data is stored in the configured `SIRCHMUNK_WORK_PATH` (default: `~/.sirchmunk/`):
 
 ```
-{WORK_PATH}/
+{SIRCHMUNK_WORK_PATH}/
   ├── .cache/
     ├── history/              # Chat session history (DuckDB)
     │   └── chat_history.db
@@ -410,7 +525,7 @@ Any OpenAI-compatible API endpoint, including (but not limited too):
 Simply specify the path in your search query:
 
 ```python
-result = await search.search(
+result = await searcher.search(
     query="Your question",
     search_paths=["/path/to/folder", "/path/to/file.pdf"]
 )
@@ -425,7 +540,7 @@ No pre-processing or indexing required!
 
 Knowledge clusters are persisted in Parquet format at:
 ```
-{WORK_PATH}/.cache/knowledge/knowledge_clusters.parquet
+{SIRCHMUNK_WORK_PATH}/.cache/knowledge/knowledge_clusters.parquet
 ```
 
 You can query them using DuckDB or the `KnowledgeManager` API.
@@ -437,7 +552,7 @@ You can query them using DuckDB or the `KnowledgeManager` API.
 
 1. **Web Dashboard**: Visit the Monitor page for real-time statistics
 2. **API**: `GET /api/v1/monitor/llm` returns usage metrics
-3. **Code**: Access `search.llm_usages` after search completion
+3. **Code**: Access `searcher.llm_usages` after search completion
 
 </details>
 

sirchmunk-0.0.2.dist-info/RECORD

@@ -0,0 +1,69 @@
+sirchmunk/__init__.py,sha256=5sdppELUcjnTofJtwZ2ACuUscmLYIPhCwj-G-MMSl-M,184
+sirchmunk/base.py,sha256=qVQ63QfEWhEvOJl3OxQvC2rOUNTZCD5weXRn-1vvEkU,439
+sirchmunk/search.py,sha256=rOfcdrXZbm5gcZWKRHSJWmK_kZDgP0pKS0uQsGIlTXQ,34270
+sirchmunk/version.py,sha256=QvlVh4JTl3JL7jQAja76yKtT-IvF4631ASjWY1wS6AQ,22
+sirchmunk/api/__init__.py,sha256=310L84MdAIw4THnzf5YsLiUhW_oaxgJHHcZZeMso3jY,61
+sirchmunk/api/chat.py,sha256=5Y3ujcgK3GPkPDrCnS1uduAQYr4dpzYmHjztCJQWITA,41103
+sirchmunk/api/history.py,sha256=iKRzMtvy-fRVSfqd5mkisHPTttNb2XOVh1P1Zv3Ohgk,9135
+sirchmunk/api/knowledge.py,sha256=fpXX16O2XD-JIrywBUH5JOt8arVBz8X9JoNhxSBfofo,12948
+sirchmunk/api/main.py,sha256=FVSbIa_5EKTRldDRwzAvDoezr6VyvktJTiQhv5CKq2k,3253
+sirchmunk/api/monitor.py,sha256=SIG_kYxERpVbLHBzOYAuPH9fbVAy8gGq_mNuVO14v1M,5655
+sirchmunk/api/run_server.py,sha256=FvYwlvrQ0MUhk4Yn7lbvoXWnJ6d7k6D0T9hUV6EDEmE,1632
+sirchmunk/api/search.py,sha256=-eqmewxy4541apNoB-7SU_gXtrPz-7MLKBa_OkbJwCA,6776
+sirchmunk/api/settings.py,sha256=w7mpyynBhtLmjkvofGVsu7IqO5YrjE0M08yntK3kPwk,10544
+sirchmunk/api/tools.py,sha256=VMYPkxm4ofiSqHrZISrpEZ0QQ83Ef6MUN4MRKUo1kiI,11227
+sirchmunk/api/components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sirchmunk/api/components/history_storage.py,sha256=5ug-CitkIounlLct5n6iK8va3hFM1t4FRGgiGM5SB60,13969
+sirchmunk/api/components/monitor_tracker.py,sha256=oBLq3My3Fq0WTpbIvcRW0MJR6Nf_eiEy0tIWb4AzeOM,18424
+sirchmunk/api/components/settings_storage.py,sha256=kPG87q5iZjUZrcNPeZFEhb-6Fy3oY7_4AfVBESdn1uA,11415
+sirchmunk/cli/__init__.py,sha256=ivKDHPZFyKEQ5GDhtK6ZjuQWduUgziF5sUtNFwXFW10,253
+sirchmunk/cli/cli.py,sha256=QsF9vIgzySQrX0bdbjJqA97RMMvem2RLXSvFij6jbPU,23463
+sirchmunk/insight/__init__.py,sha256=7sQeT0fSg5-b9jrwthA9_fSCR1Q5qSvm33L9l7kHjLY,144
+sirchmunk/insight/text_insights.py,sha256=BwFqmDNG8FmOz4Lv3qO1Mz2xRv_LVg4b3wj6YUhVLVE,9075
+sirchmunk/learnings/__init__.py,sha256=310L84MdAIw4THnzf5YsLiUhW_oaxgJHHcZZeMso3jY,61
+sirchmunk/learnings/evidence_processor.py,sha256=QDg-qReSte8R8I2BrRRC-d-Glyfjcnqazbe57-PSDHU,17922
+sirchmunk/learnings/knowledge_base.py,sha256=K-Rx2jnZH5dGzQk6lrwoSFaaVZ4QJJQtnTFO3rkyr-I,8496
+sirchmunk/llm/__init__.py,sha256=4ynF6R63afMWW1d9T21C8JqfVc9xJTiOFXZNzDwPLww,98
+sirchmunk/llm/openai_chat.py,sha256=ET7HqEoFTbbvhUTlAUZNOoJwxF9hA73qeH3xFTsNK-w,8138
+sirchmunk/llm/prompts.py,sha256=uCNH4mbwtAL7QUJuGEzNGI7rXKPXW5Sbs0g1ZRMAKqI,9274
+sirchmunk/retrieve/__init__.py,sha256=310L84MdAIw4THnzf5YsLiUhW_oaxgJHHcZZeMso3jY,61
+sirchmunk/retrieve/base.py,sha256=VDpCwdwhjVYuj0mbe78qg_FhbcQkasbCS0cd66hQ4hk,618
+sirchmunk/retrieve/text_retriever.py,sha256=hqvdCYFKwLJlhnGXabxwEDWPhBK8sQ9fxb3oIhEGE_s,46980
+sirchmunk/scan/__init__.py,sha256=310L84MdAIw4THnzf5YsLiUhW_oaxgJHHcZZeMso3jY,61
+sirchmunk/scan/base.py,sha256=3Jqvn0W9KJ00u2oy8U-qD16KksV7taAx4s9qhCpRD-c,496
+sirchmunk/scan/file_scanner.py,sha256=7w7wTvYc2SZpfIo02HvWxHEnJGmw_rifWPTXMMpYn6s,14698
+sirchmunk/scan/web_scanner.py,sha256=YQRBQ5JnATwNdek2o1_Y8GB0eG0IWQvx4YhqW4Zc5Tw,531
+sirchmunk/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sirchmunk/schema/__init__.py,sha256=kvzKU8Rt1t91Rm5tuy6KqgZXj8lfYQ2q1XvLYGNLUh4,103
+sirchmunk/schema/cognition.py,sha256=NDgCOjMpqyUBxd_ymeMQrjSr3Ri-nJeMUNRuWfLUhBI,4031
+sirchmunk/schema/context.py,sha256=wwsSBE5EJXmrvh-RCluf3nJm7hcxkWaiNWfJ10coGY4,628
+sirchmunk/schema/knowledge.py,sha256=4OiLvNK_wKT96FxeUsOxZG4ZmJ5RCyCtlyehbIBTpLg,14824
+sirchmunk/schema/metadata.py,sha256=VdUD6GPCB_jtyBkowkGEpCPMYL1c-sFVSUloKPSZRwE,22810
+sirchmunk/schema/request.py,sha256=EPum-IzmN15EgFmwN12oL-CNoI-KvTcHN4nLuO-c09c,7702
+sirchmunk/schema/response.py,sha256=6xc5tvAnqL_zpUhQtchVGwbrt1btzy4QArCeS81DrIU,488
+sirchmunk/schema/snapshot.py,sha256=zZSKDRN8jMtpOIH3TL0FCrAn7AlUc_5zO8qFX2f2u_s,12341
+sirchmunk/storage/__init__.py,sha256=maUD6CAUydaWYHewhGLLaaKAH1ap-qzVTjVbb_XeEmg,231
+sirchmunk/storage/duckdb.py,sha256=Jw_EK9i589YyKXYhi-yjAmX7zIM7txDsoxy0Gx-XY8o,23092
+sirchmunk/storage/knowledge_storage.py,sha256=I7KdBbul9TTAWsClYJMyXNK_4b2o76PzGoycOYwHGTs,36009
+sirchmunk/utils/__init__.py,sha256=33bVrhpUfPXpM85r_SEB07QJnGDjD-4BE5p9vpF-fXw,265
+sirchmunk/utils/constants.py,sha256=Ki7XtUD152q6jZDK5rUqZJ8H_AMeFGO40Wvc2UBKqV0,703
+sirchmunk/utils/deps.py,sha256=QTL0k7CN1t0_r-CrZ4TM__pdK8U2X1d0OKvJUtQe_xA,563
+sirchmunk/utils/embedding_util.py,sha256=5xEH773XklRqMu5kNBDKmIMqtFjU-H_c8DtjYiPxvhk,6603
+sirchmunk/utils/file_utils.py,sha256=9OtYNffXbo1Pz6XuJsOECQS_mYRBg1NpGUsglNgfWnU,2257
+sirchmunk/utils/install_rga.py,sha256=i7sWYi6u2A32dc0mq5LB_OtqcMWqKezSz6WILWaK1Oc,5215
+sirchmunk/utils/log_utils.py,sha256=HujosgEXV9fpSVd6JjRh7KEQuskhrcpC4Kit6aIpnW0,14195
+sirchmunk/utils/tokenizer_util.py,sha256=BC-U3ahUPID0ctuRGw9dEc_IfNe_vAKZuiwQ_GSib6g,2848
+sirchmunk/utils/utils.py,sha256=qnwZ8R9xLqsMooW0IdcWoPKto7q4AQ49-SOR33rCy1g,3394
+sirchmunk-0.0.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sirchmunk_mcp/__init__.py,sha256=EFeOjlxKZJjPhAyb6ISdSUT-7arY3EZw0zZyvYPVx24,612
+sirchmunk_mcp/cli.py,sha256=uYcahqzYMPcJWcrVOzuvkQeoUWKeGhkIv2R5nHW7S3M,15341
+sirchmunk_mcp/config.py,sha256=7nbN9G0ECCqnINWEuqh6Hddxs_o_JznpdwOyGgFf5LE,8956
+sirchmunk_mcp/server.py,sha256=XH3_EohE-91zJtiAeLdwH69oISn7Hizytipwbu-gJmc,11515
+sirchmunk_mcp/service.py,sha256=d5G0IJHlBeaTkxo_MmFIF1h9GZu62oOTjvWX7Pd29XI,12184
+sirchmunk_mcp/setup.py,sha256=ELDZu3FIhiAd8u-GgIxsvfmK-58cYdC1NLEJBAZnhsI,370
+sirchmunk_mcp/tools.py,sha256=8L95ajxB51eu4GWyXGpq2sorE30HwZ2KLA7tyqLii-Q,12739
+sirchmunk-0.0.2.dist-info/METADATA,sha256=nWhcRloB7ESH8zsTw3JqrGk3TtczzvvKZlhOTmIZT7o,21813
+sirchmunk-0.0.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+sirchmunk-0.0.2.dist-info/entry_points.txt,sha256=lpnP-Ll2CUY0P1wYm6kutcBMrxwG67astmgY-vVhF14,56
+sirchmunk-0.0.2.dist-info/top_level.txt,sha256=TBWdnogNJmu7ZizAztBsxTVOqkpoAc-5o-aKCU8KpBc,24
+sirchmunk-0.0.2.dist-info/RECORD,,

{sirchmunk-0.0.1.post1.dist-info → sirchmunk-0.0.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.1)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

sirchmunk-0.0.2.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+sirchmunk
+sirchmunk_mcp

sirchmunk_mcp/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""
+Sirchmunk MCP Server
+
+A Model Context Protocol (MCP) server that exposes Sirchmunk's intelligent
+code and document search capabilities as MCP tools.
+
+Uses FastMCP for simplified MCP server implementation.
+"""
+
+__version__ = "0.1.0"
+__author__ = "ModelScope Contributors"
+
+from .server import create_server, run_stdio_server, run_http_server
+from .service import SirchmunkService
+from .config import Config
+
+__all__ = [
+    "create_server",
+    "run_stdio_server",
+    "run_http_server",
+    "SirchmunkService",
+    "Config",
+    "__version__",
+]