sirchmunk 0.0.0__py3-none-any.whl → 0.0.1.post1__py3-none-any.whl

sirchmunk/learnings/knowledge_base.py

@@ -0,0 +1,232 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import json
+import hashlib
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Union
+
+
+from sirchmunk.learnings.evidence_processor import (
+    MonteCarloEvidenceSampling,
+    RoiResult,
+)
+from sirchmunk.llm.openai_chat import OpenAIChat
+from sirchmunk.llm.prompts import EVIDENCE_SUMMARY
+from sirchmunk.schema.knowledge import (
+    AbstractionLevel,
+    EvidenceUnit,
+    KnowledgeCluster,
+    Lifecycle,
+)
+from sirchmunk.schema.metadata import FileInfo
+from sirchmunk.schema.request import Request
+from sirchmunk.utils.constants import DEFAULT_WORK_PATH
+from sirchmunk.utils.file_utils import StorageStructure, fast_extract
+from sirchmunk.utils import create_logger, LogCallback
+from sirchmunk.utils.utils import extract_fields
+
+class KnowledgeBase:
+    """
+    A knowledge base that manages knowledge clusters built from retrieved information and metadata dynamically.
+    """
+
+    def __init__(
+        self,
+        llm: OpenAIChat,
+        metadata_map: Dict[str, Any] = None,
+        work_path: Union[str, Path] = None,
+        log_callback: LogCallback = None,
+    ):
+        """
+        Initialize the KnowledgeBase with an LLM and metadata mapping.
+
+        Args:
+            llm (OpenAIChat): An instance of the OpenAIChat LLM for processing text.
+            metadata_map (Dict[str, Any]): A mapping of all metadata information.
+                k: metadata cache key, refers to `FileInfo.cache_key`
+                v: metadata path or content
+            work_path: Working directory path
+            log_callback: Optional log callback function for custom logging
+        """
+        self.llm = llm
+        self.metadata_map = metadata_map
+        self.work_path: Path = (
+            DEFAULT_WORK_PATH if work_path is None else Path(work_path).resolve()
+        )
+        self.metadata_path: Path = (
+            self.work_path / StorageStructure.CACHE_DIR / StorageStructure.METADATA_DIR
+        )
+        
+        # Store log_callback for passing to child components
+        self.log_callback = log_callback
+        
+        # Create bound logger with callback - returns AsyncLogger instance
+        self._log = create_logger(log_callback=log_callback)
+
+        self.llm_usages: List[Dict[str, Any]] = []
+
+    @staticmethod
+    def _get_file_info(
+        file_or_url: str, metadata_path: Union[str, Path]
+    ) -> Optional[FileInfo]:
+
+        cache_key: str = FileInfo.get_cache_key(file_or_url=file_or_url)
+        meta_file: Path = Path(metadata_path) / f"{cache_key}.json"
+
+        if not meta_file.exists():
+            return None
+
+        with open(meta_file, "r", encoding="utf-8") as f:
+            metadata_content = json.load(f)
+
+        return FileInfo.from_dict(info=metadata_content)
+
+    @staticmethod
+    def _compose_cluster_text(
+        name: Optional[str],
+        description: Union[List[str], str, None],
+        content: Union[List[str], str, None],
+    ) -> str:
+        """
+        Compose a stable text representation of a cluster from name, description, and content.
+        This is used for deterministic cluster ID generation.
+        """
+        parts: List[str] = []
+        if name:
+            parts.append(str(name).strip())
+
+        if description:
+            if isinstance(description, list):
+                parts.extend([str(item).strip() for item in description if item])
+            else:
+                parts.append(str(description).strip())
+
+        if content:
+            if isinstance(content, list):
+                parts.extend([str(item).strip() for item in content if item])
+            else:
+                parts.append(str(content).strip())
+
+        return "\n\n".join([part for part in parts if part])
+
+    async def build(
+        self,
+        request: Request,
+        retrieved_infos: List[Dict[str, Any]],
+        keywords: Dict[str, float] = None,
+        top_k_files: Optional[int] = 3,
+        top_k_snippets: Optional[int] = 5,
+        confidence_threshold: Optional[float] = 8.0,
+        verbose: bool = True,
+    ) -> Union[KnowledgeCluster, None]:
+        """Build a knowledge cluster from retrieved information and metadata dynamically."""
+
+        if len(retrieved_infos) == 0:
+            await self._log.warning(
+                "No retrieved information available to build knowledge cluster."
+            )
+            return None
+
+        retrieved_infos = retrieved_infos[:top_k_files]
+
+        keywords = keywords or {}
+
+        # Get evidence units (regions of interest) from raw retrieved infos
+        evidences: List[EvidenceUnit] = []
+        for info in retrieved_infos:
+            file_path_or_url: str = info["path"]
+
+            # TODO: handle more file types; deal with large files; Async adaptive
+            extraction_result = await fast_extract(file_path=file_path_or_url)
+            doc_content: str = extraction_result.content
+
+            sampler = MonteCarloEvidenceSampling(
+                llm=self.llm,
+                doc_content=doc_content,
+                verbose=verbose,
+                log_callback=self.log_callback,
+            )
+            roi_result: RoiResult = await sampler.get_roi(
+                query=request.get_user_input(),
+                keywords=keywords,
+                confidence_threshold=confidence_threshold,
+                top_k=top_k_snippets,
+            )
+
+            evidence_unit = EvidenceUnit(
+                doc_id=FileInfo.get_cache_key(file_path_or_url),
+                file_or_url=Path(file_path_or_url),
+                summary=roi_result.summary,
+                is_found=roi_result.is_found,
+                snippets=roi_result.snippets,
+                extracted_at=datetime.now(),
+                conflict_group=[],
+            )
+            self.llm_usages.extend(sampler.llm_usages)
+            evidences.append(evidence_unit)
+
+        if len(evidences) == 0:
+            await self._log.warning("No evidence units extracted from retrieved information.")
+            return None
+
+        # Get `name`, `description` and `content` from user request and evidences using LLM
+        # TODO: to be processed other type of segments
+        evidence_contents: List[str] = [ev.summary for ev in evidences]
+
+        evidence_summary_prompt: str = EVIDENCE_SUMMARY.format(
+            user_input=request.get_user_input(),
+            evidences="\n\n".join(evidence_contents),
+        )
+
+        evidence_summary_llm_response = await self.llm.achat(
+            messages=[{"role": "user", "content": evidence_summary_prompt}],
+            stream=True,
+        )
+        evidence_summary_response: str = evidence_summary_llm_response.content
+        self.llm_usages.append(evidence_summary_llm_response.usage)
+
+        cluster_infos: Dict[str, Any] = extract_fields(
+            content=evidence_summary_response
+        )
+        if len(cluster_infos) == 0:
+            await self._log.warning(
+                "Failed to extract knowledge cluster information from LLM response."
+            )
+            return None
+
+        cluster_name = cluster_infos.get("name")
+        cluster_description = cluster_infos.get("description")
+        cluster_content = cluster_infos.get("content")
+
+        cluster_text = self._compose_cluster_text(
+            name=cluster_name,
+            description=cluster_description,
+            content=cluster_content,
+        )
+        if not cluster_text:
+            cluster_text = request.get_user_input() or "unknown"
+
+        cluster_id = f"C{hashlib.sha256(cluster_text.encode('utf-8')).hexdigest()[:10]}"
+
+        cluster = KnowledgeCluster(
+            id=cluster_id,
+            name=cluster_name,
+            description=[cluster_description] if cluster_description else [],
+            content=cluster_content,
+            scripts=[],
+            resources=[],
+            patterns=[],
+            constraints=[],
+            evidences=evidences,
+            confidence=0.5,
+            abstraction_level=AbstractionLevel.TECHNIQUE,
+            landmark_potential=0.5,
+            hotness=0.5,
+            lifecycle=Lifecycle.EMERGING,
+            create_time=datetime.now(),
+            last_modified=datetime.now(),
+            version=1,
+            related_clusters=[],
+        )
+
+        return cluster

sirchmunk/llm/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from .openai_chat import OpenAIChat

sirchmunk/llm/openai_chat.py

@@ -0,0 +1,247 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from dataclasses import dataclass, field
+
+from openai import AsyncOpenAI, OpenAI
+from sirchmunk.utils import create_logger, LogCallback
+
+if TYPE_CHECKING:
+    pass
+
+
+@dataclass
+class OpenAIChatResponse:
+    """
+    Data class representing the response from the OpenAI Chat API.
+    """
+    content: str
+    role: str = "assistant"
+    usage: Dict[str, int] = field(default_factory=dict)
+    model: str = None
+    finish_reason: str = None
+    logprobs: Any = None
+
+    def __str__(self):
+        return self.content
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Convert the response to a dictionary.
+
+        Returns:
+            Dict[str, Any]: The response as a dictionary.
+        """
+        return {
+            "content": self.content,
+            "role": self.role,
+            "usage": self.usage,
+            "model": self.model,
+            "finish_reason": self.finish_reason,
+            "logprobs": self.logprobs,
+        }
+
+
+class OpenAIChat:
+    """
+    A client for interacting with OpenAI's chat completion API.
+    """
+
+    def __init__(
+            self,
+            api_key: str = None,
+            base_url: str = None,
+            model: str = None,
+            log_callback: LogCallback = None,
+            **kwargs,
+    ):
+        """
+        Initialize the OpenAIChat client.
+
+        Args:
+            api_key (str): The API key for OpenAI.
+            base_url (str): The base URL for the OpenAI API.
+            model (str): The model to use for chat completions.
+            log_callback (LogCallback): Optional callback for logging.
+            **kwargs: Additional keyword arguments passed to the OpenAI client create method.
+        """
+        self._client = OpenAI(
+            api_key=api_key,
+            base_url=base_url,
+        )
+
+        self._async_client = AsyncOpenAI(
+            api_key=api_key,
+            base_url=base_url,
+        )
+
+        self._model = model
+        self._kwargs = kwargs
+
+        # Initialize synchronous and asynchronous loggers
+        self._logger = create_logger(log_callback=log_callback, enable_async=False)
+        self._logger_async = create_logger(log_callback=log_callback, enable_async=True)
+
+    def chat(
+            self,
+            messages: List[Dict[str, Any]],
+            stream: bool = True,
+    ) -> OpenAIChatResponse:
+        """
+        Generate a chat completion synchronously.
+
+        Args:
+            messages (List[Dict[str, Any]]): A list of messages for the chat.
+            stream (bool): Whether to stream the response.
+
+        Returns:
+            OpenAIChatResponse: The structured response containing content, usage, etc.
+        """
+        # Ensure we try to get usage metrics even in streaming mode if supported by the API version
+        request_kwargs = self._kwargs.copy()
+        if stream and "stream_options" not in request_kwargs:
+            request_kwargs["stream_options"] = {"include_usage": True}
+
+        resp = self._client.chat.completions.create(
+            model=self._model, messages=messages, stream=stream, **request_kwargs
+        )
+
+        res_content: str = ""
+        role: str = "assistant"
+        usage: Dict[str, int] = {}
+        finish_reason: str = None
+        response_model: str = self._model
+
+        if stream:
+            for chunk in resp:
+                # Extract usage if present (usually in the last chunk if stream_options is set)
+                if chunk.usage:
+                    usage = chunk.usage.model_dump()
+
+                # Update model name if provided in chunks
+                if chunk.model:
+                    response_model = chunk.model
+
+                if not chunk.choices:
+                    continue
+
+                delta = chunk.choices[0].delta
+
+                # Capture role (usually only in the first chunk)
+                if delta.role:
+                    role = delta.role
+                    self._logger.info(f"[role={delta.role}] ", end="", flush=True)
+
+                # Capture content
+                if delta.content:
+                    self._logger.info(delta.content, end="", flush=True)
+                    res_content += delta.content
+
+                # Capture finish reason
+                if chunk.choices[0].finish_reason:
+                    finish_reason = chunk.choices[0].finish_reason
+
+            # Print a newline at the end of streaming for cleaner logs
+            self._logger.info("", end="\n", flush=True)
+
+        else:
+            # Non-streaming response
+            message = resp.choices[0].message
+            res_content = message.content or ""
+            role = message.role
+            finish_reason = resp.choices[0].finish_reason
+            response_model = resp.model
+            if resp.usage:
+                usage = resp.usage.model_dump()
+
+            # Log the full response content since we didn't stream it
+            self._logger.info(f"[role={role}] {res_content}")
+
+        return OpenAIChatResponse(
+            content=res_content,
+            role=role,
+            usage=usage,
+            model=response_model,
+            finish_reason=finish_reason
+        )
+
+    async def achat(
+            self,
+            messages: List[Dict[str, Any]],
+            stream: bool = True,
+    ) -> OpenAIChatResponse:
+        """
+        Generate a chat completion asynchronously.
+
+        Args:
+            messages (List[Dict[str, Any]]): A list of messages for the chat.
+            stream (bool): Whether to stream the response.
+
+        Returns:
+            OpenAIChatResponse: The structured response containing content, usage, etc.
+        """
+        # Ensure we try to get usage metrics even in streaming mode
+        request_kwargs = self._kwargs.copy()
+        if stream and "stream_options" not in request_kwargs:
+            request_kwargs["stream_options"] = {"include_usage": True}
+
+        resp = await self._async_client.chat.completions.create(
+            model=self._model, messages=messages, stream=stream, **request_kwargs
+        )
+
+        res_content: str = ""
+        role: str = "assistant"
+        usage: Dict[str, int] = {}
+        finish_reason: str = None
+        response_model: str = self._model
+
+        if stream:
+            async for chunk in resp:
+                # Extract usage if present (usually in the last chunk if stream_options is set)
+                if chunk.usage:
+                    usage = chunk.usage.model_dump()
+
+                if chunk.model:
+                    response_model = chunk.model
+
+                if not chunk.choices:
+                    continue
+
+                delta = chunk.choices[0].delta
+
+                # Capture role
+                if delta.role:
+                    role = delta.role
+                    await self._logger_async.info(f"[role={delta.role}] ", end="", flush=True)
+
+                # Capture content
+                if delta.content:
+                    await self._logger_async.info(delta.content, end="", flush=True)
+                    res_content += delta.content
+
+                # Capture finish reason
+                if chunk.choices[0].finish_reason:
+                    finish_reason = chunk.choices[0].finish_reason
+
+            # Print a newline at the end of streaming for cleaner logs
+            await self._logger_async.info("", end="\n", flush=True)
+
+        else:
+            # Non-streaming response
+            message = resp.choices[0].message
+            res_content = message.content or ""
+            role = message.role
+            finish_reason = resp.choices[0].finish_reason
+            response_model = resp.model
+            if resp.usage:
+                usage = resp.usage.model_dump()
+
+            # Log the full response content since we didn't stream it
+            await self._logger_async.info(f"[role={role}] {res_content}")
+
+        return OpenAIChatResponse(
+            content=res_content,
+            role=role,
+            usage=usage,
+            model=response_model,
+            finish_reason=finish_reason
+        )

sirchmunk/llm/prompts.py

@@ -0,0 +1,216 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+# flake8: noqa
+# yapf: disable
+
+
+SNAPSHOT_KEYWORDS_EXTRACTION = """
+Analyze the following document and extract the most relevant and representative key phrases.
+Prioritize terms that capture the core topics, central concepts, important entities (e.g., people, organizations, locations), and domain-specific terminology.
+Exclude generic words (e.g., "the", "and", "result", "study") unless they are part of a meaningful multi-word phrase.
+Limit the output to {max_num} concise key phrases, ranked by significance.
+You **MUST** output the key phrases as a comma-separated list without any additional explanation or formatting.
+You **MUST** adjust the language of the key phrases to be consistent with the language of the input document.
+
+**Intput Document**:
+{document_content}
+"""
+
+
+SNAPSHOT_TOC_EXTRACTION = """
+Generate a Table of Contents (ToC) from the given document, adapting its depth and content density to the document’s inherent complexity.
+
+Requirements:
+
+1. Adaptive Hierarchy Depth: Dynamically set the depth between 3 to 5 levels, based on the document’s structural and semantic complexity (e.g., 3 levels for simple notices, 5 for technical specs).
+2. Summarized Entries: Each ToC item must concisely summarize the section’s core content (10–25 words), not just repeat headings. Capture purpose, key actions, or critical info.
+3. Faithfulness: Do not invent sections. Infer headings only from logical paragraph groupings if explicit titles are absent.
+4. Format: Use Markdown nested lists with 2-space indents per level (e.g., - → - → -). Output ToC only—no preamble or commentary.
+
+**Input Document**:
+{document_content}
+"""
+
+
+QUERY_KEYWORDS_EXTRACTION = """
+### Role: Search Optimization Expert & Information Retrieval Specialist
+
+### Task:
+Extract **{num_levels} sets** of keywords from the user query with **different granularities** to maximize search hit rate.
+
+### Multi-Level Keyword Granularity Strategy:
+
+Extract {num_levels} levels of keywords with progressively finer granularity:
+
+{level_descriptions}
+
+### IDF Value Guidelines:
+- Estimate the **IDF (Inverse Document Frequency)** for each keyword based on its rarity in general corpus
+- IDF range: **[0-10]** where:
+  - 0-3: Very common terms (e.g., "the", "is", "data")
+  - 4-6: Moderately common terms (e.g., "algorithm", "network")
+  - 7-9: Rare/specific terms (e.g., "backpropagation", "xgboost")
+  - 10: Extremely rare/specialized terms
+- IDF values are **independent** of keyword level - focus on term rarity, not granularity
+
+### Requirements:
+- Each level should have 3-5 keywords
+- Keywords must become progressively **finer-grained** from Level 1 to Level {num_levels}
+- **Level 1**: Coarse-grained phrases/multi-word expressions
+- **Level {num_levels}**: Fine-grained single words or precise technical terms
+- ONLY extract from the user query context; do NOT add external information
+- Ensure keywords at different levels are complementary, not redundant
+
+### Output Format:
+Output {num_levels} separate JSON-like dicts within their respective tags:
+
+{output_format_example}
+
+### User Query:
+{{user_input}}
+
+### {num_levels}-Level Keywords (Coarse to Fine):
+"""
+
+
+def generate_keyword_extraction_prompt(num_levels: int = 3) -> str:
+    """
+    Generate a dynamic keyword extraction prompt template based on the number of levels.
+    
+    The returned template still contains {{user_input}} placeholder that needs to be
+    filled in by the caller.
+    
+    Args:
+        num_levels: Number of granularity levels (default: 3)
+    
+    Returns:
+        Prompt template string with {{user_input}} placeholder
+    """
+    # Generate level descriptions with granularity focus
+    level_descriptions = []
+    for i in range(1, num_levels + 1):
+        # Define granularity characteristics
+        if i == 1:
+            granularity = "Coarse-grained"
+            desc_text = "Multi-word phrases, compound expressions, broader concepts"
+            examples = '"machine learning algorithms", "data processing pipeline", "neural network training"'
+        elif i == num_levels:
+            granularity = "Fine-grained"
+            desc_text = "Single words, precise terms, atomic concepts"
+            examples = '"optimization", "gradient", "tensor", "epoch"'
+        else:
+            granularity = f"Medium-grained (Level {i})"
+            desc_text = "2-3 word phrases or compound terms transitioning to single words"
+            examples = '"deep learning", "batch normalization", "learning rate"'
+        
+        level_descriptions.append(
+            f"**Level {i}** ({granularity}):\n"
+            f"   - Granularity: {desc_text}\n"
+            f"   - Example keywords: {examples}\n"
+            f"   - Note: IDF values should reflect term rarity, not granularity level"
+        )
+    
+    # Generate output format examples (avoiding f-string interpolation issues)
+    output_examples = []
+    for i in range(1, num_levels + 1):
+        # Use double braces to escape them in the format string
+        example_dict = '{{"keyword1": idf_value, "keyword2": idf_value, ...}}'
+        output_examples.append(
+            f"<KEYWORDS_LEVEL_{i}>\n{example_dict}\n</KEYWORDS_LEVEL_{i}>"
+        )
+    
+    # Format the template with num_levels, descriptions, and examples
+    # Note: {{user_input}} becomes {user_input} after this format call
+    return QUERY_KEYWORDS_EXTRACTION.format(
+        num_levels=num_levels,
+        level_descriptions="\n\n".join(level_descriptions),
+        output_format_example="\n\n".join(output_examples)
+    )
+
+
+EVIDENCE_SUMMARY = """
+## Role: High-Precision Information Synthesis Expert
+
+## Task:
+Synthesize a structured response based on the User Input and the provided Evidences.
+
+### Critical Constraint:
+1. **Language Consistency:** All output fields (<DESCRIPTION>, <NAME>, and <CONTENT>) MUST be written in the **same language** as the User Input.
+2. **Ignore irrelevant noise:** Focus exclusively on information that directly relates to the User Input. If evidences contain conflicting or redundant data, prioritize accuracy and relevance.
+
+### Input Data:
+- **User Input:** {user_input}
+- **Retrieved Evidences:** {evidences}
+
+### Output Instructions:
+1. **<DESCRIPTION>**: A high-level, concise synthesis of how the evidences address the user input.
+   - *Constraint:* Maximum 3 sentences. Written in the language of {user_input}.
+2. **<NAME>**: A ultra-short, catchy title or identifier for the description.
+   - *Constraint:* Exactly 1 sentence, maximum 30 characters. Written in the language of {user_input}.
+3. **<CONTENT>**: A detailed and comprehensive summary of all relevant key points extracted from the evidences.
+   - *Constraint:* Written in the language of {user_input}.
+
+### Output Format:
+<DESCRIPTION>[Concise synthesis]</DESCRIPTION>
+<NAME>[Short title]</NAME>
+<CONTENT>[Detailed summary]</CONTENT>
+"""
+
+
+SEARCH_RESULT_SUMMARY = """
+### Task
+Analyze the provided {text_content} and generate a concise summary in the form of a Markdown Briefing.
+
+### Constraints
+1. **Language Continuity**: The output must be in the SAME language as the User Input.
+2. **Format**: Use Markdown (headings, bullet points, and bold text) for high readability.
+3. **Style**: Keep it professional, objective, and clear. Avoid fluff.
+
+### Input Data
+- **User Input**: {user_input}
+- **Search Result Text**: {text_content}
+
+### Output
+[Generate the Markdown Briefing here]
+"""
+
+
+EVALUATE_EVIDENCE_SAMPLE = """
+You are a document retrieval assistant. Please evaluate if the text snippet contains clues to answer the user's question.
+
+### Language Constraint:
+Detect the language of the "Query" and provide the "reasoning" and "output" in the same language (e.g., if the query is in Chinese, the reasoning must be in Chinese).
+
+### Inputs:
+Query: "{query}"
+
+Text Snippet (Source: {sample_source}):
+"...{sample_content}..."
+
+### Output Requirement:
+Return JSON:
+- score (0-10):
+  0-3: Completely irrelevant.
+  4-7: Contains relevant keywords or context but no direct answer.
+  8-10: Contains exact data, facts, or direct answer.
+- reasoning: Short reasoning in the SAME language as the query.
+
+JSON format only.
+"""
+
+
+ROI_RESULT_SUMMARY = """
+### Task
+Analyze the provided {text_content} and generate a concise summary in the form of a Markdown Briefing.
+
+### Constraints
+1. **Language Continuity**: The output must be in the SAME language as the User Input.
+2. **Format**: Use Markdown (headings, bullet points, and bold text) for high readability.
+3. **Style**: Keep it professional, objective, and clear. Avoid fluff.
+
+### Input Data
+- **User Input**: {user_input}
+- **Search Result Text**: {text_content}
+
+### Output
+[Generate the Markdown Briefing here]
+"""

sirchmunk/retrieve/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.