hyperagents 1.2.3rc5.post3__py3-none-any.whl

civilization/retrieval/__init__.py

@@ -0,0 +1,17 @@
+"""
+Retrieval module for Civilization Knowledge Layer.
+Handles RAG, context injection, and relevance ranking.
+"""
+from .civilization_rag import CivilizationRAG, RAGResult, RAGConfig
+from .context_injector import ContextInjector, InjectionMode
+from .relevance_ranker import RelevanceRanker, RankingResult
+
+__all__ = [
+    "CivilizationRAG",
+    "RAGResult",
+    "RAGConfig",
+    "ContextInjector",
+    "InjectionMode",
+    "RelevanceRanker",
+    "RankingResult",
+]

civilization/retrieval/civilization_rag.py

@@ -0,0 +1,316 @@
+"""
+RAG (Retrieval Augmented Generation) for Civilization Knowledge Layer.
+Retrieves and formats organizational knowledge for agent consumption.
+"""
+from __future__ import annotations
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Callable
+from uuid import UUID
+
+from ..schema import CivilizationNode, NodeType
+from ..store import CivilizationStore
+from ..ingestion.embedder import CivilizationEmbedder
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class RAGConfig:
+    """Configuration for RAG retrieval."""
+    top_k: int = 5
+    min_relevance_score: float = 0.5
+    include_metadata: bool = True
+    max_context_tokens: int = 4000
+    node_type_filters: list[NodeType] | None = None
+    tag_filters: list[str] | None = None
+    rerank: bool = True
+    expand_related: bool = True
+
+
+@dataclass
+class RetrievedNode:
+    """A node retrieved by RAG with relevance metadata."""
+    node: CivilizationNode
+    score: float
+    source: str = "vector"  # "vector", "keyword", "related"
+    highlights: list[str] = field(default_factory=list)
+
+
+@dataclass
+class RAGResult:
+    """Result of a RAG query."""
+    query: str
+    retrieved_nodes: list[RetrievedNode] = field(default_factory=list)
+    context: str = ""
+    metadata: dict = field(default_factory=dict)
+
+    @property
+    def node_count(self) -> int:
+        return len(self.retrieved_nodes)
+
+    @property
+    def has_results(self) -> bool:
+        return len(self.retrieved_nodes) > 0
+
+    @property
+    def top_node(self) -> CivilizationNode | None:
+        if self.retrieved_nodes:
+            return self.retrieved_nodes[0].node
+        return None
+
+
+class CivilizationRAG:
+    """
+    RAG system for retrieving and formatting organizational knowledge.
+    Supports vector search, keyword fallback, and context expansion.
+    """
+
+    def __init__(
+        self,
+        org_id: str,
+        store: CivilizationStore | None = None,
+        embedder: CivilizationEmbedder | None = None,
+        config: RAGConfig | None = None,
+    ):
+        self.org_id = org_id
+        self.store = store
+        self.embedder = embedder or CivilizationEmbedder()
+        self.config = config or RAGConfig()
+        self._cache: dict[str, list[RetrievedNode]] = {}
+
+    async def query(
+        self,
+        query: str,
+        config_override: RAGConfig | None = None,
+    ) -> RAGResult:
+        """
+        Query the knowledge base and retrieve relevant nodes.
+
+        Args:
+            query: The search query
+            config_override: Optional config to override defaults
+
+        Returns:
+            RAGResult with retrieved nodes and formatted context
+        """
+        config = config_override or self.config
+        result = RAGResult(query=query)
+
+        # Generate query embedding
+        query_embedding = await self.embedder.embed(query)
+
+        # Vector search
+        if self.store:
+            vector_results = await self.store.search_by_embedding(
+                org_id=self.org_id,
+                embedding=query_embedding,
+                top_k=config.top_k * 2,  # Over-fetch for reranking
+                node_types=config.node_type_filters,
+            )
+
+            for row in vector_results:
+                score = row.get("score", 0.0)
+                if score >= config.min_relevance_score:
+                    node = self.store._row_to_node(row)
+                    result.retrieved_nodes.append(RetrievedNode(
+                        node=node,
+                        score=score,
+                        source="vector",
+                    ))
+
+        # Sort by score
+        result.retrieved_nodes.sort(key=lambda x: x.score, reverse=True)
+
+        # Truncate to top_k
+        result.retrieved_nodes = result.retrieved_nodes[:config.top_k]
+
+        # Expand with related nodes if enabled
+        if config.expand_related and result.retrieved_nodes:
+            await self._expand_related(result, config)
+
+        # Generate context string
+        result.context = self._format_context(result.retrieved_nodes, config)
+
+        result.metadata = {
+            "total_retrieved": len(result.retrieved_nodes),
+            "config": {
+                "top_k": config.top_k,
+                "min_score": config.min_relevance_score,
+            },
+        }
+
+        return result
+
+    async def query_by_type(
+        self,
+        query: str,
+        node_type: NodeType,
+        top_k: int = 5,
+    ) -> RAGResult:
+        """Query for a specific node type."""
+        config = RAGConfig(
+            top_k=top_k,
+            node_type_filters=[node_type],
+        )
+        return await self.query(query, config)
+
+    async def get_sop_for_task(self, task_description: str) -> RAGResult:
+        """Find relevant SOPs for a task."""
+        return await self.query_by_type(
+            query=f"SOP procedure for: {task_description}",
+            node_type=NodeType.SOP,
+            top_k=3,
+        )
+
+    async def get_role_info(self, role_query: str) -> RAGResult:
+        """Find role definitions."""
+        config = RAGConfig(
+            top_k=3,
+            node_type_filters=[NodeType.ROLE, NodeType.JOB_DESCRIPTION],
+        )
+        return await self.query(role_query, config)
+
+    async def get_runbook_for_incident(self, incident_description: str) -> RAGResult:
+        """Find relevant runbooks for an incident."""
+        return await self.query_by_type(
+            query=f"Runbook for incident: {incident_description}",
+            node_type=NodeType.RUNBOOK,
+            top_k=3,
+        )
+
+    async def _expand_related(self, result: RAGResult, config: RAGConfig) -> None:
+        """Expand results with related nodes."""
+        if not self.store:
+            return
+
+        # Get IDs of related nodes from top results
+        related_ids: set[UUID] = set()
+        for rn in result.retrieved_nodes[:3]:
+            # Check for related_sops, related_roles, etc.
+            node_dict = rn.node.model_dump()
+            for key, value in node_dict.items():
+                if key.startswith("related_") and isinstance(value, list):
+                    for item in value:
+                        if isinstance(item, (UUID, str)):
+                            try:
+                                related_ids.add(UUID(str(item)))
+                            except ValueError:
+                                pass
+
+        # Fetch and add related nodes
+        existing_ids = {rn.node.id for rn in result.retrieved_nodes}
+        for rel_id in related_ids:
+            if rel_id not in existing_ids and len(result.retrieved_nodes) < config.top_k + 2:
+                related = await self.store.get(rel_id, self.org_id)
+                if related:
+                    result.retrieved_nodes.append(RetrievedNode(
+                        node=related,
+                        score=0.5,  # Default score for related
+                        source="related",
+                    ))
+
+    def _format_context(
+        self,
+        nodes: list[RetrievedNode],
+        config: RAGConfig,
+    ) -> str:
+        """Format retrieved nodes into context string for LLM."""
+        if not nodes:
+            return ""
+
+        parts = []
+        token_estimate = 0
+        max_tokens = config.max_context_tokens
+
+        for i, rn in enumerate(nodes, 1):
+            node = rn.node
+            section = self._format_node(node, i, config.include_metadata)
+
+            # Rough token estimate (4 chars per token)
+            section_tokens = len(section) // 4
+            if token_estimate + section_tokens > max_tokens:
+                break
+
+            parts.append(section)
+            token_estimate += section_tokens
+
+        return "\n\n---\n\n".join(parts)
+
+    def _format_node(
+        self,
+        node: CivilizationNode,
+        index: int,
+        include_metadata: bool,
+    ) -> str:
+        """Format a single node for context."""
+        lines = [f"[{index}] {node.node_type.value.upper()}: {node.title}"]
+
+        if include_metadata:
+            if node.owner_id:
+                lines.append(f"Owner: {node.owner_id}")
+            if node.tags:
+                lines.append(f"Tags: {', '.join(node.tags)}")
+
+        # Type-specific formatting
+        node_dict = node.model_dump()
+
+        if node.node_type == NodeType.SOP:
+            if "purpose" in node_dict:
+                lines.append(f"Purpose: {node_dict['purpose']}")
+            if "steps" in node_dict:
+                lines.append("Steps:")
+                for step in node_dict["steps"][:5]:
+                    lines.append(f"  {step.get('step_number', '?')}. {step.get('title', 'Untitled')}")
+                    if step.get("description"):
+                        lines.append(f"     {step['description'][:150]}...")
+
+        elif node.node_type == NodeType.ROLE:
+            if "role_title" in node_dict:
+                lines.append(f"Title: {node_dict['role_title']}")
+            if "responsibilities" in node_dict:
+                lines.append("Responsibilities:")
+                for r in node_dict["responsibilities"][:5]:
+                    lines.append(f"  - {r}")
+
+        elif node.node_type == NodeType.RUNBOOK:
+            if "scenario" in node_dict:
+                lines.append(f"Scenario: {node_dict['scenario']}")
+            if "steps" in node_dict:
+                lines.append("Steps:")
+                for step in node_dict["steps"][:5]:
+                    lines.append(f"  {step.get('step_number', '?')}. {step.get('action', 'Action')}")
+
+        elif node.node_type == NodeType.CHECKLIST:
+            if "items" in node_dict:
+                lines.append("Items:")
+                for item in node_dict["items"][:8]:
+                    lines.append(f"  [ ] {item.get('description', '')}")
+
+        return "\n".join(lines)
+
+    async def hybrid_search(
+        self,
+        query: str,
+        keyword_weight: float = 0.3,
+    ) -> RAGResult:
+        """
+        Perform hybrid search combining vector and keyword matching.
+        """
+        # Vector search
+        vector_result = await self.query(query)
+
+        # For keyword search, we'd need text search capability in the store
+        # This is a placeholder that could be enhanced
+        result = RAGResult(
+            query=query,
+            retrieved_nodes=vector_result.retrieved_nodes,
+            metadata={"search_type": "hybrid"},
+        )
+
+        result.context = self._format_context(result.retrieved_nodes, self.config)
+        return result
+
+    def clear_cache(self) -> None:
+        """Clear the query cache."""
+        self._cache.clear()

civilization/retrieval/context_injector.py

@@ -0,0 +1,342 @@
+"""
+Context injection for agent prompts.
+Formats and injects organizational knowledge into agent context.
+"""
+from __future__ import annotations
+import logging
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+from ..schema import CivilizationNode, NodeType
+
+logger = logging.getLogger(__name__)
+
+
+class InjectionMode(str, Enum):
+    FULL = "full"  # Include complete node content
+    SUMMARY = "summary"  # Include only summaries
+    REFERENCE = "reference"  # Include only titles and IDs
+    STRUCTURED = "structured"  # Include structured data format
+    MINIMAL = "minimal"  # Bare minimum context
+
+
+@dataclass
+class InjectionConfig:
+    """Configuration for context injection."""
+    mode: InjectionMode = InjectionMode.SUMMARY
+    max_tokens: int = 2000
+    include_metadata: bool = True
+    include_relationships: bool = True
+    priority_types: list[NodeType] | None = None
+    format: str = "markdown"  # "markdown", "xml", "json"
+
+
+@dataclass
+class InjectionResult:
+    """Result of context injection."""
+    context: str
+    nodes_included: int
+    truncated: bool = False
+    token_estimate: int = 0
+
+
+class ContextInjector:
+    """
+    Injects civilization knowledge into agent prompts.
+    Handles formatting, token budgeting, and prioritization.
+    """
+
+    PRIORITY_ORDER = [
+        NodeType.SOP,
+        NodeType.RUNBOOK,
+        NodeType.CHECKLIST,
+        NodeType.ROLE,
+        NodeType.WORKFLOW,
+        NodeType.POLICY,
+        NodeType.JOB_DESCRIPTION,
+        NodeType.CLIENT_PROFILE,
+        NodeType.KNOWLEDGE_ARTICLE,
+        NodeType.ORG_CHART,
+        NodeType.PERSON,
+        NodeType.PERSONAL_ROUTINE,
+    ]
+
+    def __init__(self, config: InjectionConfig | None = None):
+        self.config = config or InjectionConfig()
+
+    def inject(
+        self,
+        nodes: list[CivilizationNode],
+        config_override: InjectionConfig | None = None,
+    ) -> InjectionResult:
+        """
+        Generate injection context from nodes.
+
+        Args:
+            nodes: Nodes to include in context
+            config_override: Optional config override
+
+        Returns:
+            InjectionResult with formatted context
+        """
+        config = config_override or self.config
+        result = InjectionResult(context="", nodes_included=0)
+
+        if not nodes:
+            return result
+
+        # Sort by priority
+        sorted_nodes = self._prioritize_nodes(nodes, config)
+
+        # Format based on mode
+        formatter = self._get_formatter(config.format)
+        sections = []
+        total_tokens = 0
+
+        for node in sorted_nodes:
+            section = self._format_node(node, config)
+            section_tokens = len(section) // 4  # Rough estimate
+
+            if total_tokens + section_tokens > config.max_tokens:
+                result.truncated = True
+                break
+
+            sections.append(section)
+            total_tokens += section_tokens
+            result.nodes_included += 1
+
+        result.context = formatter(sections, config)
+        result.token_estimate = total_tokens
+
+        return result
+
+    def _prioritize_nodes(
+        self,
+        nodes: list[CivilizationNode],
+        config: InjectionConfig,
+    ) -> list[CivilizationNode]:
+        """Sort nodes by priority."""
+        priority_types = config.priority_types or self.PRIORITY_ORDER
+
+        def sort_key(node: CivilizationNode) -> tuple:
+            try:
+                type_priority = priority_types.index(node.node_type)
+            except ValueError:
+                type_priority = len(priority_types)
+            return (type_priority, -node.updated_at.timestamp())
+
+        return sorted(nodes, key=sort_key)
+
+    def _format_node(
+        self,
+        node: CivilizationNode,
+        config: InjectionConfig,
+    ) -> str:
+        """Format a single node based on injection mode."""
+        if config.mode == InjectionMode.REFERENCE:
+            return self._format_reference(node)
+        elif config.mode == InjectionMode.MINIMAL:
+            return self._format_minimal(node)
+        elif config.mode == InjectionMode.SUMMARY:
+            return self._format_summary(node, config)
+        elif config.mode == InjectionMode.STRUCTURED:
+            return self._format_structured(node, config)
+        else:  # FULL
+            return self._format_full(node, config)
+
+    def _format_reference(self, node: CivilizationNode) -> str:
+        """Format as reference only."""
+        return f"- [{node.node_type.value}] {node.title} (ID: {node.id})"
+
+    def _format_minimal(self, node: CivilizationNode) -> str:
+        """Format with minimal information."""
+        return f"{node.node_type.value.upper()}: {node.title}"
+
+    def _format_summary(
+        self,
+        node: CivilizationNode,
+        config: InjectionConfig,
+    ) -> str:
+        """Format as summary."""
+        lines = [f"**{node.node_type.value.upper()}**: {node.title}"]
+
+        node_dict = node.model_dump()
+
+        # Add key fields based on node type
+        if node.node_type == NodeType.SOP and "purpose" in node_dict:
+            lines.append(f"Purpose: {node_dict['purpose']}")
+            if "steps" in node_dict:
+                step_count = len(node_dict["steps"])
+                lines.append(f"Steps: {step_count} total")
+
+        elif node.node_type == NodeType.ROLE:
+            if "role_title" in node_dict:
+                lines.append(f"Role: {node_dict['role_title']}")
+            if "responsibilities" in node_dict:
+                lines.append(f"Key responsibilities: {len(node_dict['responsibilities'])} defined")
+
+        elif node.node_type == NodeType.RUNBOOK:
+            if "scenario" in node_dict:
+                lines.append(f"Scenario: {node_dict['scenario']}")
+            if "severity" in node_dict:
+                lines.append(f"Severity: {node_dict['severity']}")
+
+        if config.include_metadata and node.tags:
+            lines.append(f"Tags: {', '.join(node.tags[:5])}")
+
+        return "\n".join(lines)
+
+    def _format_structured(
+        self,
+        node: CivilizationNode,
+        config: InjectionConfig,
+    ) -> str:
+        """Format as structured data."""
+        import json
+        data = {
+            "type": node.node_type.value,
+            "title": node.title,
+            "id": str(node.id),
+        }
+
+        node_dict = node.model_dump()
+
+        # Include type-specific fields
+        if node.node_type == NodeType.SOP:
+            data["purpose"] = node_dict.get("purpose", "")
+            data["step_count"] = len(node_dict.get("steps", []))
+            data["roles"] = node_dict.get("roles_involved", [])
+
+        elif node.node_type == NodeType.ROLE:
+            data["role_title"] = node_dict.get("role_title", "")
+            data["department"] = node_dict.get("department", "")
+
+        if config.include_metadata:
+            data["tags"] = node.tags
+            data["owner"] = node.owner_id
+
+        return json.dumps(data, indent=2)
+
+    def _format_full(
+        self,
+        node: CivilizationNode,
+        config: InjectionConfig,
+    ) -> str:
+        """Format with full content."""
+        lines = [
+            f"## {node.node_type.value.upper()}: {node.title}",
+            "",
+        ]
+
+        node_dict = node.model_dump()
+
+        # Full content varies by type
+        if node.node_type == NodeType.SOP:
+            lines.append(f"**Purpose:** {node_dict.get('purpose', 'N/A')}")
+            lines.append(f"**Scope:** {node_dict.get('scope', 'N/A')}")
+            lines.append("")
+
+            if "steps" in node_dict:
+                lines.append("### Steps")
+                for step in node_dict["steps"]:
+                    lines.append(f"{step.get('step_number', '?')}. **{step.get('title', 'Step')}**")
+                    lines.append(f"   {step.get('description', '')}")
+                    if step.get("responsible_role"):
+                        lines.append(f"   - Role: {step['responsible_role']}")
+                    if step.get("tools_required"):
+                        lines.append(f"   - Tools: {', '.join(step['tools_required'])}")
+                    lines.append("")
+
+        elif node.node_type == NodeType.ROLE:
+            lines.append(f"**Title:** {node_dict.get('role_title', 'N/A')}")
+            lines.append(f"**Department:** {node_dict.get('department', 'N/A')}")
+            lines.append("")
+
+            if "responsibilities" in node_dict:
+                lines.append("### Responsibilities")
+                for r in node_dict["responsibilities"]:
+                    lines.append(f"- {r}")
+
+            if "accountabilities" in node_dict:
+                lines.append("")
+                lines.append("### Accountabilities")
+                for a in node_dict["accountabilities"]:
+                    lines.append(f"- {a}")
+
+        elif node.node_type == NodeType.CHECKLIST:
+            lines.append(f"**Purpose:** {node_dict.get('purpose', 'N/A')}")
+            lines.append("")
+
+            if "items" in node_dict:
+                lines.append("### Items")
+                for item in node_dict["items"]:
+                    req = "[Required]" if item.get("required") else "[Optional]"
+                    lines.append(f"- {req} {item.get('description', '')}")
+
+        if config.include_metadata:
+            lines.append("")
+            lines.append("---")
+            lines.append(f"*Owner: {node.owner_id or 'Unassigned'}*")
+            lines.append(f"*Tags: {', '.join(node.tags) if node.tags else 'None'}*")
+            lines.append(f"*Last updated: {node.updated_at.isoformat()}*")
+
+        return "\n".join(lines)
+
+    def _get_formatter(self, format_type: str):
+        """Get formatter function for output format."""
+        if format_type == "xml":
+            return self._format_as_xml
+        elif format_type == "json":
+            return self._format_as_json
+        else:  # markdown
+            return self._format_as_markdown
+
+    def _format_as_markdown(
+        self,
+        sections: list[str],
+        config: InjectionConfig,
+    ) -> str:
+        """Format sections as markdown."""
+        header = "# Organizational Knowledge Context\n\n"
+        return header + "\n\n---\n\n".join(sections)
+
+    def _format_as_xml(
+        self,
+        sections: list[str],
+        config: InjectionConfig,
+    ) -> str:
+        """Format sections as XML."""
+        import html
+        items = "\n".join(
+            f"<knowledge_item>\n{html.escape(s)}\n</knowledge_item>"
+            for s in sections
+        )
+        return f"<organizational_knowledge>\n{items}\n</organizational_knowledge>"
+
+    def _format_as_json(
+        self,
+        sections: list[str],
+        config: InjectionConfig,
+    ) -> str:
+        """Format sections as JSON array."""
+        import json
+        return json.dumps({"knowledge_items": sections}, indent=2)
+
+    def build_system_prompt_section(
+        self,
+        nodes: list[CivilizationNode],
+        section_name: str = "ORGANIZATIONAL_CONTEXT",
+    ) -> str:
+        """
+        Build a section suitable for system prompts.
+        """
+        result = self.inject(nodes)
+        if not result.context:
+            return ""
+
+        return f"""
+<{section_name}>
+{result.context}
+</{section_name}>
+"""