hyperagents 1.2.3rc5.post3__py3-none-any.whl

civilization/ingestion/extractor.py

@@ -0,0 +1,224 @@
+"""
+Entity and metadata extraction from documents for Civilization Knowledge Layer.
+Uses LLM-based extraction for structured information.
+"""
+from __future__ import annotations
+import re
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+from uuid import UUID
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ExtractedEntity:
+    entity_type: str  # "person", "role", "tool", "system", "process", "metric"
+    name: str
+    context: str
+    confidence: float = 1.0
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass
+class ExtractedMetadata:
+    title: str | None = None
+    purpose: str | None = None
+    owner: str | None = None
+    department: str | None = None
+    tags: list[str] = field(default_factory=list)
+    roles_mentioned: list[str] = field(default_factory=list)
+    tools_mentioned: list[str] = field(default_factory=list)
+    systems_mentioned: list[str] = field(default_factory=list)
+    metrics_mentioned: list[str] = field(default_factory=list)
+    estimated_complexity: str | None = None  # "low", "medium", "high"
+    document_type: str | None = None
+
+
+class MetadataExtractor:
+    """
+    Extracts metadata from document text using pattern matching and heuristics.
+    For LLM-based extraction, use with an agent wrapper.
+    """
+
+    # Common role patterns
+    ROLE_PATTERNS = [
+        r"(?:responsible|owner|lead|manager|coordinator|assigned to|handled by)\s*[:\-]?\s*([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)",
+        r"([A-Z][a-z]+\s+(?:Manager|Director|Lead|Coordinator|Specialist|Engineer|Analyst|Administrator))",
+    ]
+
+    # Common tool patterns
+    TOOL_PATTERNS = [
+        r"(?:using|via|through|in|with)\s+([A-Z][a-zA-Z0-9]+(?:\s+[A-Z][a-zA-Z0-9]+)?)",
+        r"(Slack|Jira|Confluence|Notion|Asana|Trello|GitHub|GitLab|Salesforce|HubSpot|Zendesk)",
+    ]
+
+    # System patterns
+    SYSTEM_PATTERNS = [
+        r"([A-Z][a-zA-Z0-9]+(?:DB|API|Service|System|Platform|Server))",
+        r"(?:database|system|service|platform)\s*[:\-]?\s*([A-Z][a-zA-Z0-9]+)",
+    ]
+
+    def extract(self, text: str) -> ExtractedMetadata:
+        """Extract metadata from raw document text."""
+        metadata = ExtractedMetadata()
+
+        # Extract title (first line or heading)
+        lines = text.strip().split("\n")
+        if lines:
+            first_line = lines[0].strip()
+            # Remove markdown heading markers
+            if first_line.startswith("#"):
+                first_line = re.sub(r"^#+\s*", "", first_line)
+            if len(first_line) < 200:
+                metadata.title = first_line
+
+        # Extract purpose (look for common patterns)
+        purpose_match = re.search(
+            r"(?:purpose|objective|goal|overview)\s*[:\-]?\s*(.+?)(?:\n|$)",
+            text, re.IGNORECASE
+        )
+        if purpose_match:
+            metadata.purpose = purpose_match.group(1).strip()[:500]
+
+        # Extract roles
+        for pattern in self.ROLE_PATTERNS:
+            matches = re.findall(pattern, text)
+            metadata.roles_mentioned.extend(matches)
+        metadata.roles_mentioned = list(set(metadata.roles_mentioned))
+
+        # Extract tools
+        for pattern in self.TOOL_PATTERNS:
+            matches = re.findall(pattern, text, re.IGNORECASE)
+            metadata.tools_mentioned.extend([m for m in matches if len(m) > 2])
+        metadata.tools_mentioned = list(set(metadata.tools_mentioned))
+
+        # Extract systems
+        for pattern in self.SYSTEM_PATTERNS:
+            matches = re.findall(pattern, text)
+            metadata.systems_mentioned.extend(matches)
+        metadata.systems_mentioned = list(set(metadata.systems_mentioned))
+
+        # Extract tags from hashtags
+        hashtags = re.findall(r"#([a-zA-Z][a-zA-Z0-9_-]+)", text)
+        metadata.tags = list(set(hashtags))
+
+        # Estimate complexity based on length and structure
+        word_count = len(text.split())
+        step_count = len(re.findall(r"(?:step|item|\d+\.)\s", text, re.IGNORECASE))
+        if word_count < 300 and step_count < 5:
+            metadata.estimated_complexity = "low"
+        elif word_count < 1000 and step_count < 15:
+            metadata.estimated_complexity = "medium"
+        else:
+            metadata.estimated_complexity = "high"
+
+        # Detect document type
+        metadata.document_type = self._detect_document_type(text)
+
+        return metadata
+
+    def _detect_document_type(self, text: str) -> str | None:
+        """Detect the type of document based on content patterns."""
+        text_lower = text.lower()
+
+        if any(kw in text_lower for kw in ["sop", "standard operating procedure", "procedure:"]):
+            return "sop"
+        if any(kw in text_lower for kw in ["checklist", "□", "☐", "[ ]", "[x]"]):
+            return "checklist"
+        if any(kw in text_lower for kw in ["runbook", "playbook", "incident", "on-call"]):
+            return "runbook"
+        if any(kw in text_lower for kw in ["job description", "responsibilities:", "qualifications:"]):
+            return "job_description"
+        if any(kw in text_lower for kw in ["org chart", "organization chart", "reporting structure"]):
+            return "org_chart"
+        if any(kw in text_lower for kw in ["workflow", "process flow", "→", "->"]):
+            return "workflow"
+        if any(kw in text_lower for kw in ["policy", "must comply", "prohibited", "required"]):
+            return "policy"
+        return "knowledge_article"
+
+
+class EntityExtractor:
+    """
+    Extracts named entities and their relationships from text.
+    """
+
+    def __init__(self):
+        self.entity_patterns = {
+            "person": [
+                r"([A-Z][a-z]+\s+[A-Z][a-z]+)",  # First Last
+                r"@([a-zA-Z][a-zA-Z0-9_]+)",  # @username
+            ],
+            "email": [
+                r"([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})",
+            ],
+            "url": [
+                r"(https?://[^\s<>\"]+)",
+            ],
+            "metric": [
+                r"(\d+(?:\.\d+)?%)",  # Percentages
+                r"(\$\d+(?:,\d{3})*(?:\.\d{2})?)",  # Currency
+                r"(\d+(?:\.\d+)?\s*(?:hours?|mins?|minutes?|days?|weeks?))",  # Time durations
+            ],
+            "date": [
+                r"(\d{4}-\d{2}-\d{2})",  # ISO date
+                r"(\d{1,2}/\d{1,2}/\d{2,4})",  # US date
+            ],
+        }
+
+    def extract(self, text: str) -> list[ExtractedEntity]:
+        """Extract all entities from text."""
+        entities = []
+
+        for entity_type, patterns in self.entity_patterns.items():
+            for pattern in patterns:
+                for match in re.finditer(pattern, text):
+                    # Get surrounding context (50 chars before and after)
+                    start = max(0, match.start() - 50)
+                    end = min(len(text), match.end() + 50)
+                    context = text[start:end]
+
+                    entities.append(ExtractedEntity(
+                        entity_type=entity_type,
+                        name=match.group(1) if match.lastindex else match.group(0),
+                        context=context,
+                        confidence=0.8,
+                    ))
+
+        # Deduplicate by name within each type
+        seen = set()
+        unique_entities = []
+        for entity in entities:
+            key = (entity.entity_type, entity.name)
+            if key not in seen:
+                seen.add(key)
+                unique_entities.append(entity)
+
+        return unique_entities
+
+    def extract_relationships(self, text: str) -> list[dict]:
+        """
+        Extract relationships between entities.
+        Returns list of {"source": str, "target": str, "relationship": str}
+        """
+        relationships = []
+
+        # Pattern: "X reports to Y" or "X manages Y"
+        report_patterns = [
+            (r"([A-Z][a-z]+\s+[A-Z][a-z]+)\s+reports?\s+to\s+([A-Z][a-z]+\s+[A-Z][a-z]+)", "reports_to"),
+            (r"([A-Z][a-z]+\s+[A-Z][a-z]+)\s+manages?\s+([A-Z][a-z]+\s+[A-Z][a-z]+)", "manages"),
+            (r"([A-Z][a-z]+\s+[A-Z][a-z]+)\s+owns?\s+([A-Z][a-z]+)", "owns"),
+            (r"([A-Z][a-z]+\s+[A-Z][a-z]+)\s+is\s+responsible\s+for\s+([A-Z][a-z]+)", "responsible_for"),
+        ]
+
+        for pattern, rel_type in report_patterns:
+            for match in re.finditer(pattern, text, re.IGNORECASE):
+                relationships.append({
+                    "source": match.group(1),
+                    "target": match.group(2),
+                    "relationship": rel_type,
+                })
+
+        return relationships

civilization/interview/__init__.py

@@ -0,0 +1,19 @@
+"""
+Interview module for Civilization Knowledge Layer.
+Handles knowledge elicitation through interactive interviews.
+"""
+from .interviewer import Interviewer, InterviewSession, InterviewState
+from .gap_detector import GapDetector, KnowledgeGap, GapSeverity
+from .questioner import Questioner, Question, QuestionType
+
+__all__ = [
+    "Interviewer",
+    "InterviewSession",
+    "InterviewState",
+    "GapDetector",
+    "KnowledgeGap",
+    "GapSeverity",
+    "Questioner",
+    "Question",
+    "QuestionType",
+]

civilization/interview/gap_detector.py

@@ -0,0 +1,380 @@
+"""
+Gap detector for identifying missing or incomplete knowledge in the Civilization layer.
+Analyzes existing nodes to find coverage gaps, outdated content, and missing relationships.
+"""
+from __future__ import annotations
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from enum import Enum
+from typing import Any
+from uuid import UUID
+
+from ..schema import (
+    NodeType, NodeStatus, CivilizationNode,
+    SOP, Role, Person, Checklist, Runbook, Workflow, OrgChart,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class GapSeverity(str, Enum):
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+
+class GapType(str, Enum):
+    MISSING_NODE = "missing_node"
+    INCOMPLETE_NODE = "incomplete_node"
+    ORPHAN_NODE = "orphan_node"
+    MISSING_RELATIONSHIP = "missing_relationship"
+    STALE_CONTENT = "stale_content"
+    MISSING_OWNER = "missing_owner"
+    COVERAGE_GAP = "coverage_gap"
+    INCONSISTENCY = "inconsistency"
+
+
+@dataclass
+class KnowledgeGap:
+    """Represents a detected gap in organizational knowledge."""
+    id: UUID
+    gap_type: GapType
+    severity: GapSeverity
+    title: str
+    description: str
+    affected_node_id: UUID | None = None
+    affected_node_type: NodeType | None = None
+    suggested_action: str | None = None
+    detected_at: datetime = field(default_factory=datetime.utcnow)
+    resolved_at: datetime | None = None
+    metadata: dict = field(default_factory=dict)
+
+    @property
+    def is_resolved(self) -> bool:
+        return self.resolved_at is not None
+
+
+@dataclass
+class GapReport:
+    """Summary report of knowledge gaps."""
+    org_id: str
+    generated_at: datetime = field(default_factory=datetime.utcnow)
+    gaps: list[KnowledgeGap] = field(default_factory=list)
+    coverage_score: float = 0.0  # 0-100
+    health_score: float = 0.0  # 0-100
+    recommendations: list[str] = field(default_factory=list)
+
+    @property
+    def critical_count(self) -> int:
+        return sum(1 for g in self.gaps if g.severity == GapSeverity.CRITICAL)
+
+    @property
+    def high_count(self) -> int:
+        return sum(1 for g in self.gaps if g.severity == GapSeverity.HIGH)
+
+    def by_type(self, gap_type: GapType) -> list[KnowledgeGap]:
+        return [g for g in self.gaps if g.gap_type == gap_type]
+
+
+class GapDetector:
+    """
+    Detects gaps and inconsistencies in organizational knowledge.
+    Analyzes nodes, relationships, and coverage to identify areas needing attention.
+    """
+
+    # Thresholds for gap detection
+    STALE_DAYS_WARNING = 90
+    STALE_DAYS_CRITICAL = 180
+    MIN_SOP_STEPS = 2
+    MIN_CHECKLIST_ITEMS = 2
+    MIN_ROLE_RESPONSIBILITIES = 3
+
+    def __init__(self, org_id: str, store=None):
+        self.org_id = org_id
+        self.store = store
+
+    async def analyze(self, nodes: list[CivilizationNode]) -> GapReport:
+        """
+        Perform comprehensive gap analysis on a set of nodes.
+
+        Args:
+            nodes: List of CivilizationNode instances to analyze
+
+        Returns:
+            GapReport with all detected gaps and recommendations
+        """
+        from uuid import uuid4
+
+        report = GapReport(org_id=self.org_id)
+        gaps: list[KnowledgeGap] = []
+
+        # Group nodes by type
+        nodes_by_type: dict[NodeType, list[CivilizationNode]] = {}
+        for node in nodes:
+            nodes_by_type.setdefault(node.node_type, []).append(node)
+
+        # Run all detection checks
+        gaps.extend(self._check_incomplete_nodes(nodes))
+        gaps.extend(self._check_orphan_nodes(nodes, nodes_by_type))
+        gaps.extend(self._check_stale_content(nodes))
+        gaps.extend(self._check_missing_owners(nodes))
+        gaps.extend(self._check_coverage_gaps(nodes_by_type))
+        gaps.extend(self._check_relationship_gaps(nodes, nodes_by_type))
+
+        report.gaps = gaps
+        report.coverage_score = self._calculate_coverage_score(nodes_by_type)
+        report.health_score = self._calculate_health_score(gaps, len(nodes))
+        report.recommendations = self._generate_recommendations(gaps, nodes_by_type)
+
+        return report
+
+    def _check_incomplete_nodes(self, nodes: list[CivilizationNode]) -> list[KnowledgeGap]:
+        """Check for nodes with missing required fields."""
+        from uuid import uuid4
+        gaps = []
+
+        for node in nodes:
+            issues = []
+
+            if isinstance(node, SOP):
+                if len(node.steps) < self.MIN_SOP_STEPS:
+                    issues.append("SOP has too few steps")
+                if not node.purpose:
+                    issues.append("SOP missing purpose")
+                if not node.roles_involved:
+                    issues.append("SOP missing roles")
+
+            elif isinstance(node, Role):
+                if len(node.responsibilities) < self.MIN_ROLE_RESPONSIBILITIES:
+                    issues.append("Role has too few responsibilities")
+                if not node.accountabilities:
+                    issues.append("Role missing accountabilities")
+
+            elif isinstance(node, Checklist):
+                if len(node.items) < self.MIN_CHECKLIST_ITEMS:
+                    issues.append("Checklist has too few items")
+
+            elif isinstance(node, Runbook):
+                if not node.steps:
+                    issues.append("Runbook has no steps")
+                if not node.escalation_contacts:
+                    issues.append("Runbook missing escalation contacts")
+
+            if issues:
+                gaps.append(KnowledgeGap(
+                    id=uuid4(),
+                    gap_type=GapType.INCOMPLETE_NODE,
+                    severity=GapSeverity.MEDIUM,
+                    title=f"Incomplete {node.node_type.value}: {node.title}",
+                    description="; ".join(issues),
+                    affected_node_id=node.id,
+                    affected_node_type=node.node_type,
+                    suggested_action=f"Review and complete the {node.node_type.value}",
+                ))
+
+        return gaps
+
+    def _check_orphan_nodes(
+        self,
+        nodes: list[CivilizationNode],
+        nodes_by_type: dict[NodeType, list[CivilizationNode]],
+    ) -> list[KnowledgeGap]:
+        """Check for nodes with no relationships."""
+        from uuid import uuid4
+        gaps = []
+
+        # SOPs without related SOPs or roles
+        for sop in nodes_by_type.get(NodeType.SOP, []):
+            if isinstance(sop, SOP):
+                if not sop.related_sops and not sop.roles_involved:
+                    gaps.append(KnowledgeGap(
+                        id=uuid4(),
+                        gap_type=GapType.ORPHAN_NODE,
+                        severity=GapSeverity.LOW,
+                        title=f"Isolated SOP: {sop.title}",
+                        description="SOP has no related SOPs or roles defined",
+                        affected_node_id=sop.id,
+                        affected_node_type=NodeType.SOP,
+                        suggested_action="Link this SOP to related procedures and roles",
+                    ))
+
+        return gaps
+
+    def _check_stale_content(self, nodes: list[CivilizationNode]) -> list[KnowledgeGap]:
+        """Check for outdated content."""
+        from uuid import uuid4
+        gaps = []
+        now = datetime.utcnow()
+
+        for node in nodes:
+            age_days = (now - node.updated_at).days
+
+            if age_days > self.STALE_DAYS_CRITICAL:
+                gaps.append(KnowledgeGap(
+                    id=uuid4(),
+                    gap_type=GapType.STALE_CONTENT,
+                    severity=GapSeverity.HIGH,
+                    title=f"Critically stale: {node.title}",
+                    description=f"Not updated in {age_days} days",
+                    affected_node_id=node.id,
+                    affected_node_type=node.node_type,
+                    suggested_action="Review and update this content urgently",
+                    metadata={"days_since_update": age_days},
+                ))
+            elif age_days > self.STALE_DAYS_WARNING:
+                gaps.append(KnowledgeGap(
+                    id=uuid4(),
+                    gap_type=GapType.STALE_CONTENT,
+                    severity=GapSeverity.MEDIUM,
+                    title=f"Stale content: {node.title}",
+                    description=f"Not updated in {age_days} days",
+                    affected_node_id=node.id,
+                    affected_node_type=node.node_type,
+                    suggested_action="Consider reviewing this content",
+                    metadata={"days_since_update": age_days},
+                ))
+
+        return gaps
+
+    def _check_missing_owners(self, nodes: list[CivilizationNode]) -> list[KnowledgeGap]:
+        """Check for nodes without owners."""
+        from uuid import uuid4
+        gaps = []
+
+        for node in nodes:
+            if not node.owner_id:
+                # SOPs and Runbooks without owners are higher severity
+                severity = GapSeverity.HIGH if node.node_type in {NodeType.SOP, NodeType.RUNBOOK} else GapSeverity.LOW
+                gaps.append(KnowledgeGap(
+                    id=uuid4(),
+                    gap_type=GapType.MISSING_OWNER,
+                    severity=severity,
+                    title=f"No owner: {node.title}",
+                    description=f"{node.node_type.value} has no assigned owner",
+                    affected_node_id=node.id,
+                    affected_node_type=node.node_type,
+                    suggested_action="Assign an owner to this content",
+                ))
+
+        return gaps
+
+    def _check_coverage_gaps(
+        self,
+        nodes_by_type: dict[NodeType, list[CivilizationNode]],
+    ) -> list[KnowledgeGap]:
+        """Check for missing essential node types."""
+        from uuid import uuid4
+        gaps = []
+
+        # Essential node types for a well-documented org
+        essential_types = {
+            NodeType.ORG_CHART: "Organization structure",
+            NodeType.ROLE: "Role definitions",
+            NodeType.SOP: "Standard operating procedures",
+        }
+
+        for node_type, description in essential_types.items():
+            if node_type not in nodes_by_type or len(nodes_by_type[node_type]) == 0:
+                gaps.append(KnowledgeGap(
+                    id=uuid4(),
+                    gap_type=GapType.COVERAGE_GAP,
+                    severity=GapSeverity.HIGH,
+                    title=f"Missing {description}",
+                    description=f"No {node_type.value} nodes found in knowledge base",
+                    suggested_action=f"Create {node_type.value} documentation",
+                ))
+
+        return gaps
+
+    def _check_relationship_gaps(
+        self,
+        nodes: list[CivilizationNode],
+        nodes_by_type: dict[NodeType, list[CivilizationNode]],
+    ) -> list[KnowledgeGap]:
+        """Check for missing relationships between nodes."""
+        from uuid import uuid4
+        gaps = []
+
+        # Check if roles mentioned in SOPs exist
+        role_titles = {
+            r.role_title.lower() for r in nodes_by_type.get(NodeType.ROLE, [])
+            if isinstance(r, Role)
+        }
+
+        for sop in nodes_by_type.get(NodeType.SOP, []):
+            if isinstance(sop, SOP):
+                for role_name in sop.roles_involved:
+                    if role_name.lower() not in role_titles:
+                        gaps.append(KnowledgeGap(
+                            id=uuid4(),
+                            gap_type=GapType.MISSING_RELATIONSHIP,
+                            severity=GapSeverity.MEDIUM,
+                            title=f"Undefined role: {role_name}",
+                            description=f"Role '{role_name}' referenced in SOP '{sop.title}' but not defined",
+                            affected_node_id=sop.id,
+                            affected_node_type=NodeType.SOP,
+                            suggested_action=f"Create role definition for '{role_name}'",
+                        ))
+
+        return gaps
+
+    def _calculate_coverage_score(
+        self,
+        nodes_by_type: dict[NodeType, list[CivilizationNode]],
+    ) -> float:
+        """Calculate coverage score (0-100) based on node type diversity."""
+        essential_types = {NodeType.ORG_CHART, NodeType.ROLE, NodeType.SOP, NodeType.CHECKLIST}
+        covered = sum(1 for t in essential_types if t in nodes_by_type and nodes_by_type[t])
+        return (covered / len(essential_types)) * 100
+
+    def _calculate_health_score(self, gaps: list[KnowledgeGap], total_nodes: int) -> float:
+        """Calculate health score (0-100) based on gap severity."""
+        if total_nodes == 0:
+            return 0.0
+
+        # Weight by severity
+        severity_weights = {
+            GapSeverity.CRITICAL: 4,
+            GapSeverity.HIGH: 2,
+            GapSeverity.MEDIUM: 1,
+            GapSeverity.LOW: 0.5,
+        }
+
+        total_weight = sum(severity_weights[g.severity] for g in gaps)
+        # Higher weight = lower score
+        max_weight = total_nodes * 2  # Assume average 2 issues per node is worst case
+        score = max(0, 100 - (total_weight / max_weight) * 100)
+        return min(100, score)
+
+    def _generate_recommendations(
+        self,
+        gaps: list[KnowledgeGap],
+        nodes_by_type: dict[NodeType, list[CivilizationNode]],
+    ) -> list[str]:
+        """Generate prioritized recommendations."""
+        recommendations = []
+
+        # Critical issues first
+        critical = [g for g in gaps if g.severity == GapSeverity.CRITICAL]
+        if critical:
+            recommendations.append(f"Address {len(critical)} critical gaps immediately")
+
+        # Coverage gaps
+        coverage_gaps = [g for g in gaps if g.gap_type == GapType.COVERAGE_GAP]
+        if coverage_gaps:
+            types = ", ".join(g.title.replace("Missing ", "") for g in coverage_gaps)
+            recommendations.append(f"Document missing areas: {types}")
+
+        # Stale content
+        stale = [g for g in gaps if g.gap_type == GapType.STALE_CONTENT]
+        if stale:
+            recommendations.append(f"Review {len(stale)} pieces of stale content")
+
+        # Missing owners
+        unowned = [g for g in gaps if g.gap_type == GapType.MISSING_OWNER]
+        if unowned:
+            recommendations.append(f"Assign owners to {len(unowned)} unowned items")
+
+        return recommendations