hyperagents 1.2.3rc5.post3__py3-none-any.whl

civilization/retrieval/relevance_ranker.py

@@ -0,0 +1,287 @@
+"""
+Relevance ranking for retrieved civilization nodes.
+Provides multi-factor ranking beyond simple vector similarity.
+"""
+from __future__ import annotations
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Callable
+from uuid import UUID
+
+from ..schema import CivilizationNode, NodeType, NodeStatus
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class RankingFactors:
+    """Individual factors contributing to ranking score."""
+    semantic_score: float = 0.0  # Vector similarity
+    recency_score: float = 0.0  # How recently updated
+    status_score: float = 0.0  # Node status (active > deprecated)
+    type_match_score: float = 0.0  # How well type matches query intent
+    popularity_score: float = 0.0  # Usage/reference frequency
+    owner_score: float = 0.0  # Whether owner is relevant
+    tag_match_score: float = 0.0  # Tag overlap with query
+
+
+@dataclass
+class RankingResult:
+    """Result of ranking a single node."""
+    node: CivilizationNode
+    final_score: float
+    factors: RankingFactors
+    rank: int = 0
+    explanation: str = ""
+
+
+@dataclass
+class RankingConfig:
+    """Configuration for relevance ranking."""
+    # Factor weights (should sum to ~1.0)
+    semantic_weight: float = 0.4
+    recency_weight: float = 0.15
+    status_weight: float = 0.1
+    type_match_weight: float = 0.15
+    popularity_weight: float = 0.1
+    tag_match_weight: float = 0.1
+
+    # Recency decay
+    recency_half_life_days: int = 30
+
+    # Type preferences for different query intents
+    type_preferences: dict[str, list[NodeType]] = field(default_factory=dict)
+
+
+class RelevanceRanker:
+    """
+    Multi-factor relevance ranker for civilization nodes.
+    Combines semantic similarity with recency, status, and other factors.
+    """
+
+    DEFAULT_TYPE_PREFERENCES = {
+        "how to": [NodeType.SOP, NodeType.CHECKLIST, NodeType.RUNBOOK],
+        "procedure": [NodeType.SOP, NodeType.CHECKLIST],
+        "incident": [NodeType.RUNBOOK, NodeType.SOP],
+        "who is": [NodeType.PERSON, NodeType.ROLE, NodeType.JOB_DESCRIPTION],
+        "role": [NodeType.ROLE, NodeType.JOB_DESCRIPTION],
+        "policy": [NodeType.POLICY],
+        "workflow": [NodeType.WORKFLOW, NodeType.SOP],
+        "client": [NodeType.CLIENT_PROFILE],
+        "organization": [NodeType.ORG_CHART, NodeType.ROLE],
+    }
+
+    STATUS_SCORES = {
+        NodeStatus.ACTIVE: 1.0,
+        NodeStatus.UNDER_REVIEW: 0.8,
+        NodeStatus.DRAFT: 0.6,
+        NodeStatus.DEPRECATED: 0.3,
+        NodeStatus.ARCHIVED: 0.1,
+    }
+
+    def __init__(self, config: RankingConfig | None = None):
+        self.config = config or RankingConfig()
+        self.config.type_preferences = {
+            **self.DEFAULT_TYPE_PREFERENCES,
+            **self.config.type_preferences,
+        }
+        self._popularity_cache: dict[UUID, float] = {}
+
+    def rank(
+        self,
+        nodes: list[tuple[CivilizationNode, float]],  # (node, semantic_score)
+        query: str,
+        query_tags: list[str] | None = None,
+        preferred_types: list[NodeType] | None = None,
+    ) -> list[RankingResult]:
+        """
+        Rank nodes by multi-factor relevance.
+
+        Args:
+            nodes: List of (node, semantic_score) tuples
+            query: The original query text
+            query_tags: Optional tags to match against
+            preferred_types: Optional type preferences
+
+        Returns:
+            Sorted list of RankingResults (highest first)
+        """
+        results = []
+
+        # Detect query intent
+        detected_types = preferred_types or self._detect_type_intent(query)
+
+        for node, semantic_score in nodes:
+            factors = self._compute_factors(
+                node=node,
+                semantic_score=semantic_score,
+                query=query,
+                query_tags=query_tags or [],
+                preferred_types=detected_types,
+            )
+
+            final_score = self._compute_final_score(factors)
+
+            results.append(RankingResult(
+                node=node,
+                final_score=final_score,
+                factors=factors,
+                explanation=self._generate_explanation(factors),
+            ))
+
+        # Sort by final score
+        results.sort(key=lambda r: r.final_score, reverse=True)
+
+        # Assign ranks
+        for i, result in enumerate(results):
+            result.rank = i + 1
+
+        return results
+
+    def _compute_factors(
+        self,
+        node: CivilizationNode,
+        semantic_score: float,
+        query: str,
+        query_tags: list[str],
+        preferred_types: list[NodeType],
+    ) -> RankingFactors:
+        """Compute all ranking factors for a node."""
+        return RankingFactors(
+            semantic_score=semantic_score,
+            recency_score=self._compute_recency_score(node),
+            status_score=self._compute_status_score(node),
+            type_match_score=self._compute_type_match_score(node, preferred_types),
+            popularity_score=self._compute_popularity_score(node),
+            tag_match_score=self._compute_tag_match_score(node, query_tags),
+        )
+
+    def _compute_final_score(self, factors: RankingFactors) -> float:
+        """Compute weighted final score from factors."""
+        cfg = self.config
+        return (
+            factors.semantic_score * cfg.semantic_weight +
+            factors.recency_score * cfg.recency_weight +
+            factors.status_score * cfg.status_weight +
+            factors.type_match_score * cfg.type_match_weight +
+            factors.popularity_score * cfg.popularity_weight +
+            factors.tag_match_score * cfg.tag_match_weight
+        )
+
+    def _compute_recency_score(self, node: CivilizationNode) -> float:
+        """Compute recency score with exponential decay."""
+        import math
+        days_old = (datetime.utcnow() - node.updated_at).days
+        half_life = self.config.recency_half_life_days
+        return math.exp(-0.693 * days_old / half_life)
+
+    def _compute_status_score(self, node: CivilizationNode) -> float:
+        """Get score based on node status."""
+        return self.STATUS_SCORES.get(node.status, 0.5)
+
+    def _compute_type_match_score(
+        self,
+        node: CivilizationNode,
+        preferred_types: list[NodeType],
+    ) -> float:
+        """Score based on how well type matches preferences."""
+        if not preferred_types:
+            return 0.5  # Neutral
+
+        if node.node_type in preferred_types:
+            # Higher score for earlier in preference list
+            index = preferred_types.index(node.node_type)
+            return 1.0 - (index * 0.2)  # 1.0, 0.8, 0.6, ...
+        return 0.3
+
+    def _compute_popularity_score(self, node: CivilizationNode) -> float:
+        """Get popularity score (cached or computed)."""
+        if node.id in self._popularity_cache:
+            return self._popularity_cache[node.id]
+        # Default to neutral - could be enhanced with usage tracking
+        return 0.5
+
+    def _compute_tag_match_score(
+        self,
+        node: CivilizationNode,
+        query_tags: list[str],
+    ) -> float:
+        """Score based on tag overlap."""
+        if not query_tags or not node.tags:
+            return 0.5  # Neutral
+
+        node_tags_lower = set(t.lower() for t in node.tags)
+        query_tags_lower = set(t.lower() for t in query_tags)
+
+        overlap = len(node_tags_lower & query_tags_lower)
+        max_possible = min(len(node_tags_lower), len(query_tags_lower))
+
+        if max_possible == 0:
+            return 0.5
+        return overlap / max_possible
+
+    def _detect_type_intent(self, query: str) -> list[NodeType]:
+        """Detect preferred node types from query text."""
+        query_lower = query.lower()
+
+        for pattern, types in self.config.type_preferences.items():
+            if pattern in query_lower:
+                return types
+
+        return []  # No specific preference
+
+    def _generate_explanation(self, factors: RankingFactors) -> str:
+        """Generate human-readable explanation of ranking."""
+        parts = []
+
+        if factors.semantic_score > 0.8:
+            parts.append("high semantic match")
+        elif factors.semantic_score > 0.6:
+            parts.append("moderate semantic match")
+
+        if factors.recency_score > 0.8:
+            parts.append("recently updated")
+        elif factors.recency_score < 0.3:
+            parts.append("may be outdated")
+
+        if factors.type_match_score > 0.8:
+            parts.append("matches expected type")
+
+        if factors.status_score < 0.5:
+            parts.append("non-active status")
+
+        if factors.tag_match_score > 0.7:
+            parts.append("matching tags")
+
+        return ", ".join(parts) if parts else "standard relevance"
+
+    def update_popularity(self, node_id: UUID, score: float) -> None:
+        """Update popularity score for a node."""
+        self._popularity_cache[node_id] = min(1.0, max(0.0, score))
+
+    def boost_node(
+        self,
+        results: list[RankingResult],
+        node_id: UUID,
+        boost_factor: float = 1.5,
+    ) -> list[RankingResult]:
+        """Boost a specific node's ranking."""
+        for result in results:
+            if result.node.id == node_id:
+                result.final_score *= boost_factor
+
+        # Re-sort and re-rank
+        results.sort(key=lambda r: r.final_score, reverse=True)
+        for i, result in enumerate(results):
+            result.rank = i + 1
+
+        return results
+
+    def filter_by_threshold(
+        self,
+        results: list[RankingResult],
+        min_score: float = 0.3,
+    ) -> list[RankingResult]:
+        """Filter results below a score threshold."""
+        return [r for r in results if r.final_score >= min_score]

civilization/schema.py

@@ -0,0 +1,299 @@
+"""
+Pydantic v2 models for all Civilization Knowledge Layer node types.
+Every node has: id, org_id, version, created_at, updated_at, embedding.
+"""
+from __future__ import annotations
+from enum import Enum
+from pydantic import BaseModel, Field, field_validator
+from uuid import UUID, uuid4
+from datetime import datetime
+from typing import Any
+
+
+class NodeType(str, Enum):
+    SOP = "sop"
+    JOB_DESCRIPTION = "job_description"
+    CHECKLIST = "checklist"
+    RUNBOOK = "runbook"
+    WORKFLOW = "workflow"
+    ORG_CHART = "org_chart"
+    ROLE = "role"
+    PERSON = "person"
+    CLIENT_PROFILE = "client_profile"
+    PERSONAL_ROUTINE = "personal_routine"
+    POLICY = "policy"
+    KNOWLEDGE_ARTICLE = "knowledge_article"
+
+
+class NodeStatus(str, Enum):
+    DRAFT = "draft"
+    ACTIVE = "active"
+    UNDER_REVIEW = "under_review"
+    DEPRECATED = "deprecated"
+    ARCHIVED = "archived"
+
+
+class CivilizationNode(BaseModel):
+    id: UUID = Field(default_factory=uuid4)
+    org_id: str
+    node_type: NodeType
+    title: str
+    status: NodeStatus = NodeStatus.ACTIVE
+    version: str = "1.0.0"
+    owner_id: str | None = None
+    tags: list[str] = []
+    source: str | None = None
+    source_url: str | None = None
+    embedding: list[float] | None = None
+    metadata: dict[str, Any] = {}
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    updated_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class SOPStep(BaseModel):
+    step_number: int
+    title: str
+    description: str
+    responsible_role: str | None = None
+    tools_required: list[str] = []
+    estimated_duration_minutes: int | None = None
+    decision_points: list[str] = []
+    on_failure: str | None = None
+    substeps: list["SOPStep"] = []
+
+
+class SOP(CivilizationNode):
+    node_type: NodeType = NodeType.SOP
+    purpose: str
+    scope: str
+    triggers: list[str] = []
+    steps: list[SOPStep]
+    roles_involved: list[str] = []
+    tools_required: list[str] = []
+    estimated_total_minutes: int | None = None
+    success_criteria: list[str] = []
+    related_sops: list[UUID] = []
+    kpis: list[str] = []
+
+
+class JobDescription(CivilizationNode):
+    node_type: NodeType = NodeType.JOB_DESCRIPTION
+    role_title: str
+    department: str
+    reports_to: str | None = None
+    direct_reports: list[str] = []
+    level: str | None = None
+    employment_type: str = "full_time"
+    summary: str
+    responsibilities: list[str]
+    required_skills: list[str]
+    preferred_skills: list[str] = []
+    required_experience_years: int | None = None
+    tools_used: list[str] = []
+    kpis: list[str] = []
+    okrs: list[str] = []
+    success_metrics: list[str] = []
+    compensation_band: str | None = None
+    related_roles: list[UUID] = []
+
+
+class Role(CivilizationNode):
+    node_type: NodeType = NodeType.ROLE
+    role_title: str
+    department: str
+    accountabilities: list[str]
+    responsibilities: list[str]
+    decision_authority: list[str]
+    escalation_path: list[str]
+    interfaces: list[str]
+    tools: list[str] = []
+    sops_owned: list[UUID] = []
+    checklists_owned: list[UUID] = []
+
+
+class Person(CivilizationNode):
+    node_type: NodeType = NodeType.PERSON
+    name: str
+    email: str | None = None
+    role_id: UUID | None = None
+    manager_id: UUID | None = None
+    direct_report_ids: list[UUID] = []
+    department: str | None = None
+    location: str | None = None
+    timezone: str | None = None
+    communication_preferences: dict = {}
+    expertise_areas: list[str] = []
+    current_projects: list[str] = []
+    personal_routine_id: UUID | None = None
+
+
+class ChecklistItem(BaseModel):
+    item_number: int
+    description: str
+    required: bool = True
+    verification_method: str | None = None
+    responsible_role: str | None = None
+    time_estimate_minutes: int | None = None
+    notes: str | None = None
+
+
+class Checklist(CivilizationNode):
+    node_type: NodeType = NodeType.CHECKLIST
+    purpose: str
+    frequency: str | None = None
+    trigger: str | None = None
+    items: list[ChecklistItem]
+    roles_involved: list[str] = []
+    estimated_total_minutes: int | None = None
+    related_sop_id: UUID | None = None
+
+
+class RunbookStep(BaseModel):
+    step_number: int
+    action: str
+    command: str | None = None
+    expected_output: str | None = None
+    on_success: str | None = None
+    on_failure: str | None = None
+    rollback: str | None = None
+
+
+class Runbook(CivilizationNode):
+    node_type: NodeType = NodeType.RUNBOOK
+    system: str
+    scenario: str
+    severity: str | None = None
+    prerequisites: list[str] = []
+    steps: list[RunbookStep]
+    rollback_procedure: list[str] = []
+    escalation_contacts: list[str] = []
+    post_incident_actions: list[str] = []
+
+
+class WorkflowNode(BaseModel):
+    node_id: str
+    label: str
+    node_type: str  # "start"|"end"|"task"|"decision"|"approval"
+    responsible_role: str | None = None
+    tool: str | None = None
+    description: str | None = None
+
+
+class WorkflowEdge(BaseModel):
+    from_node: str
+    to_node: str
+    condition: str | None = None
+
+
+class Workflow(CivilizationNode):
+    node_type: NodeType = NodeType.WORKFLOW
+    description: str
+    department: str | None = None
+    nodes: list[WorkflowNode]
+    edges: list[WorkflowEdge]
+    triggers: list[str] = []
+    sla_hours: int | None = None
+    related_sop_id: UUID | None = None
+
+
+class OrgUnit(BaseModel):
+    unit_id: str
+    name: str
+    unit_type: str
+    head_person_id: UUID | None = None
+    parent_unit_id: str | None = None
+    member_ids: list[UUID] = []
+    mission: str | None = None
+    budget_owner: bool = False
+
+
+class OrgChart(CivilizationNode):
+    node_type: NodeType = NodeType.ORG_CHART
+    organization_name: str
+    units: list[OrgUnit]
+    total_headcount: int | None = None
+    last_updated: datetime = Field(default_factory=datetime.utcnow)
+
+
+class ClientProfile(CivilizationNode):
+    node_type: NodeType = NodeType.CLIENT_PROFILE
+    client_name: str
+    industry: str | None = None
+    tier: str | None = None
+    relationship_owner_id: UUID | None = None
+    key_contacts: list[dict] = []
+    contract_value_usd: float | None = None
+    contract_start: datetime | None = None
+    contract_end: datetime | None = None
+    health_score: float | None = None
+    goals: list[str] = []
+    pain_points: list[str] = []
+    preferences: dict = {}
+    active_projects: list[str] = []
+    communication_cadence: str | None = None
+    notes: str | None = None
+    related_sops: list[UUID] = []
+
+    @field_validator("health_score")
+    @classmethod
+    def validate_health_score(cls, v):
+        if v is not None and not (0.0 <= v <= 1.0):
+            raise ValueError("health_score must be between 0.0 and 1.0")
+        return v
+
+
+class RoutineBlock(BaseModel):
+    time: str  # "06:00" 24h format
+    duration_minutes: int
+    activity: str
+    category: str
+    priority: str = "medium"
+    tools: list[str] = []
+    notes: str | None = None
+
+    @field_validator("time")
+    @classmethod
+    def validate_time_format(cls, v):
+        import re
+        if not re.match(r"^\d{2}:\d{2}$", v):
+            raise ValueError("time must be in HH:MM format")
+        return v
+
+
+class PersonalRoutine(CivilizationNode):
+    node_type: NodeType = NodeType.PERSONAL_ROUTINE
+    person_id: UUID
+    routine_type: str
+    day_of_week: list[str] = []
+    blocks: list[RoutineBlock]
+    goals: list[str] = []
+    non_negotiables: list[str] = []
+    flexibility_rules: list[str] = []
+
+
+class Policy(CivilizationNode):
+    node_type: NodeType = NodeType.POLICY
+    policy_name: str
+    category: str
+    applies_to: list[str] = []
+    effective_date: datetime | None = None
+    review_date: datetime | None = None
+    summary: str
+    full_text: str
+    exceptions: list[str] = []
+    related_policies: list[UUID] = []
+    compliance_requirements: list[str] = []
+
+
+class KnowledgeArticle(CivilizationNode):
+    node_type: NodeType = NodeType.KNOWLEDGE_ARTICLE
+    topic: str
+    category: str
+    summary: str
+    content: str
+    author_id: UUID | None = None
+    reviewers: list[UUID] = []
+    related_articles: list[UUID] = []
+    related_sops: list[UUID] = []
+    faq: list[dict] = []
+    last_reviewed: datetime | None = None