apsimo 1.3.0__py3-none-any.whl

apsimo/intelligence/graph/schema.py

@@ -0,0 +1,337 @@
+"""Colony Graph Schema — Node and edge type definitions.
+
+Every node and relationship type used in the Colony Neo4j graph is modelled
+here as a Pydantic ``BaseModel`` (for validation / serialisation) together
+with string‐constant edge types.
+"""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+# ──────────────────────────────────────────────────────────────────────
+# Trust tiers (mirrors RelationshipScorer thresholds)
+# ──────────────────────────────────────────────────────────────────────
+
+class TrustTier(str, Enum):
+    INNER_CIRCLE = "inner_circle"
+    TRUSTED = "trusted"
+    REGULAR = "regular"
+    PERIPHERAL = "peripheral"
+    SILENCED = "silenced"
+
+
+# ──────────────────────────────────────────────────────────────────────
+# Node types
+# ──────────────────────────────────────────────────────────────────────
+
+class Owner(BaseModel):
+    """The Colony owner (singleton node — one per graph)."""
+
+    id: str = Field(..., description="Unique owner identifier")
+    name: str
+    timezone: str = "UTC"
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Agent(BaseModel):
+    """An autonomous entity (Colony itself, or other agents/bots)."""
+
+    id: str = Field(..., description="Unique agent identifier")
+    name: str
+    version: Optional[str] = None
+    status: str = "active"
+    capabilities: List[str] = Field(default_factory=list)
+    health_score: float = Field(default=1.0, ge=0.0, le=1.0)
+    last_tick_at: Optional[datetime] = None
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Subsystem(BaseModel):
+    """A Colony component that can be monitored and restarted."""
+
+    id: str = Field(..., description="Unique subsystem identifier")
+    name: str
+    status: str = "active"
+    latency_ms: Optional[float] = None
+    error_rate: Optional[float] = None
+    last_check_at: Optional[datetime] = None
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Capability(BaseModel):
+    """A tool or skill that Colony has or needs."""
+
+    id: str = Field(..., description="Unique capability identifier")
+    name: str
+    description: Optional[str] = None
+    available: bool = True
+    status: str = "available"  # available | deprecated | missing | planned
+    failure_count: int = 0
+    last_failure_at: Optional[datetime] = None
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Project(BaseModel):
+    """An active work item."""
+
+    id: str = Field(..., description="Unique project identifier")
+    name: str
+    description: Optional[str] = None
+    status: str = "active"
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Goal(BaseModel):
+    """A goal or objective."""
+
+    id: str = Field(..., description="Unique goal identifier")
+    title: str
+    description: Optional[str] = None
+    status: str = "active"
+    priority: float = Field(default=0.5, ge=0.0, le=1.0)
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Task(BaseModel):
+    """An actionable work unit."""
+
+    id: str = Field(..., description="Unique task identifier")
+    title: str
+    description: Optional[str] = None
+    status: str = "pending"
+    priority: float = Field(default=0.5, ge=0.0, le=1.0)
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Pattern(BaseModel):
+    """A recurring behavioral pattern."""
+
+    id: str = Field(..., description="Unique pattern identifier")
+    name: str
+    description: Optional[str] = None
+    confidence: float = Field(default=0.5, ge=0.0, le=1.0)
+    occurrences: int = 0
+    pattern_type: str = "behavioral"  # behavioral | workflow | preference | correction
+    trigger: Optional[str] = None
+    action: Optional[str] = None
+    recurrence_count: int = 0
+    last_triggered_at: Optional[datetime] = None
+    is_active: bool = True
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Concept(BaseModel):
+    """A knowledge domain or concept Colony has encountered."""
+
+    id: str = Field(..., description="Unique concept identifier")
+    name: str
+    domain: str = "general"  # e.g., "technology", "science", "person"
+    description: Optional[str] = None
+    confidence_score: float = 0.0  # 0 = unknown, 1 = expert
+    encounter_count: int = 0
+    last_researched_at: Optional[datetime] = None
+    last_encountered_at: Optional[datetime] = None
+    source: Optional[str] = None  # "web_search", "tool_failure", "owner_query"
+    status: str = "open"  # open | researching | learned | archived
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Preference(BaseModel):
+    """A learned preference or behavioral rule."""
+
+    id: str = Field(..., description="Unique preference identifier")
+    trigger: str
+    expected: str
+    source: str = "behavioral_correction"  # behavioral_correction, owner_config, inferred
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    updated_at: Optional[datetime] = None
+
+
+class InitiativeCategory(BaseModel):
+    """A dynamic self-initiative category registered at runtime."""
+
+    id: str = Field(..., description="Unique category identifier")
+    name: str
+    description: Optional[str] = None
+    trigger_query: Optional[str] = None
+    action_type: str = "auto_fix"  # auto_fix, propose, research, notify
+    executor_skill: str
+    priority_formula: Optional[str] = None
+    cooldown_minutes: int = 30
+    auto_execute: bool = True
+    requires_approval: bool = False
+    effectiveness_score: float = Field(default=0.5, ge=0.0, le=1.0)
+    total_triggered: int = 0
+    total_successful: int = 0
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    updated_at: Optional[datetime] = None
+
+
+class Person(BaseModel):
+    """A person in the owner's relationship network."""
+
+    id: str = Field(..., description="Unique person identifier (UUID)")
+    name: str
+    tier: TrustTier = TrustTier.PERIPHERAL
+    score: float = Field(default=0.0, ge=0.0, le=100.0)
+    last_interaction: Optional[datetime] = None
+    contact_info: Optional[Dict[str, str]] = None
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Memory(BaseModel):
+    """An episodic / semantic / procedural memory."""
+
+    id: str = Field(..., description="UUID assigned by Neo4j randomUUID()")
+    content: str
+    type: str = Field(..., description="e.g. episodic, semantic, procedural")
+    strength: float = Field(default=1.0, ge=0.0, le=1.0)
+    embedding: Optional[List[float]] = None
+    metadata: Optional[Dict[str, Any]] = None
+    sources: Optional[List[str]] = None
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    accessed_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Entity(BaseModel):
+    """A named entity extracted from memories (person, place, concept …)."""
+
+    name: str = Field(..., description="Canonical entity name (unique)")
+    entity_type: Optional[str] = None  # person, place, org, concept
+    first_seen: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Signal(BaseModel):
+    """A single behavioral signal emitted by a Person."""
+
+    id: str
+    signal_type: str  # message_length, sentiment, response_latency, …
+    raw_value: float
+    normalized_value: float
+    timestamp: datetime
+    source: str  # message, reaction, call
+    context: Optional[Dict[str, Any]] = None
+
+
+class Context(BaseModel):
+    """Temporal context attached to a Person (job change, vacation, …)."""
+
+    id: str
+    label: str
+    description: Optional[str] = None
+    start_date: date
+    end_date: Optional[date] = None
+    active: bool = True
+
+
+class ScoreEvent(BaseModel):
+    """Audit record for a relationship score change."""
+
+    id: str
+    score: float
+    tier: TrustTier
+    delta: float
+    reason: str
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+class Prediction(BaseModel):
+    """A forward‐looking behavioural prediction."""
+
+    id: str
+    prediction_type: str  # timing, need, action, trajectory
+    description: str
+    probability: float = Field(ge=0.0, le=1.0)
+    person_id: str
+    reasoning: List[str] = Field(default_factory=list)
+    expires_at: datetime
+    resolved: Optional[bool] = None
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+
+# ──────────────────────────────────────────────────────────────────────
+# Edge (relationship) types
+# ──────────────────────────────────────────────────────────────────────
+
+class EdgeType(str, Enum):
+    """All relationship types in the Colony graph."""
+
+    # Owner ↔ Person
+    KNOWS = "KNOWS"
+
+    # Memory → Entity
+    MENTIONS = "MENTIONS"
+
+    # Person → Signal
+    EXHIBITED = "EXHIBITED"
+
+    # Person → Context
+    HAS_CONTEXT = "HAS_CONTEXT"
+
+    # Person → ScoreEvent
+    SCORE_CHANGED = "SCORE_CHANGED"
+
+    # Person → Prediction
+    PREDICTED = "PREDICTED"
+
+    # Memory → Memory (causal / logical chains)
+    CAUSED_BY = "CAUSED_BY"
+    LED_TO = "LED_TO"
+    SUPPORTS = "SUPPORTS"
+    MERGED_INTO = "MERGED_INTO"
+
+    # Owner → Memory (ownership)
+    REMEMBERS = "REMEMBERS"
+
+    # Memory → Person (memory about a person)
+    ABOUT = "ABOUT"
+
+    # Agent → Person (agent manages relationship with person)
+    MANAGES = "MANAGES"
+
+    # Person → Project (person owns/works on project)
+    OWNS = "OWNS"
+
+    # Subsystem → Subsystem (component dependency)
+    DEPENDS_ON = "DEPENDS_ON"
+
+    # Agent → Capability (agent has tool)
+    HAS_CAPABILITY = "HAS_CAPABILITY"
+
+    # Agent → Capability (agent lacks tool)
+    NEEDS_CAPABILITY = "NEEDS_CAPABILITY"
+
+    # Agent → Initiative (agent created initiative)
+    GENERATED = "GENERATED"
+
+    # Initiative → Subsystem (initiative targets component)
+    TARGETS = "TARGETS"
+
+    # Task → Project (task belongs to project)
+    BELONGS_TO = "BELONGS_TO"
+
+    # Goal → Goal (goal blocks another)
+    BLOCKS = "BLOCKS"
+
+    # Person → Pattern (person exhibits pattern)
+    EXHIBITS = "EXHIBITS"
+
+    # Pattern → InitiativeCategory (pattern triggers category)
+    TRIGGERS = "TRIGGERS"
+
+
+# ──────────────────────────────────────────────────────────────────────
+# Convenience exports
+# ──────────────────────────────────────────────────────────────────────
+
+NODE_TYPES = (
+    Owner, Person, Memory, Entity, Signal, Context, ScoreEvent, Prediction,
+    Agent, Subsystem, Capability, Project, Goal, Task, Pattern, Concept, Preference, InitiativeCategory,
+)
+EDGE_TYPES = tuple(EdgeType)

apsimo/intelligence/graph/selection.py

@@ -0,0 +1,252 @@
+"""Shared bounded reranking for authorized belief and source candidates."""
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+import os
+import re
+import time
+from typing import Any, Dict, List
+
+
+def current_work_query(query):
+    """Recognize explicit present-work questions, retaining ambiguous requests.
+
+    This is a narrow selection hint, not authority or a semantic classifier.
+    History, comparisons and requests for instructions still need old answers.
+    """
+    if not isinstance(query, str) or len(query) > 8000:
+        return False
+    text = re.sub(r"^\s*\[[^\]\n]{1,512}\]\s*", "", query).strip().lower()
+    if re.search(r"\b(yesterday|histor(?:y|ical)|previous(?:ly)?|earlier|before|then|"
+                 r"last (?:time|week|month|year)|used to|compar(?:ed?|ing|ison)|versus|"
+                 r"how (?:to|do|can|should)|procedure|playbook|recipe|steps|"
+                 r"(?:give|provide|explain|show|need|include|use|follow|recall) (?:the |your |me )?instructions)\b", text):
+        return False
+    return bool(re.match(
+        r"(?:what are you (?:currently (?:doing|working on)|"
+        r"(?:doing|working on) (?:right now|now|currently))\b|"
+        r"what (?:work|tasks|sessions|jobs|crons|workers) (?:are|is) "
+        r"(?:currently |now )?(?:running|active|in flight)\b)", text))
+
+
+class RecallSelector:
+    def __init__(self, rerank_fn=None, *, calibration_metadata=None, logger=None):
+        self._rerank_fn = rerank_fn
+        self._rerank_calibration_metadata = calibration_metadata
+        self.logger = logger or logging.getLogger(__name__)
+
+    async def select_context(self, query, beliefs, quotations, *, limit=5, max_chars=6000,
+                             current_work_available=False):
+        """One rank fusion, reranking pass and budget after authority checks."""
+        from .recall import fuse_candidates, pack_memory_context
+        beliefs = [dict(row, kind="belief") for row in beliefs]
+        # Semantic retrieval can return the same words from many turns. Collapse
+        # only plain quotations by the same known speaker, after scoped claim
+        # expansion and time filtering. Keep the first occurrence's intact
+        # lineage; assertion/conflict bundles, dated events and uncertain
+        # attribution survive. A time-qualified query may need both occurrences.
+        unique, seen = [], set()
+        omit_status_replies = current_work_available and current_work_query(query)
+        for row in quotations:
+            # Only canonical request/answer pairing can identify an old status
+            # reply. Never use quotation text alone or a client owner flag.
+            if (omit_status_replies
+                    and row.get('_current_work_status_reply') is True
+                    and row.get('kind') == 'source_quote' and row.get('role') == 'assistant'
+                    and row.get('scope') == 'person'
+                    and row.get('epistemic_state') == 'quotation'
+                    and not any(row.get(key) for key in (
+                        'atomic_evidence', 'validity_status', 'procedure_context', '_annotation_ids'))):
+                continue
+            if (row.get("kind") == "source_quote" and row.get("epistemic_state") == "quotation"
+                    and not row.get("atomic_evidence") and row.get("scope") == "person"
+                    and not row.get("validity_status")
+                    and row.get("contact_id") and row.get("role") in {"user", "assistant"}):
+                key = (row["contact_id"], row["role"], row["content"])
+                if key in seen:
+                    continue
+                seen.add(key)
+            unique.append(row)
+        quotations = unique
+        # Media is an independent candidate producer. Appending its first hit
+        # after every text hit can exclude it from the bounded reranker before
+        # relevance is assessed. Fuse its rank independently; it still shares
+        # the same reranker, abstention threshold and final context budget.
+        media = [row for row in quotations if row.get("kind") == "media_description"]
+        text = [row for row in quotations if row.get("kind") != "media_description"]
+        # Confidence in a belief and certainty that words were quoted are not
+        # comparable truth scores. Preserve them as evidence metadata; select
+        # across kinds by rank and semantic relevance alone.
+        candidates = fuse_candidates(
+            beliefs, text, limit=len(beliefs) + len(quotations),
+            confidence_weighting=False, additional=(media,))
+        ranked = await self.rerank(
+            query, candidates, limit, confidence_weighting=False,
+            candidate_limit=max(1, 4 * limit))
+        ranked.sort(key=lambda row: row.get("relevance", 0), reverse=True)
+        return pack_memory_context(ranked, limit=limit, max_chars=max_chars)
+
+    async def rerank(
+        self,
+        query: str,
+        memories: List[Dict[str, Any]],
+        limit: int,
+        *,
+        strength_ranking: bool = False,
+        confidence_weighting: bool = True,
+        candidate_limit: int | None = None,
+    ) -> List[Dict[str, Any]]:
+        """Cross-encoder rerank of filtered recall candidates, bounded and
+        fail-open.
+
+        COLONY_RECALL_RERANK gates it: ``off`` (default) never calls the
+        reranker; ``shadow`` scores and logs the rank delta but returns the
+        candidate order untouched (measure p95 before flipping); ``on`` replaces the
+        vector score in the relevance blend with the rerank score. The call
+        is inline but hard-capped by COLONY_RECALL_RERANK_TIMEOUT_MS
+        (default 1200). On timeout or error, recall keeps the candidate order
+        and marks selection unavailable. A candidate set that already fits
+        skips reranking only when no calibrated abstention cutoff is active.
+        A candidate limit bounds model work, while preserving the full input
+        for disabled, shadow and failed selection.
+        """
+        mode = os.environ.get("COLONY_RECALL_RERANK", "off").strip().lower()
+        if mode not in ("shadow", "on"):
+            return memories
+        min_score = None
+        raw_min_score = os.environ.get("COLONY_RECALL_RERANK_MIN_SCORE", "").strip()
+        if raw_min_score:
+            try:
+                value = float(raw_min_score)
+                if math.isfinite(value):
+                    min_score = value
+            except (TypeError, ValueError):
+                pass
+        if min_score is not None:
+            from .recall import calibration_fingerprint
+            metadata_fn = getattr(self, "_rerank_calibration_metadata", None)
+            try:
+                metadata = metadata_fn() if metadata_fn is not None else None
+                actual = calibration_fingerprint(metadata) if metadata else None
+            except Exception:
+                metadata, actual = None, None
+            expected = os.environ.get("COLONY_RECALL_RERANK_CALIBRATION", "").strip()
+            if not actual or not expected or actual != expected:
+                status = "mismatch" if expected and actual else "unverified"
+                min_score = None
+                warning_key = (actual, expected)
+                if getattr(self, "_rerank_calibration_warned", None) != warning_key:
+                    self._rerank_calibration_warned = warning_key
+                    self.logger.warning("Rerank abstention calibration %s; threshold disabled (current configuration: %s)", status, actual or "unknown")
+            else:
+                status = ("configuration_verified" if metadata.get("weights_revision")
+                          and metadata["weights_revision"] != "unverified"
+                          else "configuration_verified_weights_unverified")
+            for memory in memories:
+                memory["rerank_calibration"] = status
+        rerank_fn = getattr(self, "_rerank_fn", None)
+        if rerank_fn is None:
+            for memory in memories:
+                memory["rerank_status"] = "unavailable"
+        if rerank_fn is None or not memories or (len(memories) <= limit and min_score is None):
+            return memories
+        try:
+            timeout_ms = float(os.environ.get(
+                "COLONY_RECALL_RERANK_TIMEOUT_MS", "1200"))
+        except (TypeError, ValueError):
+            timeout_ms = 1200.0
+
+        # Combined recall fuses several producers. Overfetch relative to the
+        # packet size without making the inline model score every producer's
+        # entire result set. Keep the full list available for fallback.
+        submitted = memories if candidate_limit is None else memories[:candidate_limit]
+        docs = [str(m.get("ranking_text", m.get("content", ""))) for m in submitted]
+        try:
+            results = await asyncio.wait_for(
+                rerank_fn(query, docs, top_k=len(docs)),
+                timeout=max(timeout_ms, 1.0) / 1000.0,
+            )
+        except Exception as exc:
+            self._warn_rerank_failure(exc)
+            for memory in memories:
+                memory["rerank_status"] = "unavailable"
+            return memories
+
+        scores: Dict[int, float] = {}
+        try:
+            for r in results or []:
+                idx = r.get("index") if isinstance(r, dict) else getattr(r, "index", None)
+                score = r.get("score") if isinstance(r, dict) else getattr(r, "score", None)
+                if not isinstance(idx, int) or isinstance(idx, bool) or idx in scores:
+                    raise ValueError("reranker returned an invalid or duplicate index")
+                score = float(score)
+                if not 0 <= idx < len(submitted) or not math.isfinite(score):
+                    raise ValueError("reranker returned an invalid score")
+                scores[idx] = score
+            # We requested every submitted document. A partial response cannot
+            # mix cross-encoder scores with original rank-fusion/ANN scores, or
+            # silently treat missing relevance judgments as abstention.
+            if len(scores) != len(submitted):
+                raise ValueError("reranker returned incomplete scores")
+        except (TypeError, ValueError, OverflowError) as exc:
+            self._warn_rerank_failure(exc)
+            for memory in memories:
+                memory["rerank_status"] = "unavailable"
+            return memories
+
+        if mode == "shadow":
+            ann_top = [m.get("id") for m in sorted(
+                memories, key=lambda m: m.get("relevance", 0),
+                reverse=True)][:limit]
+            rr_idx = sorted(range(len(submitted)),
+                            key=lambda i: scores.get(i, float("-inf")),
+                            reverse=True)
+            rr_top = [submitted[i].get("id") for i in rr_idx[:limit]]
+            moved = sum(1 for a, b in zip(ann_top, rr_top) if a != b)
+            self.logger.info(
+                "recall rerank shadow: candidates=%d limit=%d "
+                "top_overlap=%d/%d positions_changed=%d",
+                len(submitted), limit, len(set(ann_top) & set(rr_top)),
+                limit, moved)
+            return memories
+
+        # mode == "on": rerank score replaces the vector score in the blend.
+        # The unsubmitted tail has no comparable model score and cannot enter
+        # a successfully reranked packet.
+        memories = submitted
+        for i, mem in enumerate(memories):
+            effective_confidence = (float(
+                mem.get("effective_confidence", mem.get("strength", 1.0)))
+                if confidence_weighting else 1.0)
+            relevance = scores[i] * effective_confidence
+            if strength_ranking:
+                relevance *= 0.5 + 0.5 * float(mem.get("strength", 1.0))
+            mem["relevance"] = relevance
+            mem["rerank_score"] = scores[i]
+            mem["rerank_status"] = "scored"
+        if min_score is not None:
+            # Do not fill the context window with unrelated passages merely
+            # because there are fewer candidates than requested. Calibration
+            # belongs to the serving model/deployment, not a universal constant.
+            memories = [mem for i, mem in enumerate(memories)
+                        if i in scores and scores[i] >= min_score]
+        return memories
+
+    def _warn_rerank_failure(self, exc: BaseException) -> None:
+        """Warn on rerank failure at most once per ~5 minutes (fail-open is
+        by design; a dead reranker must not turn every recall into a WARNING
+        stream)."""
+        now = time.monotonic()
+        # Sentinel must be None, not 0.0: time.monotonic() is measured from an
+        # arbitrary origin (system boot on Linux), so a 0.0 default suppresses
+        # the FIRST warning entirely for the first 300s of uptime.
+        last = getattr(self, "_rerank_warn_at", None)
+        if last is None or now - last >= 300:
+            self._rerank_warn_at = now
+            self.logger.warning(
+                "recall rerank failed (fail-open to ANN order): %s", exc)
+        else:
+            self.logger.debug(
+                "recall rerank failed (fail-open to ANN order): %s", exc)

apsimo/intelligence/learning/__init__.py

@@ -0,0 +1,17 @@
+"""Colony intelligence learning sub-package.
+
+Provides:
+- FeedbackStore   — persist/retrieve user corrections
+- ContinuousLearner — near-real-time signal ingestion and weight updates
+"""
+
+from .feedback_store import FeedbackStore, UserCorrection
+from .continuous_learner import ContinuousLearner, BriefingEngagement, GoalOutcome
+
+__all__ = [
+    "FeedbackStore",
+    "UserCorrection",
+    "ContinuousLearner",
+    "BriefingEngagement",
+    "GoalOutcome",
+]