apsimo 1.3.0__py3-none-any.whl

apsimo/mining/models.py

@@ -0,0 +1,169 @@
+"""Mining models: escalation records + verbatim turn capture.
+
+Concept provenance: inspired by external scaffolding work (drowzeys
+Keys-Setup) on turning cloud/escalation corrections into local supervision
+and serving logs into training data. Concepts only; no code adopted
+(unlicensed + immature upstream).
+
+Env flags
+---------
+COLONY_ESCALATION_MINING = off | shadow | live   (default shadow)
+    off:    no capture, no detection.
+    shadow: capture verbatim turns + bank/journal escalation records.
+    live:   shadow + feed banked escalations into skills-memory distillation.
+COLONY_ESCALATION_CONSULT_REGEX
+    Pattern that marks a turn as a build-agent consultation (the agent
+    shelled out to a coding agent). Deployment-tunable.
+COLONY_ESCALATION_HEAVY_RE
+    Pattern over the per-turn model name that marks a provider escalation
+    (heavy-model or cloud-failover turn). Empty (default) disables the
+    provider detector; deployments list their heavy/cloud models.
+COLONY_MINING_TURN_CAP
+    Per-side verbatim capture cap in chars (default 8000).
+COLONY_CORPUS_EXPORT_ENABLED
+    Gate for the corpus exporter endpoint (default true; exports only run
+    on explicit request and never leave the state dir).
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+ESCALATION_KINDS = ("consultation", "provider_escalation")
+
+
+def mining_mode() -> str:
+    from apsimo.util.autonomy_preset import resolve
+    return resolve("COLONY_ESCALATION_MINING",
+                   ("off", "shadow", "live"), "shadow")
+
+
+def corpus_export_enabled() -> bool:
+    return os.environ.get(
+        "COLONY_CORPUS_EXPORT_ENABLED", "true"
+    ).strip().lower() not in ("false", "0", "no", "off")
+
+
+def consult_regex() -> str:
+    return os.environ.get(
+        "COLONY_ESCALATION_CONSULT_REGEX",
+        r"claude\s+-p|claude[-_ ]code|\bcode agent\b",
+    )
+
+
+def heavy_model_regex() -> str:
+    return os.environ.get("COLONY_ESCALATION_HEAVY_RE", "")
+
+
+def turn_cap() -> int:
+    try:
+        return max(500, int(os.environ.get("COLONY_MINING_TURN_CAP", "8000")))
+    except ValueError:
+        return 8000
+
+
+@dataclass
+class MinedTurn:
+    """One verbatim conversation exchange, banked for corpus export."""
+
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    session_id: str = ""
+    contact_id: str = ""
+    channel_id: str = ""
+    user_text: str = ""
+    assistant_text: str = ""
+    summary: str = ""
+    tools_used: List[str] = field(default_factory=list)
+    model: str = ""
+    ts: float = field(default_factory=time.time)
+
+    def to_row(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "session_id": self.session_id,
+            "contact_id": self.contact_id,
+            "channel_id": self.channel_id,
+            "user_text": self.user_text,
+            "assistant_text": self.assistant_text,
+            "summary": self.summary,
+            "tools_used": json.dumps(self.tools_used),
+            "model": self.model,
+            "ts": self.ts,
+        }
+
+    @classmethod
+    def from_row(cls, row: Dict[str, Any]) -> "MinedTurn":
+        return cls(
+            id=row["id"],
+            session_id=row.get("session_id") or "",
+            contact_id=row.get("contact_id") or "",
+            channel_id=row.get("channel_id") or "",
+            user_text=row.get("user_text") or "",
+            assistant_text=row.get("assistant_text") or "",
+            summary=row.get("summary") or "",
+            tools_used=json.loads(row.get("tools_used") or "[]"),
+            model=row.get("model") or "",
+            ts=float(row.get("ts") or 0.0),
+        )
+
+
+@dataclass
+class EscalationRecord:
+    """A turn where the agent escalated: consulted a build agent, or the
+    turn ran on a heavy/cloud model. High-value supervision material."""
+
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    kind: str = "consultation"          # one of ESCALATION_KINDS
+    session_id: str = ""
+    contact_id: str = ""
+    channel_id: str = ""
+    task_context: str = ""              # what was being asked/attempted
+    local_attempt: str = ""             # prior local answer in-session, if any
+    escalated_answer: str = ""          # the answer produced with escalation
+    model: str = ""
+    matched: str = ""                   # which signature matched (audit)
+    outcome: str = "unknown"            # unknown | followed_up
+    outcome_note: str = ""              # next user message excerpt
+    distilled: int = 0                  # fed to skills distillation (live mode)
+    ts: float = field(default_factory=time.time)
+
+    def to_row(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "kind": self.kind,
+            "session_id": self.session_id,
+            "contact_id": self.contact_id,
+            "channel_id": self.channel_id,
+            "task_context": self.task_context,
+            "local_attempt": self.local_attempt,
+            "escalated_answer": self.escalated_answer,
+            "model": self.model,
+            "matched": self.matched,
+            "outcome": self.outcome,
+            "outcome_note": self.outcome_note,
+            "distilled": self.distilled,
+            "ts": self.ts,
+        }
+
+    @classmethod
+    def from_row(cls, row: Dict[str, Any]) -> "EscalationRecord":
+        return cls(
+            id=row["id"],
+            kind=row.get("kind") or "consultation",
+            session_id=row.get("session_id") or "",
+            contact_id=row.get("contact_id") or "",
+            channel_id=row.get("channel_id") or "",
+            task_context=row.get("task_context") or "",
+            local_attempt=row.get("local_attempt") or "",
+            escalated_answer=row.get("escalated_answer") or "",
+            model=row.get("model") or "",
+            matched=row.get("matched") or "",
+            outcome=row.get("outcome") or "unknown",
+            outcome_note=row.get("outcome_note") or "",
+            distilled=int(row.get("distilled") or 0),
+            ts=float(row.get("ts") or 0.0),
+        )

apsimo/mining/store.py

@@ -0,0 +1,210 @@
+"""MiningStore: SQLite persistence for verbatim turns + escalation records.
+
+The rest of the sidecar keeps salience-gated summaries (memory graph, comms
+ledger, rolling journal); none of those is a durable verbatim corpus. This
+store is: every accepted turn is banked verbatim (capped per side), queryable
+by channel / contact / date range, feeding the escalation miner and the
+training-corpus exporter. Data never leaves COLONY_STATE_DIR.
+"""
+from __future__ import annotations
+
+import sqlite3
+import threading
+import time
+from typing import Any, List, Optional
+
+from apsimo.mining.models import EscalationRecord, MinedTurn
+
+
+class MiningStore:
+    def __init__(self, db_path: Optional[str] = None) -> None:
+        self._lock = threading.RLock()
+        self._conn = sqlite3.connect(
+            str(db_path) if db_path else ":memory:", check_same_thread=False
+        )
+        self._conn.row_factory = sqlite3.Row
+        with self._lock:
+            self._conn.execute(
+                """CREATE TABLE IF NOT EXISTS mined_turns (
+                    id TEXT PRIMARY KEY,
+                    session_id TEXT,
+                    contact_id TEXT,
+                    channel_id TEXT,
+                    user_text TEXT,
+                    assistant_text TEXT,
+                    summary TEXT,
+                    tools_used TEXT,
+                    model TEXT,
+                    ts REAL
+                )"""
+            )
+            self._conn.execute(
+                """CREATE TABLE IF NOT EXISTS escalations (
+                    id TEXT PRIMARY KEY,
+                    kind TEXT,
+                    session_id TEXT,
+                    contact_id TEXT,
+                    channel_id TEXT,
+                    task_context TEXT,
+                    local_attempt TEXT,
+                    escalated_answer TEXT,
+                    model TEXT,
+                    matched TEXT,
+                    outcome TEXT,
+                    outcome_note TEXT,
+                    distilled INTEGER DEFAULT 0,
+                    ts REAL
+                )"""
+            )
+            self._conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_turns_session ON mined_turns(session_id, ts)"
+            )
+            self._conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_turns_ts ON mined_turns(ts)"
+            )
+            self._conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_esc_session ON escalations(session_id, ts)"
+            )
+            self._conn.commit()
+
+    # -- turns ---------------------------------------------------------------
+
+    def add_turn(self, t: MinedTurn) -> MinedTurn:
+        row = t.to_row()
+        with self._lock:
+            cols = ", ".join(row)
+            ph = ", ".join(["?"] * len(row))
+            self._conn.execute(
+                f"INSERT OR REPLACE INTO mined_turns ({cols}) VALUES ({ph})",
+                list(row.values()),
+            )
+            self._conn.commit()
+        return t
+
+    def last_turn_in_session(self, session_id: str) -> Optional[MinedTurn]:
+        with self._lock:
+            r = self._conn.execute(
+                "SELECT * FROM mined_turns WHERE session_id=? ORDER BY ts DESC LIMIT 1",
+                (session_id,),
+            ).fetchone()
+        return MinedTurn.from_row(dict(r)) if r else None
+
+    def list_turns(
+        self,
+        *,
+        contact_id: Optional[str] = None,
+        channels: Optional[List[str]] = None,
+        since_ts: Optional[float] = None,
+        until_ts: Optional[float] = None,
+        limit: int = 5000,
+    ) -> List[MinedTurn]:
+        q = "SELECT * FROM mined_turns WHERE 1=1"
+        params: List[Any] = []
+        if contact_id and contact_id != "*":
+            q += " AND contact_id=?"
+            params.append(contact_id)
+        if channels:
+            q += f" AND channel_id IN ({','.join(['?'] * len(channels))})"
+            params.extend(channels)
+        if since_ts is not None:
+            q += " AND ts >= ?"
+            params.append(since_ts)
+        if until_ts is not None:
+            q += " AND ts <= ?"
+            params.append(until_ts)
+        q += " ORDER BY session_id, ts ASC LIMIT ?"
+        params.append(limit)
+        with self._lock:
+            rows = self._conn.execute(q, params).fetchall()
+        return [MinedTurn.from_row(dict(r)) for r in rows]
+
+    def turn_count(self) -> int:
+        with self._lock:
+            r = self._conn.execute("SELECT COUNT(*) AS n FROM mined_turns").fetchone()
+        return int(r["n"])
+
+    def prune_turns(self, *, retention_days: int = 0,
+                    max_turns: int = 0) -> dict:
+        """Prune banked turns by age and/or count (newest kept).
+
+        Both knobs default to 0 = keep everything: the verbatim corpus is
+        unbounded by design unless a deployment opts into retention via
+        COLONY_MINING_RETENTION_DAYS / COLONY_MINING_MAX_TURNS. Escalation
+        records are never pruned here — they are the distilled, high-value
+        output the turn bank exists to produce.
+        """
+        deleted_by_age = deleted_by_count = 0
+        with self._lock:
+            if retention_days > 0:
+                cutoff = time.time() - retention_days * 86400
+                cur = self._conn.execute(
+                    "DELETE FROM mined_turns WHERE ts < ?", (cutoff,))
+                deleted_by_age = cur.rowcount
+            if max_turns > 0:
+                cur = self._conn.execute(
+                    "DELETE FROM mined_turns WHERE id NOT IN ("
+                    "SELECT id FROM mined_turns ORDER BY ts DESC LIMIT ?)",
+                    (max_turns,))
+                deleted_by_count = cur.rowcount
+            if deleted_by_age or deleted_by_count:
+                self._conn.commit()
+        return {"deleted_by_age": deleted_by_age,
+                "deleted_by_count": deleted_by_count,
+                "remaining": self.turn_count()}
+
+    # -- escalations -----------------------------------------------------------
+
+    def add_escalation(self, e: EscalationRecord) -> EscalationRecord:
+        row = e.to_row()
+        with self._lock:
+            cols = ", ".join(row)
+            ph = ", ".join(["?"] * len(row))
+            self._conn.execute(
+                f"INSERT OR REPLACE INTO escalations ({cols}) VALUES ({ph})",
+                list(row.values()),
+            )
+            self._conn.commit()
+        return e
+
+    def update_escalation(self, e: EscalationRecord) -> EscalationRecord:
+        return self.add_escalation(e)
+
+    def latest_open_escalation(self, session_id: str) -> Optional[EscalationRecord]:
+        with self._lock:
+            r = self._conn.execute(
+                "SELECT * FROM escalations WHERE session_id=? AND outcome='unknown' "
+                "ORDER BY ts DESC LIMIT 1",
+                (session_id,),
+            ).fetchone()
+        return EscalationRecord.from_row(dict(r)) if r else None
+
+    def list_escalations(
+        self, *, kind: Optional[str] = None, limit: int = 50
+    ) -> List[EscalationRecord]:
+        q = "SELECT * FROM escalations"
+        params: List[Any] = []
+        if kind:
+            q += " WHERE kind=?"
+            params.append(kind)
+        q += " ORDER BY ts DESC LIMIT ?"
+        params.append(limit)
+        with self._lock:
+            rows = self._conn.execute(q, params).fetchall()
+        return [EscalationRecord.from_row(dict(r)) for r in rows]
+
+    def escalation_stats(self) -> dict:
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT kind, outcome, COUNT(*) AS n, SUM(distilled) AS d "
+                "FROM escalations GROUP BY kind, outcome"
+            ).fetchall()
+            total = self._conn.execute(
+                "SELECT COUNT(*) AS n FROM escalations"
+            ).fetchone()
+        by_bucket = [
+            {"kind": r["kind"], "outcome": r["outcome"], "count": int(r["n"]),
+             "distilled": int(r["d"] or 0)}
+            for r in rows
+        ]
+        return {"total": int(total["n"]), "buckets": by_bucket,
+                "turns_banked": self.turn_count()}

apsimo/models/__init__.py

@@ -0,0 +1,30 @@
+"""Colony foundation data models.
+
+Core dataclasses for people, signals, memories, and mesh nodes.
+These models are the shared vocabulary across all Colony subsystems.
+"""
+
+from .memory import Memory, MemoryStrength, MemoryType
+from .mesh import MeshNode, NodeRole, NodeStatus
+from .person import ContactInfo, ContactType, Person, RelationshipTier
+from .signal import Signal, SignalStrength, SignalType
+
+__all__ = [
+    # Person & relationships
+    "Person",
+    "ContactInfo",
+    "ContactType",
+    "RelationshipTier",
+    # Behavioral signals
+    "Signal",
+    "SignalType",
+    "SignalStrength",
+    # Memory
+    "Memory",
+    "MemoryType",
+    "MemoryStrength",
+    # Mesh
+    "MeshNode",
+    "NodeRole",
+    "NodeStatus",
+]

apsimo/models/memory.py

@@ -0,0 +1,80 @@
+"""Memory storage models.
+
+Colony's memory system stores three types of knowledge:
+- Episodic: Events and conversations (what happened)
+- Semantic: Facts and preferences (what is known)
+- Procedural: Workflows and processes (how to do things)
+
+Memories support semantic search via embeddings and decay/reinforce
+based on access patterns.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+
+class MemoryType(str, Enum):
+    """Classification of memory content.
+
+    EPISODIC: Events, conversations, interactions
+    SEMANTIC: Facts, preferences, knowledge
+    PROCEDURAL: How-to knowledge, workflows, processes
+    """
+
+    EPISODIC = "episodic"
+    SEMANTIC = "semantic"
+    PROCEDURAL = "procedural"
+
+
+class MemoryStrength(str, Enum):
+    """How strongly a memory is retained.
+
+    LOW: Weak retention, score < 0.3. Candidate for decay.
+    MEDIUM: Normal retention, score 0.3-0.7.
+    HIGH: Strong retention, score > 0.7. Frequently accessed or important.
+    """
+
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+
+
+@dataclass
+class Memory:
+    """A single memory unit in Colony's long-term store.
+
+    Memories are created from interactions, observations, and explicit input.
+    They support vector similarity search via the embedding field and have
+    strength/importance scores that evolve with access patterns.
+
+    Attributes:
+        id: Unique identifier
+        type: Category of memory (episodic, semantic, procedural)
+        content: The actual memory content as text
+        person_id: Related person, if the memory is about someone specific
+        strength: Retention strength 0-1, decays without access
+        importance: How important this memory is 0-1
+        tags: Categorization tags for filtering
+        embedding: Vector embedding for semantic search
+        source: Where this memory originated (channel, file, etc.)
+        created_at: When the memory was first stored
+        last_accessed: Most recent retrieval timestamp
+        access_count: How many times this memory has been retrieved
+        metadata: Additional structured data
+    """
+
+    id: str
+    type: MemoryType
+    content: str
+    person_id: Optional[str] = None
+    strength: float = 0.5
+    importance: float = 0.5
+    tags: List[str] = field(default_factory=list)
+    embedding: Optional[List[float]] = None
+    source: Optional[str] = None
+    created_at: Optional[datetime] = None
+    last_accessed: Optional[datetime] = None
+    access_count: int = 0
+    metadata: Dict[str, Any] = field(default_factory=dict)

apsimo/models/mesh.py

@@ -0,0 +1,72 @@
+"""Mesh node data models.
+
+Colony runs as a distributed mesh where nodes take on different roles.
+The Sovereign node is the primary brain, Regents are backup brains,
+and Vassals are worker nodes that handle specific tasks.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+
+class NodeRole(str, Enum):
+    """Role a node plays in the Colony mesh.
+
+    SOVEREIGN: The queen node with full brain capabilities.
+    REGENT: Backup brain, can assume sovereignty if needed.
+    VASSAL: Worker node, handles delegated tasks.
+    """
+
+    SOVEREIGN = "sovereign"
+    REGENT = "regent"
+    VASSAL = "vassal"
+
+
+class NodeStatus(str, Enum):
+    """Current operational status of a mesh node.
+
+    ONLINE: Node is healthy and responsive.
+    OFFLINE: Node is unreachable.
+    DEGRADED: Node is up but not fully functional.
+    SYNCING: Node is catching up on state.
+    """
+
+    ONLINE = "online"
+    OFFLINE = "offline"
+    DEGRADED = "degraded"
+    SYNCING = "syncing"
+
+
+@dataclass
+class MeshNode:
+    """A node in the Colony mesh network.
+
+    Each node has a role, capabilities, and resource score that determines
+    what work gets routed to it. Nodes communicate via encrypted channels
+    using their public keys.
+
+    Attributes:
+        id: Unique node identifier
+        role: What role this node serves (sovereign, regent, vassal)
+        status: Current operational status
+        capabilities: What this node can do ("gpu", "neo4j", "llm", etc.)
+        resource_score: Relative compute power score (higher = more capable)
+        endpoint: URL for the node's API
+        public_key: Encryption key for secure mesh communication
+        last_seen: Most recent heartbeat timestamp
+        registered_at: When the node joined the mesh
+        metadata: Additional node-specific data
+    """
+
+    id: str
+    role: NodeRole
+    status: NodeStatus = NodeStatus.OFFLINE
+    capabilities: List[str] = field(default_factory=list)
+    resource_score: int = 0
+    endpoint: Optional[str] = None
+    public_key: Optional[bytes] = None
+    last_seen: Optional[datetime] = None
+    registered_at: Optional[datetime] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)

apsimo/models/person.py

@@ -0,0 +1,104 @@
+"""Person and relationship data models.
+
+Core identity and relationship tracking for Colony's social graph.
+Each person has a tier (computed from behavioral signals) and contact info
+across multiple channels.
+
+Two-layer model
+---------------
+Colony maintains two distinct layers of relationship data:
+
+- **Colony layer** (``score``, ``tier``): Colony's own evidence-based assessment,
+  derived from interaction signals observed across connected gateways. A score
+  of 80 means Colony has observed strong, consistent engagement — not that the
+  owner designated this person as important.
+
+- **Owner layer** (``notes``, manual tier overrides in TrustTierManager): explicit
+  guidance the owner provides to inform Colony's initial orientation. Owner input
+  carries weight but does not override Colony's evidence once sufficient history
+  exists. The ``notes`` field is an owner-layer input: free-form context the
+  owner wants Colony to keep in mind.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import List, Optional
+
+
+class RelationshipTier(str, Enum):
+    """Relationship tier based on interaction score.
+
+    Score ranges:
+        INNER_CIRCLE: 80-100 — closest relationships
+        TRUSTED: 60-79 — reliable, frequent contact
+        REGULAR: 30-59 — normal interaction level
+        PERIPHERAL: 10-29 — infrequent or new contacts
+        SILENCED: manual — explicitly muted by user
+    """
+
+    INNER_CIRCLE = "inner_circle"
+    TRUSTED = "trusted"
+    REGULAR = "regular"
+    PERIPHERAL = "peripheral"
+    SILENCED = "silenced"
+
+
+class ContactType(str, Enum):
+    """Supported communication channels."""
+
+    PHONE = "phone"
+    EMAIL = "email"
+    TELEGRAM = "telegram"
+    DISCORD = "discord"
+    WHATSAPP = "whatsapp"
+    SIGNAL = "signal"
+
+
+@dataclass
+class ContactInfo:
+    """A single contact method for a person.
+
+    Attributes:
+        type: Communication channel (phone, email, etc.)
+        value: The actual contact identifier (number, address, handle)
+        label: Optional label like "personal", "work"
+        last_used: When this contact method was last used
+    """
+
+    type: ContactType
+    value: str
+    label: Optional[str] = None
+    last_used: Optional[datetime] = None
+
+
+@dataclass
+class Person:
+    """A person in Colony's social graph.
+
+    Represents anyone the Colony owner interacts with. The owner themselves
+    is represented with is_user=True.
+
+    Attributes:
+        id: Unique identifier
+        name: Display name
+        tier: Relationship tier (computed from score or manually set)
+        score: Relationship score 0-100, drives tier computation
+        aliases: Alternative names or handles
+        contacts: List of contact methods
+        first_seen: When this person first appeared in the system
+        last_interaction: Most recent interaction timestamp
+        notes: Free-form notes about the person
+        is_user: True if this represents the Colony owner
+    """
+
+    id: str
+    name: str
+    tier: RelationshipTier = RelationshipTier.PERIPHERAL
+    score: float = 0.0  # Colony's evidence-based assessment (0–100); not owner-assigned
+    aliases: List[str] = field(default_factory=list)
+    contacts: List[ContactInfo] = field(default_factory=list)
+    first_seen: Optional[datetime] = None
+    last_interaction: Optional[datetime] = None
+    notes: Optional[str] = None  # Owner-layer input: context the owner wants Colony to know
+    is_user: bool = False