apsimo 1.3.0__py3-none-any.whl

apsimo/skills_memory/store.py

@@ -0,0 +1,172 @@
+"""SkillStore -- SQLite persistence for procedure memory + strategy notes."""
+
+from __future__ import annotations
+
+import logging
+import os
+import sqlite3
+import threading
+import time
+from typing import Any, Dict, List, Optional
+
+from apsimo.skills_memory.models import Skill, signature_overlap
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_MAX = 200
+_NOTE_CAP = 600
+
+
+def skills_enabled() -> bool:
+    return os.environ.get("COLONY_SKILLS_ENABLED", "true").strip().lower() != "false"
+
+
+def skills_distill_mode() -> str:
+    from apsimo.util.autonomy_preset import resolve
+    return resolve("COLONY_SKILLS_DISTILL",
+                   ("off", "shadow", "live"), "shadow")
+
+
+def skills_max() -> int:
+    try:
+        return max(10, int(os.environ.get("COLONY_SKILLS_MAX", str(_DEFAULT_MAX))))
+    except (TypeError, ValueError):
+        return _DEFAULT_MAX
+
+
+class SkillStore:
+    def __init__(self, db_path: Optional[str] = None) -> None:
+        self._lock = threading.RLock()
+        self._conn = sqlite3.connect(str(db_path) if db_path else ":memory:",
+                                     check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        with self._lock:
+            self._conn.execute(
+                """CREATE TABLE IF NOT EXISTS skills (
+                    id TEXT PRIMARY KEY, title TEXT NOT NULL, situation TEXT,
+                    situation_signature TEXT, steps TEXT, gotchas TEXT,
+                    domain TEXT, source_ref TEXT,
+                    uses INTEGER DEFAULT 0, wins INTEGER DEFAULT 0,
+                    losses INTEGER DEFAULT 0, confidence REAL DEFAULT 0.6,
+                    created_at REAL, last_used_at REAL
+                )""")
+            self._conn.execute(
+                """CREATE TABLE IF NOT EXISTS strategy_notes (
+                    domain TEXT PRIMARY KEY, note TEXT, updated_at REAL
+                )""")
+            self._conn.commit()
+
+    # -- skills ----------------------------------------------------------
+    def add(self, skill: Skill) -> Skill:
+        row = skill.to_row()
+        with self._lock:
+            cols = ", ".join(row); ph = ", ".join(["?"] * len(row))
+            self._conn.execute(
+                f"INSERT OR REPLACE INTO skills ({cols}) VALUES ({ph})",
+                list(row.values()))
+            self._conn.commit()
+        return skill
+
+    def get(self, skill_id: str) -> Optional[Skill]:
+        with self._lock:
+            r = self._conn.execute(
+                "SELECT * FROM skills WHERE id=?", (skill_id,)).fetchone()
+        return Skill.from_row(dict(r)) if r else None
+
+    def list(self, domain: Optional[str] = None, limit: int = 100) -> List[Skill]:
+        q = "SELECT * FROM skills"
+        params: List[Any] = []
+        if domain:
+            q += " WHERE domain=?"; params.append(domain)
+        q += " ORDER BY last_used_at DESC, created_at DESC LIMIT ?"
+        params.append(limit)
+        with self._lock:
+            rows = self._conn.execute(q, params).fetchall()
+        return [Skill.from_row(dict(r)) for r in rows]
+
+    def count(self) -> int:
+        with self._lock:
+            r = self._conn.execute("SELECT COUNT(*) AS n FROM skills").fetchone()
+        return int(r["n"])
+
+    def find_similar(self, signature: str, threshold: float = 0.8) -> Optional[Skill]:
+        """Existing skill whose signature overlaps `signature` above threshold."""
+        best, best_ov = None, 0.0
+        for s in self.list(limit=100000):
+            ov = signature_overlap(signature, s.situation_signature)
+            if ov > best_ov:
+                best, best_ov = s, ov
+        return best if best is not None and best_ov > threshold else None
+
+    def bump_use(self, skill_id: str) -> None:
+        with self._lock:
+            self._conn.execute(
+                "UPDATE skills SET uses=uses+1, last_used_at=? WHERE id=?",
+                (time.time(), skill_id))
+            self._conn.commit()
+
+    def record_outcome(self, skill_id: str, win: bool) -> None:
+        col = "wins" if win else "losses"
+        with self._lock:
+            self._conn.execute(
+                f"UPDATE skills SET {col}={col}+1 WHERE id=?", (skill_id,))
+            self._conn.commit()
+
+    def evict_to_cap(self, cap: Optional[int] = None) -> int:
+        """Drop lowest-scoring skills until at most `cap` remain."""
+        cap = cap if cap is not None else skills_max()
+        skills = self.list(limit=100000)
+        excess = len(skills) - cap
+        if excess <= 0:
+            return 0
+        now = time.time()
+        victims = sorted(skills, key=lambda s: s.score(now))[:excess]
+        with self._lock:
+            for v in victims:
+                self._conn.execute("DELETE FROM skills WHERE id=?", (v.id,))
+            self._conn.commit()
+        logger.info("skills_memory: evicted %d skill(s) to cap %d", excess, cap)
+        return excess
+
+    # -- per-domain strategy notes (failure post-mortems) -----------------
+    def record_failure_note(self, domain: str, note: str) -> None:
+        """Append/refresh the short strategy note for a domain (capped)."""
+        domain = (domain or "unknown").strip().lower()
+        note = (note or "").strip()
+        if not note:
+            return
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT note FROM strategy_notes WHERE domain=?", (domain,)).fetchone()
+            existing = (row["note"] if row else "") or ""
+            if note in existing:
+                merged = existing
+            else:
+                merged = (existing + ("\n" if existing else "") + f"- {note}")
+            # keep the newest lines within the cap
+            if len(merged) > _NOTE_CAP:
+                lines = merged.splitlines()
+                while lines and len("\n".join(lines)) > _NOTE_CAP:
+                    lines.pop(0)
+                merged = "\n".join(lines)
+            self._conn.execute(
+                """INSERT INTO strategy_notes (domain, note, updated_at)
+                   VALUES (?, ?, ?)
+                   ON CONFLICT(domain) DO UPDATE SET note=?, updated_at=?""",
+                (domain, merged, time.time(), merged, time.time()))
+            self._conn.commit()
+
+    def get_note(self, domain: str) -> str:
+        domain = (domain or "").strip().lower()
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT note FROM strategy_notes WHERE domain=?", (domain,)).fetchone()
+        return (row["note"] if row else "") or ""
+
+    def snapshot(self) -> Dict[str, Any]:
+        return {
+            "count": self.count(),
+            "cap": skills_max(),
+            "distill_mode": skills_distill_mode(),
+            "skills": [s.to_row() for s in self.list(limit=200)],
+        }

apsimo/surprise/__init__.py

@@ -0,0 +1,6 @@
+"""Surprise Engine — anomaly detection when observations deviate from patterns."""
+
+from apsimo.surprise.store import SurpriseStore
+from apsimo.surprise.scorer import compute_surprise
+
+__all__ = ["SurpriseStore", "compute_surprise"]

apsimo/surprise/accumulation.py

@@ -0,0 +1,57 @@
+"""Surprise-accumulation consumer: close the loop into the workspace.
+
+The condition worker emits ``surprise.accumulation`` when unresolved
+surprises pile up, but nothing ever consumed it — the signal died on the
+WebSocket unless a client happened to be watching. This consumer turns the
+event into a workspace concern so accumulating surprises land on the
+agent's mind (one strengthening concern, keyed stably, never a pile-up).
+
+Clean no-op when the workspace is disabled or not wired: the event still
+broadcasts and journals exactly as before.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict
+
+logger = logging.getLogger(__name__)
+
+_DEDUP_KEY = "surprise:accumulation"
+
+
+def handle_surprise_accumulation(event: Dict[str, Any]) -> bool:
+    """Raise/strengthen the accumulation concern. Returns True when a
+    concern was bumped, False on the (clean) no-op paths."""
+    try:
+        from apsimo.self_model.workspace import workspace_enabled
+        if not workspace_enabled():
+            return False
+        from apsimo.api.routers.host import _workspace
+        if _workspace is None:
+            return False
+        payload = event.get("payload") or {}
+        try:
+            count = int(payload.get("unresolved_count") or 0)
+        except (TypeError, ValueError):
+            count = 0
+        _workspace.bump(
+            kind="anomaly",
+            summary=(f"{count} unresolved surprise(s) accumulating — "
+                     "reality keeps diverging from expectations; review "
+                     "and resolve them"),
+            dedup_key=_DEDUP_KEY,
+            salience=min(0.9, 0.4 + 0.05 * count),
+            sources=["surprise-store"])
+        return True
+    except Exception:
+        logger.debug("surprise.accumulation -> workspace failed",
+                     exc_info=True)
+        return False
+
+
+def register() -> None:
+    """Subscribe the consumer. Enablement is checked at EVENT time (not
+    registration time), so a workspace flipped on later just starts working."""
+    from apsimo.events.broadcaster import subscribe
+    subscribe("surprise.accumulation", handle_surprise_accumulation)

apsimo/surprise/scorer.py

@@ -0,0 +1,102 @@
+"""Surprise Scorer — compute surprise scores for observations.
+
+Scoring logic:
+- No matching pattern: surprise = 0.7 (moderately surprising)
+- Pattern violated: surprise = 0.5 + (pattern_confidence * 0.5)
+- Low-frequency pattern match: surprise = 0.2 (rare but known)
+- High-frequency pattern match: surprise = 0.0 (expected)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def compute_surprise(
+    observation: str,
+    *,
+    pattern_store: Any = None,
+    observation_type: str = "",
+    observation_data: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Compute a surprise score for an observation.
+
+    Returns dict with:
+      - surprise_score: float (0.0 to 1.0)
+      - expected: what was expected (if a pattern was found)
+      - pattern_id: the matched/violated pattern ID (if any)
+      - reasoning: human-readable explanation
+    """
+    if pattern_store is None:
+        return {
+            "surprise_score": 0.5,
+            "expected": None,
+            "pattern_id": None,
+            "reasoning": "No pattern store available, defaulting to moderate surprise",
+        }
+
+    # Look for matching patterns by searching descriptions.
+    try:
+        all_patterns = pattern_store.list_patterns(active_only=True, limit=100)
+    except Exception:
+        return {
+            "surprise_score": 0.5,
+            "expected": None,
+            "pattern_id": None,
+            "reasoning": "Could not query pattern store",
+        }
+
+    # Find patterns that mention entities from the observation.
+    matching = []
+    obs_lower = observation.lower()
+    obs_words = set(obs_lower.split())
+
+    for pattern in all_patterns.get("patterns", []):
+        desc_lower = pattern.get("description", "").lower()
+        # Check for word overlap.
+        desc_words = set(desc_lower.split())
+        overlap = obs_words & desc_words
+        if overlap or observation_type == pattern.get("pattern_type", ""):
+            matching.append(pattern)
+
+    if not matching:
+        return {
+            "surprise_score": 0.7,
+            "expected": None,
+            "pattern_id": None,
+            "reasoning": "No matching pattern found for this observation",
+        }
+
+    # Check if the observation fits the pattern or violates it.
+    best_pattern = max(matching, key=lambda p: p.get("frequency", 0) * p.get("confidence", 0.5))
+    freq = best_pattern.get("frequency", 1)
+    conf = best_pattern.get("confidence", 0.5)
+
+    if freq >= 5:
+        # High-frequency pattern match — expected.
+        return {
+            "surprise_score": 0.0,
+            "expected": best_pattern["description"],
+            "pattern_id": best_pattern["id"],
+            "reasoning": f"Matches high-frequency pattern (seen {freq} times)",
+        }
+    elif freq >= 2:
+        # Low-frequency pattern match — rare but known.
+        return {
+            "surprise_score": 0.2,
+            "expected": best_pattern["description"],
+            "pattern_id": best_pattern["id"],
+            "reasoning": f"Matches low-frequency pattern (seen {freq} times)",
+        }
+    else:
+        # Pattern seen only once — could be a violation.
+        score = 0.5 + (conf * 0.5)
+        return {
+            "surprise_score": round(score, 4),
+            "expected": best_pattern["description"],
+            "pattern_id": best_pattern["id"],
+            "reasoning": f"Pattern seen only once (confidence {conf:.1f}), possible violation",
+        }

apsimo/surprise/store.py

@@ -0,0 +1,203 @@
+"""Surprise Store — record and track unexpected observations."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+import uuid
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class SurpriseStore:
+    """SQLite-backed surprise observation store."""
+
+    def __init__(self, db_path: str) -> None:
+        self._db_path = db_path
+        self._conn = sqlite3.connect(db_path)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA foreign_keys=ON")
+        self._create_tables()
+
+    def _create_tables(self) -> None:
+        self._conn.executescript("""
+            CREATE TABLE IF NOT EXISTS surprises (
+                id TEXT PRIMARY KEY,
+                observation TEXT NOT NULL,
+                expected TEXT,
+                surprise_score REAL NOT NULL,
+                pattern_id TEXT,
+                context TEXT,
+                timestamp TEXT NOT NULL,
+                resolved INTEGER NOT NULL DEFAULT 0,
+                resolution TEXT
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_surprises_score
+                ON surprises(surprise_score DESC);
+            CREATE INDEX IF NOT EXISTS idx_surprises_timestamp
+                ON surprises(timestamp);
+            CREATE INDEX IF NOT EXISTS idx_surprises_resolved
+                ON surprises(resolved);
+        """)
+        self._conn.commit()
+
+    def create_surprise(
+        self,
+        *,
+        observation: str,
+        expected: Optional[str] = None,
+        surprise_score: float = 0.5,
+        pattern_id: Optional[str] = None,
+        context: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Record a surprise observation.
+
+        Returns the created surprise dict.
+        """
+        surprise_score = max(0.0, min(1.0, surprise_score))
+
+        surprise_id = str(uuid.uuid4())
+        now = datetime.now(timezone.utc).isoformat()
+        ctx_json = json.dumps(context) if context else None
+
+        self._conn.execute(
+            """INSERT INTO surprises (id, observation, expected, surprise_score,
+                  pattern_id, context, timestamp, resolved, resolution)
+               VALUES (?, ?, ?, ?, ?, ?, ?, 0, NULL)""",
+            (surprise_id, observation, expected, surprise_score,
+             pattern_id, ctx_json, now),
+        )
+        self._conn.commit()
+
+        result = self.get_surprise(surprise_id)
+        assert result is not None
+        return result
+
+    def get_surprise(self, surprise_id: str) -> Optional[Dict[str, Any]]:
+        row = self._conn.execute(
+            "SELECT * FROM surprises WHERE id = ?", (surprise_id,)
+        ).fetchone()
+        if row is None:
+            return None
+        d = dict(row)
+        d["resolved"] = bool(d["resolved"])
+        if d.get("context"):
+            try:
+                d["context"] = json.loads(d["context"])
+            except (json.JSONDecodeError, TypeError):
+                pass
+        return d
+
+    def list_surprises(
+        self,
+        *,
+        min_score: float = 0.0,
+        resolved: Optional[bool] = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> Dict[str, Any]:
+        """List surprises with optional filters, sorted by score descending."""
+        clauses: List[str] = []
+        params: List[Any] = []
+
+        if min_score > 0:
+            clauses.append("surprise_score >= ?")
+            params.append(min_score)
+        if resolved is not None:
+            clauses.append("resolved = ?")
+            params.append(1 if resolved else 0)
+
+        where = f" WHERE {' AND '.join(clauses)}" if clauses else ""
+
+        total_row = self._conn.execute(
+            f"SELECT COUNT(*) as cnt FROM surprises{where}", params
+        ).fetchone()
+        total = total_row["cnt"] if total_row else 0
+
+        rows = self._conn.execute(
+            f"SELECT * FROM surprises{where} ORDER BY surprise_score DESC, timestamp DESC LIMIT ? OFFSET ?",
+            params + [limit, offset],
+        ).fetchall()
+
+        surprises = []
+        for row in rows:
+            d = dict(row)
+            d["resolved"] = bool(d["resolved"])
+            if d.get("context"):
+                try:
+                    d["context"] = json.loads(d["context"])
+                except (json.JSONDecodeError, TypeError):
+                    pass
+            surprises.append(d)
+
+        return {"surprises": surprises, "total": total, "limit": limit, "offset": offset}
+
+    def get_unresolved(self, min_score: float = 0.5, limit: int = 10) -> List[Dict[str, Any]]:
+        """Get unresolved high-score surprises."""
+        result = self.list_surprises(min_score=min_score, resolved=False, limit=limit)
+        return result["surprises"]
+
+    def resolve_surprise(
+        self,
+        surprise_id: str,
+        *,
+        resolution: Optional[str] = None,
+    ) -> Optional[Dict[str, Any]]:
+        """Mark a surprise as resolved. Returns updated surprise or None."""
+        existing = self.get_surprise(surprise_id)
+        if existing is None:
+            return None
+
+        self._conn.execute(
+            "UPDATE surprises SET resolved = 1, resolution = ? WHERE id = ?",
+            (resolution, surprise_id),
+        )
+        self._conn.commit()
+        return self.get_surprise(surprise_id)
+
+    def delete_surprise(self, surprise_id: str) -> bool:
+        """Delete a surprise. Returns True if deleted."""
+        cursor = self._conn.execute("DELETE FROM surprises WHERE id = ?", (surprise_id,))
+        self._conn.commit()
+        return cursor.rowcount > 0
+
+    def resolve_stale(self, ttl_days: float) -> int:
+        """Auto-resolve unresolved surprises older than ``ttl_days``.
+
+        A surprise nobody addressed in two weeks is stale context, not a
+        live signal — leaving it unresolved forever keeps it in context
+        assembly and inflates the accumulation trigger. Marks (never
+        deletes) with an explicit auto-resolution note. ``ttl_days <= 0``
+        disables. Returns the number resolved.
+        """
+        if ttl_days <= 0:
+            return 0
+        from datetime import timedelta
+        cutoff = (datetime.now(timezone.utc)
+                  - timedelta(days=ttl_days)).isoformat()
+        cursor = self._conn.execute(
+            "UPDATE surprises SET resolved = 1, resolution = ? "
+            "WHERE resolved = 0 AND timestamp < ?",
+            (f"auto-resolved: unaddressed for {ttl_days:g} days "
+             "(COLONY_SURPRISE_TTL_DAYS)", cutoff),
+        )
+        self._conn.commit()
+        return cursor.rowcount
+
+    def count_unresolved(self, since_hours: float = 1.0) -> int:
+        """Count unresolved surprises within the last N hours."""
+        from datetime import timedelta
+        cutoff = (datetime.now(timezone.utc) - timedelta(hours=since_hours)).isoformat()
+        row = self._conn.execute(
+            "SELECT COUNT(*) as cnt FROM surprises WHERE resolved = 0 AND timestamp >= ?",
+            (cutoff,),
+        ).fetchone()
+        return row["cnt"] if row else 0
+
+    def close(self) -> None:
+        self._conn.close()

apsimo/task_queue/__init__.py

@@ -0,0 +1,69 @@
+"""Colony Distributed Task Queue.
+
+Hardware-aware distributed job scheduling across the Colony mesh.
+The Queen (Sovereign) schedules; Workers (Vassals) execute.
+
+Public API::
+
+    from apsimo.task_queue import (
+        Job, JobType, JobStatus, JobPriority,
+        JobCapabilityRequirement, JobResult,
+        WorkerCapabilities, QueueManager, WorkerNode,
+        JobHandler, Scheduler, TaskQueueConfig,
+    )
+"""
+
+from apsimo.task_queue.config import TaskQueueConfig
+from apsimo.task_queue.mesh_integration import QueueMeshEventHandler
+from apsimo.task_queue.models import (
+    AuditEntry,
+    CircularDependencyError,
+    FederatedJob,
+    HeartbeatPayload,
+    Job,
+    JobCapabilityRequirement,
+    JobPriority,
+    JobResult,
+    JobStatus,
+    JobType,
+    QueueStats,
+    WorkerCapabilities,
+    deadline_urgency,
+)
+from apsimo.task_queue.queue_manager import QueueManager
+from apsimo.task_queue.scheduler import Scheduler
+from apsimo.task_queue.worker import JobHandler, WorkerNode, detect_local_capabilities
+from apsimo.task_queue.work_control import (
+    WorkControlError,
+    WorkControlService,
+)
+
+__all__ = [
+    # Models
+    "Job",
+    "JobType",
+    "JobStatus",
+    "JobPriority",
+    "JobCapabilityRequirement",
+    "JobResult",
+    "WorkerCapabilities",
+    "HeartbeatPayload",
+    "QueueStats",
+    "AuditEntry",
+    "FederatedJob",
+    "CircularDependencyError",
+    "deadline_urgency",
+    # Core components
+    "QueueManager",
+    "WorkControlService",
+    "WorkControlError",
+    "WorkerNode",
+    "JobHandler",
+    "Scheduler",
+    # Config
+    "TaskQueueConfig",
+    # Mesh integration
+    "QueueMeshEventHandler",
+    # Utilities
+    "detect_local_capabilities",
+]