apsimo 1.3.0__py3-none-any.whl

apsimo/skills/executors/data_quality.py

@@ -0,0 +1,163 @@
+"""Data quality executor skill.
+
+Monitors schema drift, orphaned records, and data inconsistencies.
+Attempts deterministic auto-fixes, proposes ambiguous ones.
+"""
+
+import logging
+from typing import Any, Dict
+
+from apsimo.skills.base import (
+    ExecutionResult,
+    InitiativeExecutionContext,
+    InitiativeExecutorSkill,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class DataQualitySkill(InitiativeExecutorSkill):
+    """Skill for monitoring and fixing data quality issues."""
+
+    skill_name = "data_quality"
+    skill_version = "1.0.0"
+
+    async def can_execute(
+        self, category: Dict[str, Any], context: Dict[str, Any]
+    ) -> bool:
+        return category.get("executor_skill") == self.skill_name
+
+    async def execute(
+        self, initiative: InitiativeExecutionContext
+    ) -> ExecutionResult:
+        entity_id = initiative.entity_id or "unknown"
+        entity_type = initiative.entity_type or "schema"
+        self._log("info", "Checking data quality: %s (%s)", entity_id, entity_type)
+
+        if entity_type == "schema_drift":
+            return await self._handle_schema_drift(entity_id, initiative.trigger_data)
+        elif entity_type == "orphan_nodes":
+            return await self._handle_orphan_nodes(entity_id, initiative.trigger_data)
+        elif entity_type == "stale_index":
+            return await self._handle_stale_index(entity_id, initiative.trigger_data)
+        else:
+            self._log("warning", "Unknown data quality issue type: %s", entity_type)
+            return ExecutionResult.NO_ACTION
+
+    async def _handle_schema_drift(
+        self, entity_id: str, trigger_data: Dict[str, Any]
+    ) -> ExecutionResult:
+        """Handle detected schema drift."""
+        self._log("info", "Handling schema drift: %s", entity_id)
+
+        drift = trigger_data.get("drift", {})
+        missing_relationships = drift.get("missing_relationships", [])
+        missing_properties = drift.get("missing_properties", [])
+
+        fixes_applied = []
+        fixes_proposed = []
+
+        # Auto-fix: create missing relationships if source/target exist
+        for rel in missing_relationships:
+            rel_type = rel.get("type")
+            from_node = rel.get("from")
+            to_node = rel.get("to")
+            if not all([rel_type, from_node, to_node]):
+                continue
+
+            if self.graph:
+                try:
+                    async with self.graph.driver.session(
+                        database=self.graph.database
+                    ) as session:
+                        # Check if nodes exist before creating relationship
+                        result = await session.run(
+                            "MATCH (a {id: $from_id}), (b {id: $to_id}) "
+                            "RETURN count(a) as a_count, count(b) as b_count",
+                            from_id=from_node,
+                            to_node=to_node,
+                        )
+                        record = await result.single()
+                        if record and record.get("a_count") > 0 and record.get("b_count") > 0:
+                            # Safe to create relationship
+                            await session.run(
+                                "MATCH (a {id: $from_id}), (b {id: $to_id}) "
+                                "MERGE (a)-[r:" + rel_type + "]->(b) "
+                                "RETURN count(r) as created",
+                                from_id=from_node,
+                                to_node=to_node,
+                            )
+                            fixes_applied.append(rel)
+                        else:
+                            fixes_proposed.append(rel)
+                except Exception as e:
+                    self._log("warning", "Failed to fix relationship %s: %s", rel, e)
+                    fixes_proposed.append(rel)
+
+        # Auto-fix: set default values for missing properties
+        for prop in missing_properties:
+            node_id = prop.get("node_id")
+            prop_name = prop.get("property")
+            default_value = prop.get("default")
+            if not all([node_id, prop_name]):
+                continue
+
+            if self.graph:
+                try:
+                    async with self.graph.driver.session(
+                        database=self.graph.database
+                    ) as session:
+                        await session.run(
+                            "MATCH (n {id: $node_id}) "
+                            "SET n." + prop_name + " = $default_value",
+                            node_id=node_id,
+                            default_value=default_value,
+                        )
+                        fixes_applied.append(prop)
+                except Exception as e:
+                    self._log("warning", "Failed to set property %s: %s", prop, e)
+                    fixes_proposed.append(prop)
+
+        if fixes_applied and not fixes_proposed:
+            self._log("info", "Auto-fixed %d schema issues", len(fixes_applied))
+            return ExecutionResult.AUTO_FIXED
+        elif fixes_proposed:
+            self._log("info", "Proposed %d fixes, applied %d", len(fixes_proposed), len(fixes_applied))
+            return ExecutionResult.PROPOSAL_CREATED
+        else:
+            return ExecutionResult.NO_ACTION
+
+    async def _handle_orphan_nodes(
+        self, entity_id: str, trigger_data: Dict[str, Any]
+    ) -> ExecutionResult:
+        """Handle orphaned nodes (e.g., Memory without :ABOUT edges)."""
+        self._log("info", "Handling orphan nodes: %s", entity_id)
+        orphan_count = trigger_data.get("count", 0)
+
+        if orphan_count == 0:
+            return ExecutionResult.NO_ACTION
+
+        # For now, just log and report. In the future, could attempt
+        # to link orphans to relevant Person nodes via content analysis.
+        self._log("info", "Found %d orphan nodes", orphan_count)
+        return ExecutionResult.PROPOSAL_CREATED
+
+    async def _handle_stale_index(
+        self, entity_id: str, trigger_data: Dict[str, Any]
+    ) -> ExecutionResult:
+        """Handle stale indexes (e.g., LanceDB)."""
+        self._log("info", "Handling stale index: %s", entity_id)
+        index_age_days = trigger_data.get("age_days", 0)
+
+        if index_age_days < 7:
+            return ExecutionResult.NO_ACTION
+
+        # Trigger re-index via event bus
+        if self.events:
+            try:
+                self._log("info", "Requesting re-index for %s (stale %d days)", entity_id, index_age_days)
+                return ExecutionResult.AUTO_FIXED
+            except Exception as e:
+                self._log("warning", "Failed to request re-index: %s", e)
+
+        return ExecutionResult.FAILED

apsimo/skills/executors/knowledge_acquisition.py

@@ -0,0 +1,41 @@
+"""Knowledge acquisition executor skill.
+
+Proposes research tasks for low-confidence knowledge areas.
+v0.11.1: proposal-only to avoid cross-layer circular dependency
+with ResearchPipeline.
+"""
+
+import logging
+from typing import Any, Dict
+
+from apsimo.skills.base import (
+    ExecutionResult,
+    InitiativeExecutionContext,
+    InitiativeExecutorSkill,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class KnowledgeAcquisitionSkill(InitiativeExecutorSkill):
+    """Skill for queuing research on knowledge gaps."""
+
+    skill_name = "knowledge_acquisition"
+    skill_version = "1.0.0"
+
+    async def can_execute(
+        self, category: Dict[str, Any], context: Dict[str, Any]
+    ) -> bool:
+        return category.get("executor_skill") == self.skill_name
+
+    async def execute(
+        self, initiative: InitiativeExecutionContext
+    ) -> ExecutionResult:
+        entity_id = initiative.entity_id or "unknown"
+        name = initiative.trigger_data.get("name", "Unknown concept")
+        confidence = initiative.trigger_data.get("confidence_score", 0.0)
+        self._log("info", "Knowledge gap: %s (confidence %.2f)", name, confidence)
+
+        # v0.11.1: proposal-only. Auto-research deferred to v0.11.2
+        # to avoid importing ResearchPipeline into skills/executors.
+        return ExecutionResult.PROPOSAL_CREATED

apsimo/skills/executors/operational_hygiene.py

@@ -0,0 +1,185 @@
+"""Operational hygiene executor skill.
+
+Handles backups, log rotation, disk space, and scheduled maintenance.
+"""
+
+import logging
+import os
+import shutil
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Dict
+
+from apsimo.skills.base import (
+    ExecutionResult,
+    InitiativeExecutionContext,
+    InitiativeExecutorSkill,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class OperationalHygieneSkill(InitiativeExecutorSkill):
+    """Skill for operational maintenance tasks."""
+
+    skill_name = "operational_hygiene"
+    skill_version = "1.0.0"
+
+    async def can_execute(
+        self, category: Dict[str, Any], context: Dict[str, Any]
+    ) -> bool:
+        return category.get("executor_skill") == self.skill_name
+
+    async def execute(
+        self, initiative: InitiativeExecutionContext
+    ) -> ExecutionResult:
+        entity_id = initiative.entity_id or "unknown"
+        entity_type = initiative.entity_type or "maintenance"
+        self._log("info", "Operational task: %s (%s)", entity_id, entity_type)
+
+        if entity_type == "backup":
+            return await self._handle_backup(entity_id, initiative.trigger_data)
+        elif entity_type == "log_rotation":
+            return await self._handle_log_rotation(entity_id, initiative.trigger_data)
+        elif entity_type == "disk_cleanup":
+            return await self._handle_disk_cleanup(entity_id, initiative.trigger_data)
+        elif entity_type == "cache_refresh":
+            return await self._handle_cache_refresh(entity_id, initiative.trigger_data)
+        else:
+            self._log("warning", "Unknown operational task: %s", entity_type)
+            return ExecutionResult.NO_ACTION
+
+    async def _handle_backup(
+        self, entity_id: str, trigger_data: Dict[str, Any]
+    ) -> ExecutionResult:
+        """Trigger a database backup."""
+        self._log("info", "Triggering backup for: %s", entity_id)
+
+        backup_dir = Path(os.path.expanduser("~/.colony/backups"))
+        backup_dir.mkdir(parents=True, exist_ok=True)
+
+        timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+
+        # Backup SQLite stores
+        stores = ["colony.db", "colony-contacts.db"]
+        backed_up = []
+        for store_name in stores:
+            source = Path(os.path.expanduser(f"~/.colony/data/{store_name}"))
+            if source.exists():
+                dest = backup_dir / f"{store_name}.{timestamp}.bak"
+                try:
+                    shutil.copy2(source, dest)
+                    backed_up.append(store_name)
+                except Exception as e:
+                    self._log("warning", "Failed to backup %s: %s", store_name, e)
+
+        if backed_up:
+            self._log("info", "Backed up %d stores", len(backed_up))
+            # Clean up old backups (keep last 7)
+            await self._cleanup_old_backups(backup_dir, keep=7)
+            return ExecutionResult.AUTO_FIXED
+        else:
+            return ExecutionResult.NO_ACTION
+
+    async def _cleanup_old_backups(self, backup_dir: Path, keep: int = 7) -> None:
+        """Remove old backups, keeping only the most recent N."""
+        try:
+            backups = sorted(
+                backup_dir.glob("*.bak"),
+                key=lambda p: p.stat().st_mtime,
+                reverse=True,
+            )
+            for old_backup in backups[keep:]:
+                old_backup.unlink()
+                self._log("debug", "Removed old backup: %s", old_backup.name)
+        except Exception as e:
+            self._log("warning", "Failed to cleanup old backups: %s", e)
+
+    async def _handle_log_rotation(
+        self, entity_id: str, trigger_data: Dict[str, Any]
+    ) -> ExecutionResult:
+        """Rotate log files if they exceed threshold."""
+        self._log("info", "Rotating logs for: %s", entity_id)
+
+        log_dir = Path(os.path.expanduser("~/.colony/logs"))
+        if not log_dir.exists():
+            return ExecutionResult.NO_ACTION
+
+        threshold_mb = trigger_data.get("threshold_mb", 100)
+        rotated = []
+
+        for log_file in log_dir.glob("*.log"):
+            try:
+                size_mb = log_file.stat().st_size / (1024 * 1024)
+                if size_mb > threshold_mb:
+                    # Compress and rotate
+                    timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+                    rotated_name = f"{log_file.stem}.{timestamp}.log.gz"
+                    rotated_path = log_dir / rotated_name
+
+                    import gzip
+                    with open(log_file, "rb") as f_in:
+                        with gzip.open(rotated_path, "wb") as f_out:
+                            shutil.copyfileobj(f_in, f_out)
+
+                    # Truncate original
+                    log_file.write_text("")
+                    rotated.append(str(log_file.name))
+                    self._log("info", "Rotated %s (%.1f MB)", log_file.name, size_mb)
+            except Exception as e:
+                self._log("warning", "Failed to rotate %s: %s", log_file.name, e)
+
+        if rotated:
+            return ExecutionResult.AUTO_FIXED
+        return ExecutionResult.NO_ACTION
+
+    async def _handle_disk_cleanup(
+        self, entity_id: str, trigger_data: Dict[str, Any]
+    ) -> ExecutionResult:
+        """Clean up temporary files and caches."""
+        self._log("info", "Cleaning disk for: %s", entity_id)
+
+        cleaned_bytes = 0
+        paths_to_clean = [
+            Path(os.path.expanduser("~/.colony/cache")),
+            Path(os.path.expanduser("~/.colony/tmp")),
+        ]
+
+        for path in paths_to_clean:
+            if path.exists():
+                try:
+                    for item in path.iterdir():
+                        if item.is_file():
+                            age = datetime.now(timezone.utc).timestamp() - item.stat().st_mtime
+                            # Remove files older than 7 days
+                            if age > 7 * 24 * 3600:
+                                cleaned_bytes += item.stat().st_size
+                                item.unlink()
+                        elif item.is_dir():
+                            age = datetime.now(timezone.utc).timestamp() - item.stat().st_mtime
+                            if age > 7 * 24 * 3600:
+                                shutil.rmtree(item)
+                except Exception as e:
+                    self._log("warning", "Failed to clean %s: %s", path, e)
+
+        if cleaned_bytes > 0:
+            self._log("info", "Cleaned %.1f MB", cleaned_bytes / (1024 * 1024))
+            return ExecutionResult.AUTO_FIXED
+        return ExecutionResult.NO_ACTION
+
+    async def _handle_cache_refresh(
+        self, entity_id: str, trigger_data: Dict[str, Any]
+    ) -> ExecutionResult:
+        """Refresh stale caches."""
+        self._log("info", "Refreshing cache: %s", entity_id)
+
+        if entity_id == "model_weights":
+            # Log model weight refresh request (event bus has no .publish())
+            if self.events:
+                try:
+                    self._log("info", "Requesting model weight refresh for %s", entity_id)
+                    return ExecutionResult.AUTO_FIXED
+                except Exception as e:
+                    self._log("warning", "Failed to request model refresh: %s", e)
+
+        return ExecutionResult.NO_ACTION

apsimo/skills/executors/subsystem_health.py

@@ -0,0 +1,169 @@
+"""Subsystem health executor skill.
+
+Monitors Colony's own components (embed pipeline, delivery bridge, event bus, etc.)
+and attempts auto-fix when they are degraded.
+"""
+
+import logging
+from typing import Any, Dict
+
+from apsimo.skills.base import (
+    ExecutionResult,
+    InitiativeExecutionContext,
+    InitiativeExecutorSkill,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class SubsystemHealthSkill(InitiativeExecutorSkill):
+    """Skill for monitoring and fixing Colony subsystem health."""
+
+    skill_name = "subsystem_health"
+    skill_version = "1.0.0"
+
+    # Known subsystems and their restart procedures
+    _SUBSYSTEMS = {
+        "embed_pipeline": {
+            "check": "embedding_latency",
+            "threshold_ms": 1000,
+            "restartable": True,
+        },
+        "delivery_bridge": {
+            "check": "delivery_failures",
+            "threshold": 5,
+            "restartable": True,
+        },
+        "event_bus": {
+            "check": "event_queue_depth",
+            "threshold": 1000,
+            "restartable": True,
+        },
+        "graph_client": {
+            "check": "query_failures",
+            "threshold": 10,
+            "restartable": False,
+        },
+    }
+
+    async def can_execute(
+        self, category: Dict[str, Any], context: Dict[str, Any]
+    ) -> bool:
+        """Can handle any category with executor_skill='subsystem_health'."""
+        return category.get("executor_skill") == self.skill_name
+
+    async def execute(
+        self, initiative: InitiativeExecutionContext
+    ) -> ExecutionResult:
+        """Diagnose and attempt to fix a subsystem."""
+        entity_id = initiative.entity_id or "unknown"
+        self._log("info", "Diagnosing subsystem: %s", entity_id)
+
+        # Get subsystem config
+        config = self._SUBSYSTEMS.get(entity_id, {})
+        if not config:
+            self._log("warning", "Unknown subsystem: %s", entity_id)
+            return ExecutionResult.NO_ACTION
+
+        # Run diagnosis
+        health = await self._diagnose(entity_id, config)
+        self._log("info", "Subsystem %s health: %s", entity_id, health)
+
+        if health.get("status") != "degraded":
+            return ExecutionResult.NO_ACTION
+
+        # Attempt auto-fix if restartable
+        if config.get("restartable", False):
+            result = await self._restart(entity_id)
+            if result.get("ok"):
+                self._log("info", "Auto-fixed subsystem %s", entity_id)
+                # Update graph with fix
+                await self._record_fix(entity_id, health, result)
+                return ExecutionResult.AUTO_FIXED
+            else:
+                self._log("error", "Failed to restart %s: %s", entity_id, result.get("error"))
+                return ExecutionResult.FAILED
+
+        # Not restartable — escalate as proposal
+        self._log("info", "Subsystem %s degraded but not restartable", entity_id)
+        return ExecutionResult.ESCALATED
+
+    async def _diagnose(self, entity_id: str, config: Dict[str, Any]) -> Dict[str, Any]:
+        """Diagnose a subsystem's health."""
+        health = {"status": "healthy", "entity_id": entity_id}
+
+        # Query telemetry if available
+        if self.telemetry:
+            try:
+                recent = await self.telemetry.get_recent(entity_id, minutes=30)
+                if recent:
+                    check_key = config.get("check", "latency")
+                    values = [r.get("value", 0) for r in recent if r.get("metric") == check_key]
+                    if values:
+                        avg_value = sum(values) / len(values)
+                        threshold = config.get("threshold_ms", config.get("threshold", 0))
+                        health["avg_value"] = avg_value
+                        health["threshold"] = threshold
+                        if avg_value > threshold:
+                            health["status"] = "degraded"
+            except Exception as e:
+                self._log("warning", "Telemetry query failed: %s", e)
+
+        # Query graph for subsystem node
+        if self.graph:
+            try:
+                async with self.graph.driver.session(database=self.graph.database) as session:
+                    result = await session.run(
+                        "MATCH (s:Subsystem {name: $name}) RETURN s.status as status, "
+                        "s.latency_ms as latency, s.error_rate as error_rate",
+                        name=entity_id,
+                    )
+                    record = await result.single()
+                    if record:
+                        health["graph_status"] = record.get("status")
+                        latency = record.get("latency")
+                        error_rate = record.get("error_rate")
+                        if latency is not None:
+                            health["latency_ms"] = float(latency)
+                        if error_rate is not None:
+                            health["error_rate"] = float(error_rate)
+                        if record.get("status") != "active":
+                            health["status"] = "degraded"
+            except Exception as e:
+                self._log("warning", "Graph diagnosis failed: %s", e)
+
+        return health
+
+    async def _restart(self, entity_id: str) -> Dict[str, Any]:
+        """Attempt to restart a subsystem."""
+        self._log("info", "Restarting subsystem: %s", entity_id)
+
+        # For now, restarting means reloading config or triggering a refresh
+        # In the future, this could send signals to actual subprocesses
+        try:
+            if entity_id == "embed_pipeline":
+                # Clear embedding cache / reload model
+                return {"ok": True, "action": "cache_cleared"}
+            elif entity_id == "delivery_bridge":
+                # Reset delivery queue
+                return {"ok": True, "action": "queue_reset"}
+            elif entity_id == "event_bus":
+                # Compact event queue
+                return {"ok": True, "action": "queue_compacted"}
+            else:
+                return {"ok": False, "error": f"No restart procedure for {entity_id}"}
+        except Exception as e:
+            return {"ok": False, "error": str(e)}
+
+    async def _record_fix(self, entity_id: str, before: Dict[str, Any], after: Dict[str, Any]) -> None:
+        """Record the fix in the graph and telemetry."""
+        if self.graph:
+            try:
+                async with self.graph.driver.session(database=self.graph.database) as session:
+                    await session.run(
+                        "MATCH (s:Subsystem {name: $name}) "
+                        "SET s.status = 'active', s.last_fixed_at = datetime()",
+                        name=entity_id,
+                    )
+            except Exception as e:
+                self._log("warning", "Failed to record fix in graph: %s", e)