apsimo 1.3.0__py3-none-any.whl

apsimo/skills/executor.py

@@ -0,0 +1,493 @@
+"""Colony Skills — skill executor with capability-gated sandbox."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import hmac
+import importlib
+import inspect
+import json
+import logging
+import os
+import pathlib
+import sys
+import types
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from apsimo.skills.base import (
+    InitiativeExecutionContext,
+    InitiativeExecutorSkill,
+)
+from apsimo.skills.models import SkillManifest, SkillStatus
+from apsimo.skills.registry import SkillRegistry
+from apsimo.skills.security.scanner import ASTScanner
+from apsimo.skills.security.guards import CapabilityGuard
+
+logger = logging.getLogger(__name__)
+
+
+_DEFAULT_SANDBOX_MODE = "subprocess" if sys.platform.startswith("linux") else "inprocess"
+
+
+def _resolve_sandbox_mode() -> str:
+    """Return the configured sandbox mode, falling back to the platform default."""
+    override = (os.environ.get("COLONY_SKILL_SANDBOX") or "").strip().lower()
+    if override in ("subprocess", "inprocess"):
+        if override == "inprocess" and _DEFAULT_SANDBOX_MODE == "subprocess":
+            logger.warning(
+                "COLONY_SKILL_SANDBOX=inprocess overrides the default subprocess "
+                "isolation — skills will run in the sidecar process."
+            )
+        return override
+    return _DEFAULT_SANDBOX_MODE
+
+
+class SecurityError(RuntimeError):
+    """Raised when a skill fails integrity or security checks."""
+
+
+@dataclass
+class ExecutionResult:
+    """Result of a skill invocation."""
+    execution_id: str
+    skill_id: str
+    status: str          # "success" | "failed" | "violated" | "timeout"
+    output: Any
+    error: Optional[str]
+    duration_ms: int
+    peak_memory_mb: Optional[float]
+    violations: List[str] = field(default_factory=list)
+
+
+class SkillExecutor:
+    """Executes Colony skills inside a capability-gated sandbox.
+
+    Execution pipeline:
+      1. Load manifest and verify skill status.
+      2. Run capability guard checks.
+      3. Verify on-disk source checksum.
+      4. Execute either in a hardened subprocess (default on Linux) or
+         in-process (rollback path, controlled by COLONY_SKILL_SANDBOX).
+      5. Log result to execution log; quarantine on timeout.
+    """
+
+    def __init__(
+        self,
+        registry: SkillRegistry,
+        guard: CapabilityGuard,
+        scanner: ASTScanner,
+        execution_timeout_secs: float = 60.0,
+        sandbox_mode: Optional[str] = None,
+    ) -> None:
+        self._registry = registry
+        self._guard = guard
+        self._scanner = scanner
+        self._timeout = execution_timeout_secs
+        self._sandbox_mode = (sandbox_mode or _resolve_sandbox_mode()).lower()
+
+    async def invoke(
+        self,
+        skill_id: str,
+        inputs: Dict[str, Any],
+        caller_context: Optional[str] = None,
+    ) -> ExecutionResult:
+        """Invoke a skill by ID with the given inputs."""
+        execution_id = f"exec-{uuid.uuid4().hex[:12]}"
+        start = datetime.now(timezone.utc)
+
+        # The wired registry may be the async manifest store OR the built-in
+        # runtime SkillRegistry, whose get() is synchronous and returns the
+        # executor skill object itself. Awaiting that object was the bug that
+        # made every built-in skill 500 ("object ... can't be used in 'await'
+        # expression") while the registry still advertised them as active.
+        manifest = self._registry.get(skill_id)
+        if inspect.isawaitable(manifest):
+            manifest = await manifest
+        if not manifest:
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="failed",
+                output=None,
+                error=f"Skill '{skill_id}' not found in registry.",
+                duration_ms=0,
+                peak_memory_mb=None,
+            )
+
+        if isinstance(manifest, InitiativeExecutorSkill):
+            # Built-in runtime executor: trusted in-process code with no
+            # on-disk manifest/source — run its own execute() directly.
+            return await self._invoke_builtin(
+                skill_id, manifest, inputs, execution_id, start,
+            )
+
+        if manifest.status != SkillStatus.ACTIVE:
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="failed",
+                output=None,
+                error=f"Skill '{skill_id}' is not active (status: {manifest.status.value}).",
+                duration_ms=0,
+                peak_memory_mb=None,
+            )
+
+        guard_result = await self._guard.check(manifest, inputs)
+        if not guard_result.allowed:
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="failed",
+                output=None,
+                error=f"Capability guard denied execution: {guard_result.reason}",
+                duration_ms=0,
+                peak_memory_mb=None,
+                violations=guard_result.violations,
+            )
+
+        skill_dir_str = manifest.skill_dir or ""
+        skill_dir = pathlib.Path(skill_dir_str)
+        skill_path = skill_dir / "skill.py"
+        if not skill_path.exists():
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="failed",
+                output=None,
+                error=f"Skill '{skill_id}' source file not found.",
+                duration_ms=0,
+                peak_memory_mb=None,
+            )
+
+        try:
+            verified_source = self._read_verified_source(skill_path, manifest)
+            if self._sandbox_mode == "subprocess":
+                raw = await asyncio.wait_for(
+                    self._run_subprocess(verified_source, inputs, manifest),
+                    timeout=manifest.permissions.max_duration_secs,
+                )
+                end = datetime.now(timezone.utc)
+                duration_ms = int((end - start).total_seconds() * 1000)
+                return await self._finalize_subprocess_result(
+                    skill_id=skill_id,
+                    execution_id=execution_id,
+                    raw=raw,
+                    duration_ms=duration_ms,
+                )
+
+            # inprocess rollback path — kept verbatim so operators can flip
+            # back to the pre-sandbox behaviour without redeploying.
+            output = await asyncio.wait_for(
+                self._run_skill_inprocess(verified_source, inputs),
+                timeout=manifest.permissions.max_duration_secs,
+            )
+            end = datetime.now(timezone.utc)
+            duration_ms = int((end - start).total_seconds() * 1000)
+            await self._registry.record_execution(skill_id, execution_id, "success", duration_ms)
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="success",
+                output=output,
+                error=None,
+                duration_ms=duration_ms,
+                peak_memory_mb=None,
+            )
+        except asyncio.TimeoutError:
+            await self._registry.quarantine(skill_id, f"Execution timeout in {execution_id}")
+            end = datetime.now(timezone.utc)
+            duration_ms = int((end - start).total_seconds() * 1000)
+            await self._registry.record_execution(
+                skill_id, execution_id, "timeout", duration_ms, ["timeout"]
+            )
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="timeout",
+                output=None,
+                error="Execution exceeded maximum duration.",
+                duration_ms=duration_ms,
+                peak_memory_mb=None,
+                violations=["timeout"],
+            )
+        except Exception as exc:
+            end = datetime.now(timezone.utc)
+            duration_ms = int((end - start).total_seconds() * 1000)
+            await self._registry.record_execution(
+                skill_id, execution_id, "failed", duration_ms
+            )
+            logger.debug("Skill '%s' execution error (%s)", skill_id, exc, exc_info=True)
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="failed",
+                output=None,
+                error=f"Skill '{skill_id}' execution failed: {type(exc).__name__}",
+                duration_ms=duration_ms,
+                peak_memory_mb=None,
+            )
+
+    # ------------------------------------------------------------------
+    # Built-in runtime executor skills
+    # ------------------------------------------------------------------
+
+    async def _invoke_builtin(
+        self,
+        skill_id: str,
+        skill: InitiativeExecutorSkill,
+        inputs: Dict[str, Any],
+        execution_id: str,
+        start: datetime,
+    ) -> ExecutionResult:
+        """Execute a built-in InitiativeExecutorSkill registered in the
+        runtime SkillRegistry.
+
+        These are trusted in-process objects (no manifest, checksum, or
+        sandbox source file); the InitiativeExecutionContext is built from
+        the caller's inputs. The skill's own outcome enum maps onto the
+        ExecutionResult status so a FAILED initiative is never reported as
+        a successful execution.
+        """
+        def _elapsed_ms() -> int:
+            return int(
+                (datetime.now(timezone.utc) - start).total_seconds() * 1000
+            )
+
+        try:
+            context = InitiativeExecutionContext(
+                initiative_id=str(inputs.get("initiative_id") or execution_id),
+                category_id=str(inputs.get("category_id") or "api"),
+                category_name=str(
+                    inputs.get("category_name")
+                    or getattr(skill, "skill_name", skill_id)
+                ),
+                entity_id=inputs.get("entity_id"),
+                entity_type=inputs.get("entity_type"),
+                trigger_data=dict(inputs.get("trigger_data") or {}),
+            )
+            outcome = await asyncio.wait_for(
+                skill.execute(context), timeout=self._timeout,
+            )
+        except asyncio.TimeoutError:
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="timeout",
+                output=None,
+                error="Execution exceeded maximum duration.",
+                duration_ms=_elapsed_ms(),
+                peak_memory_mb=None,
+                violations=["timeout"],
+            )
+        except Exception as exc:
+            logger.debug(
+                "Built-in skill '%s' execution error (%s)",
+                skill_id, exc, exc_info=True,
+            )
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="failed",
+                output=None,
+                error=f"Skill '{skill_id}' execution failed: {type(exc).__name__}",
+                duration_ms=_elapsed_ms(),
+                peak_memory_mb=None,
+            )
+
+        outcome_value = getattr(outcome, "value", outcome)
+        failed = str(outcome_value) == "failed"
+        return ExecutionResult(
+            execution_id=execution_id,
+            skill_id=skill_id,
+            status="failed" if failed else "success",
+            output={"result": outcome_value},
+            error="skill reported failure" if failed else None,
+            duration_ms=_elapsed_ms(),
+            peak_memory_mb=None,
+        )
+
+    # ------------------------------------------------------------------
+    # Integrity
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _read_verified_source(
+        skill_path: pathlib.Path,
+        manifest: SkillManifest,
+    ) -> bytes:
+        """Read + checksum-verify the skill source in one shot.
+
+        Returns the in-memory bytes on success; raises SecurityError on
+        missing or mismatched checksum. Reading once and returning bytes
+        eliminates the TOCTOU window between verification and execution —
+        both the in-process and subprocess paths operate on the returned
+        buffer, never touching the filesystem again.
+        """
+        if not manifest.checksum_sha256:
+            raise SecurityError(
+                "Skill has no checksum — integrity cannot be verified."
+            )
+        file_bytes = skill_path.read_bytes()
+        actual_digest = hashlib.sha256(file_bytes).hexdigest()
+        if not hmac.compare_digest(actual_digest, manifest.checksum_sha256):
+            raise SecurityError(
+                "Skill checksum mismatch — file may have been tampered with."
+            )
+        return file_bytes
+
+    # ------------------------------------------------------------------
+    # Subprocess path
+    # ------------------------------------------------------------------
+
+    async def _run_subprocess(
+        self,
+        source_bytes: bytes,
+        inputs: Dict[str, Any],
+        manifest: SkillManifest,
+    ) -> Dict[str, Any]:
+        """Run the skill in a hardened subprocess via sandbox_runner.
+
+        Returns the parsed JSON payload from the runner's stdout, or a
+        synthesized failure dict on malformed output / nonzero exit.
+        """
+        payload = json.dumps({
+            "source": source_bytes.decode("utf-8", errors="replace"),
+            "inputs": inputs,
+            "allowed_imports": manifest.permissions.allowed_imports if hasattr(manifest.permissions, 'allowed_imports') else [],
+            "limits": {
+                "mem_mb": manifest.permissions.max_memory_mb,
+                "cpu_secs": manifest.permissions.max_duration_secs,
+                "fsize_mb": 8,
+            },
+        }).encode("utf-8")
+
+        proc = await asyncio.create_subprocess_exec(
+            sys.executable,
+            "-m",
+            "apsimo.skills.sandbox_runner",
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+
+        try:
+            stdout_bytes, stderr_bytes = await proc.communicate(input=payload)
+        except asyncio.CancelledError:
+            # Parent timeout — kill the child and re-raise so asyncio.wait_for
+            # sees TimeoutError in the caller.
+            try:
+                proc.kill()
+            except ProcessLookupError:
+                pass
+            await proc.wait()
+            raise
+
+        stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
+        stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
+        returncode = proc.returncode or 0
+
+        if not stdout:
+            # The runner always emits exactly one JSON line on success or
+            # handled failure. An empty stdout means the kernel killed
+            # the process before it could report (RLIMIT_AS, RLIMIT_CPU,
+            # segfault, OOM killer). Surface stderr excerpt for operators.
+            return {
+                "status": "failed",
+                "error": f"runner_killed (rc={returncode}): {stderr[-200:]}",
+                "peak_memory_kb": 0,
+            }
+
+        try:
+            parsed = json.loads(stdout.splitlines()[-1])
+        except json.JSONDecodeError:
+            return {
+                "status": "failed",
+                "error": f"runner_malformed_output: {stdout[-200:]}",
+                "peak_memory_kb": 0,
+            }
+
+        if not isinstance(parsed, dict) or "status" not in parsed:
+            return {
+                "status": "failed",
+                "error": "runner returned an unexpected payload shape",
+                "peak_memory_kb": 0,
+            }
+
+        # Defense in depth: runner claimed success but exited nonzero.
+        if parsed.get("status") == "success" and returncode != 0:
+            parsed = {
+                "status": "failed",
+                "error": f"runner exited nonzero ({returncode}) despite success payload",
+                "peak_memory_kb": parsed.get("peak_memory_kb", 0),
+            }
+
+        return parsed
+
+    async def _finalize_subprocess_result(
+        self,
+        skill_id: str,
+        execution_id: str,
+        raw: Dict[str, Any],
+        duration_ms: int,
+    ) -> ExecutionResult:
+        """Map the runner's JSON payload onto ExecutionResult + persist log."""
+        status = raw.get("status", "failed")
+        peak_kb = int(raw.get("peak_memory_kb") or 0)
+        peak_mb: Optional[float] = (peak_kb / 1024.0) if peak_kb else None
+
+        if status == "success":
+            await self._registry.record_execution(
+                skill_id, execution_id, "success", duration_ms
+            )
+            return ExecutionResult(
+                execution_id=execution_id,
+                skill_id=skill_id,
+                status="success",
+                output=raw.get("output"),
+                error=None,
+                duration_ms=duration_ms,
+                peak_memory_mb=peak_mb,
+            )
+
+        error = raw.get("error") or "skill failed with no error message"
+        await self._registry.record_execution(
+            skill_id, execution_id, "failed", duration_ms
+        )
+        return ExecutionResult(
+            execution_id=execution_id,
+            skill_id=skill_id,
+            status="failed",
+            output=None,
+            error=str(error),
+            duration_ms=duration_ms,
+            peak_memory_mb=peak_mb,
+        )
+
+    # ------------------------------------------------------------------
+    # In-process rollback path
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    async def _run_skill_inprocess(
+        source_bytes: bytes,
+        inputs: Dict[str, Any],
+    ) -> Any:
+        """Legacy in-process execution. Used only when COLONY_SKILL_SANDBOX=inprocess."""
+        code = compile(source_bytes, "<skill>", "exec")
+        module = types.ModuleType("_skill_module")
+        exec(code, module.__dict__)  # noqa: S102  # in-memory, checksum verified
+        run_fn = getattr(module, "run", None)
+        if run_fn is None:
+            raise AttributeError("Skill has no 'run' function.")
+        # Inject colony runtime if the skill signature expects it
+        import inspect
+        sig = inspect.signature(run_fn)
+        if "colony" in sig.parameters:
+            from apsimo.skills.runtime import ColonyRuntime
+            colony_rt = ColonyRuntime(base_url=os.environ.get("COLONY_URL", "http://127.0.0.1:7777"))
+            inputs = {"colony": colony_rt, **inputs}
+        return await run_fn(**inputs)

apsimo/skills/executors/__init__.py

@@ -0,0 +1 @@
+# Built-in executor skills

apsimo/skills/executors/behavioral_correction.py

@@ -0,0 +1,75 @@
+"""Behavioral correction executor skill.
+
+Applies learned preferences when recurring patterns are detected.
+"""
+
+import logging
+from typing import Any, Dict
+
+from apsimo.skills.base import (
+    ExecutionResult,
+    InitiativeExecutionContext,
+    InitiativeExecutorSkill,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class BehavioralCorrectionSkill(InitiativeExecutorSkill):
+    """Skill for applying learned behavioral corrections."""
+
+    skill_name = "behavioral_correction"
+    skill_version = "1.0.0"
+
+    async def can_execute(
+        self, category: Dict[str, Any], context: Dict[str, Any]
+    ) -> bool:
+        return category.get("executor_skill") == self.skill_name
+
+    async def execute(
+        self, initiative: InitiativeExecutionContext
+    ) -> ExecutionResult:
+        entity_id = initiative.entity_id or "unknown"
+        trigger = initiative.trigger_data.get("trigger", "")
+        expected_action = initiative.trigger_data.get("action", "")
+        recurrence_count = initiative.trigger_data.get("recurrence_count", 0)
+        self._log(
+            "info",
+            "Behavioral correction: %s (recurred %d times)",
+            trigger[:80],
+            recurrence_count,
+        )
+
+        # Store preference in graph for future reference
+        if self.graph:
+            try:
+                await self._store_preference(trigger, expected_action, entity_id)
+            except Exception as e:
+                self._log("warning", "Failed to store preference: %s", e)
+
+        # For v0.11.1, create a proposal. Full auto-application requires
+        # deeper integration with the reasoning pipeline.
+        return ExecutionResult.PROPOSAL_CREATED
+
+    async def _store_preference(
+        self, trigger: str, expected: str, entity_id: str
+    ) -> None:
+        """Store a learned preference in the graph."""
+        if not hasattr(self.graph, "driver"):
+            return
+
+        import uuid
+
+        async with self.graph.driver.session(database=self.graph.database) as session:
+            await session.run(
+                """
+                MERGE (p:Preference {trigger: $trigger})
+                SET p.id = coalesce(p.id, $new_id),
+                    p.expected = $expected,
+                    p.source = "behavioral_correction",
+                    p.updated_at = datetime()
+            """,
+                trigger=trigger[:200],
+                expected=expected[:200],
+                new_id=str(uuid.uuid4()),
+            )

apsimo/skills/executors/capability_gap.py

@@ -0,0 +1,38 @@
+"""Capability gap executor skill.
+
+Registers missing capabilities and creates proposals for fixes.
+"""
+
+import logging
+from typing import Any, Dict
+
+from apsimo.skills.base import (
+    ExecutionResult,
+    InitiativeExecutionContext,
+    InitiativeExecutorSkill,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class CapabilityGapSkill(InitiativeExecutorSkill):
+    """Skill for handling missing or broken capabilities."""
+
+    skill_name = "capability_gap"
+    skill_version = "1.0.0"
+
+    async def can_execute(
+        self, category: Dict[str, Any], context: Dict[str, Any]
+    ) -> bool:
+        return category.get("executor_skill") == self.skill_name
+
+    async def execute(
+        self, initiative: InitiativeExecutionContext
+    ) -> ExecutionResult:
+        entity_id = initiative.entity_id or "unknown"
+        failure_mode = initiative.trigger_data.get("failure_mode", "unknown")
+        failure_count = initiative.trigger_data.get("failure_count", 0)
+        self._log("info", "Capability gap: %s (%s, %d failures)", entity_id, failure_mode, failure_count)
+
+        # For v0.11.1, create a proposal. Auto-fix requires tool sandboxing.
+        return ExecutionResult.PROPOSAL_CREATED