apsimo 1.3.0__py3-none-any.whl

apsimo/projects/event_outbox.py

@@ -0,0 +1,188 @@
+"""Crash-recoverable ProjectStore to host-journal projection.
+
+The project ledger and host event journal are separate durable stores.  A
+ProjectStore transaction therefore stages immutable event data in its own
+outbox first.  This projector uses the journal's keyed append handshake, then
+records the returned sequence in the project database.  Replaying after a
+crash can finish either side without creating a second host event.
+"""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any, Callable, Dict, Mapping, Optional
+
+from apsimo.events.journal import event_record_request_digest
+from apsimo.projects.store import ProjectStore
+
+
+_RECEIPT_ACKNOWLEDGEMENT_FIELDS = frozenset({
+    "expected_seq",
+    "expected_event_id",
+    "expected_recorded_at",
+    "expected_request_digest",
+})
+
+
+class ProjectEventProjector:
+    """Drain immutable project events into the canonical host journal."""
+
+    def __init__(
+        self,
+        store: ProjectStore,
+        *,
+        journal_projector: Optional[Callable[..., Optional[Mapping[str, Any]]]] = None,
+        journal_acknowledger: Optional[Callable[..., bool]] = None,
+    ) -> None:
+        self.store = store
+        if journal_projector is None:
+            from apsimo.events.journal import append_event_record
+
+            journal_projector = append_event_record
+        if journal_acknowledger is None:
+            from apsimo.events.journal import acknowledge_event_record
+
+            journal_acknowledger = acknowledge_event_record
+        self._project = journal_projector
+        self._acknowledge = journal_acknowledger
+        self._acknowledge_with_receipt = self._supports_receipt_acknowledgement(
+            journal_acknowledger
+        )
+        if not self._acknowledge_with_receipt:
+            raise TypeError(
+                "project journal acknowledger must accept the exact projection "
+                "receipt fields"
+            )
+
+    @staticmethod
+    def _supports_receipt_acknowledgement(callback: Callable[..., bool]) -> bool:
+        """Require an acknowledger that validates the exact journal receipt."""
+
+        try:
+            parameters = inspect.signature(callback).parameters
+        except (TypeError, ValueError):
+            return False
+        if any(
+            parameter.kind is inspect.Parameter.VAR_KEYWORD
+            for parameter in parameters.values()
+        ):
+            return True
+        return _RECEIPT_ACKNOWLEDGEMENT_FIELDS.issubset(parameters)
+
+    def _validated_payload(self, row: Mapping[str, Any]) -> Dict[str, Any]:
+        payload_error = str(row.get("payload_error") or "")
+        payload = row.get("payload")
+        if payload_error:
+            raise ValueError(payload_error)
+        if not isinstance(payload, dict):
+            raise ValueError("project outbox payload is not an object")
+        expected = self.store.project_event_envelope_digest(
+            event_key=str(row.get("event_key") or ""),
+            event_type=str(row.get("event_type") or ""),
+            occurred_at=str(row.get("occurred_at") or ""),
+            payload=payload,
+        )
+        if expected != str(row.get("event_digest") or ""):
+            raise ValueError("project outbox envelope digest mismatch")
+        return payload
+
+    def _validated_projection_receipt(
+        self,
+        row: Mapping[str, Any],
+        payload: Mapping[str, Any],
+    ) -> Dict[str, Any]:
+        if row.get("state") != "projected":
+            raise ValueError("project outbox event is not projected")
+        expected = self.store.project_event_projection_receipt_digest(
+            event_key=str(row.get("event_key") or ""),
+            event_type=str(row.get("event_type") or ""),
+            event_digest=str(row.get("event_digest") or ""),
+            occurred_at=str(row.get("occurred_at") or ""),
+            journal_seq=row.get("journal_seq"),
+            journal_event_id=str(row.get("journal_event_id") or ""),
+            journal_recorded_at=str(row.get("journal_recorded_at") or ""),
+        )
+        if str(row.get("projection_receipt_digest") or "") != expected:
+            raise ValueError("project outbox projection receipt digest mismatch")
+        return {
+            "expected_seq": row["journal_seq"],
+            "expected_event_id": str(row["journal_event_id"]),
+            "expected_recorded_at": str(row["journal_recorded_at"]),
+            "expected_request_digest": event_record_request_digest(
+                str(row["event_type"]),
+                dict(payload),
+                str(row["occurred_at"]),
+            ),
+        }
+
+    def _acknowledge_row(self, row: Mapping[str, Any]) -> None:
+        key = str(row.get("event_key") or "")
+        payload = self._validated_payload(row)
+        receipt = self._validated_projection_receipt(row, payload)
+        acknowledged = self._acknowledge(key, **receipt)
+        if not acknowledged:
+            raise RuntimeError("journal event-key acknowledgement failed")
+        self.store.acknowledge_project_event(key)
+
+    def _cleanup_acknowledgements(self, *, limit: int) -> tuple[int, int]:
+        cleaned = 0
+        failed = 0
+        for row in self.store.unacknowledged_project_events(limit=limit):
+            key = str(row["event_key"])
+            try:
+                self._acknowledge_row(row)
+                cleaned += 1
+            except Exception as exc:
+                self.store.fail_project_event(key, str(exc))
+                failed += 1
+        return cleaned, failed
+
+    def run_once(self, *, limit: int = 100) -> Dict[str, Any]:
+        bound = max(1, min(500, int(limit)))
+        acknowledged, acknowledgement_failures = self._cleanup_acknowledgements(
+            limit=bound,
+        )
+        projected = 0
+        failed = 0
+        retained = 0
+        for row in self.store.pending_project_events(limit=bound):
+            key = str(row["event_key"])
+            try:
+                payload = self._validated_payload(row)
+                record = self._project(
+                    str(row["event_type"]),
+                    payload,
+                    occurred_at=str(row["occurred_at"]),
+                    event_key=key,
+                )
+                if not isinstance(record, Mapping):
+                    raise RuntimeError("project event journal projection failed")
+                completed = self.store.complete_project_event(key, dict(record))
+                projected += 1
+                retained += int(bool(record.get("retained", True)))
+            except Exception as exc:
+                self.store.fail_project_event(key, str(exc))
+                failed += 1
+                continue
+            try:
+                self._acknowledge_row(completed)
+                acknowledged += 1
+            except Exception as exc:
+                self.store.fail_project_event(key, str(exc))
+                acknowledgement_failures += 1
+        status = self.store.project_event_outbox_status()
+        return {
+            "processed": projected + failed,
+            "projected": projected,
+            "failed": failed,
+            "retained": retained,
+            "acknowledged": acknowledged,
+            "acknowledgement_failures": acknowledgement_failures,
+            "outbox": status,
+        }
+
+    def status(self) -> Dict[str, Any]:
+        return self.store.project_event_outbox_status()
+
+
+__all__ = ["ProjectEventProjector"]

apsimo/projects/models.py

@@ -0,0 +1,216 @@
+"""Project + Step data model."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+# Fixed step-action vocabulary. Each kind routes through an existing gated
+# sub-path; anything outside this set is dropped at validation.
+#   analyze / research / internal -> sidecar reasoning turn (internal tools)
+#   directed                      -> DirectedActionService (dry_run/approval)
+#   deliver                       -> guarded proposal delivery
+ACTION_KINDS = frozenset({"analyze", "research", "directed", "deliver", "internal"})
+
+PROJECT_STATUSES = ("planning", "active", "blocked", "completed", "abandoned")
+STEP_STATUSES = ("pending", "active", "done", "failed", "skipped")
+
+
+def projects_mode() -> str:
+    from apsimo.util.autonomy_preset import resolve
+    return resolve("COLONY_PROJECTS_MODE", ("off", "shadow", "live"), "shadow")
+
+
+def projects_max_steps() -> int:
+    try:
+        return max(1, min(50, int(os.environ.get("COLONY_PROJECTS_MAX_STEPS", "12"))))
+    except (TypeError, ValueError):
+        return 12
+
+
+def projects_max_replans() -> int:
+    try:
+        return max(0, int(os.environ.get("COLONY_PROJECTS_MAX_REPLANS", "3")))
+    except (TypeError, ValueError):
+        return 3
+
+
+def projects_review_secs() -> float:
+    try:
+        return max(30.0, float(os.environ.get("COLONY_PROJECTS_REVIEW_SECS", "900")))
+    except (TypeError, ValueError):
+        return 900.0
+
+
+@dataclass
+class Project:
+    title: str
+    objective: str = ""
+    source: str = "owner"               # owner | thinker | directive
+    status: str = "planning"
+    outcome: str = "pending"             # lifecycle status != evidence outcome
+    entity_ids: List[str] = field(default_factory=list)
+    reason: str = ""                    # abandon/complete/blocked reason
+    replans: int = 0
+    next_review_at: float = 0.0         # 0 -> due immediately
+    # P3 autonomous-goal provenance.  Legacy/owner projects leave these
+    # empty; cognition-spine projects carry every upstream ID and scope into
+    # WorkOrder.context_refs without copying private context text.
+    concern_id: str = ""
+    source_event_refs: List[str] = field(default_factory=list)
+    thought_job_id: str = ""
+    thought_result_ref: str = ""
+    goal_proposal_id: str = ""
+    evidence_refs: List[str] = field(default_factory=list)
+    policy_decision_refs: List[str] = field(default_factory=list)
+    subject_person_id: str = ""
+    viewer_scope: str = "owner"
+    shareability: str = "owner_private"
+    capability_allowlist: List[str] = field(default_factory=list)
+    goal_fingerprint: str = ""
+    id: str = field(default_factory=lambda: f"proj-{uuid.uuid4().hex[:12]}")
+    created_at: float = field(default_factory=time.time)
+    updated_at: float = field(default_factory=time.time)
+
+    def subject_text(self) -> str:
+        """Boundary-checkable subject for this project."""
+        return f"{self.title} {self.objective}"
+
+    def provenance_refs(self) -> List[str]:
+        """Return bounded reference-only lineage for a WorkOrder."""
+
+        refs: List[str] = []
+        if self.concern_id:
+            refs.append(f"concern:{self.concern_id}")
+        refs.extend(self.source_event_refs)
+        if self.thought_job_id:
+            refs.append(f"thought-job:{self.thought_job_id}")
+        if self.thought_result_ref:
+            refs.append(self.thought_result_ref)
+        if self.goal_proposal_id:
+            refs.append(self.goal_proposal_id)
+        refs.extend(self.evidence_refs)
+        refs.extend(self.policy_decision_refs)
+        return list(dict.fromkeys(ref for ref in refs if ref))[:60]
+
+    def to_row(self) -> Dict[str, Any]:
+        return {
+            "id": self.id, "title": self.title, "objective": self.objective,
+            "source": self.source, "status": self.status,
+            "outcome": self.outcome,
+            "entity_ids": json.dumps(self.entity_ids), "reason": self.reason,
+            "replans": self.replans, "next_review_at": self.next_review_at,
+            "concern_id": self.concern_id,
+            "source_event_refs": json.dumps(self.source_event_refs),
+            "thought_job_id": self.thought_job_id,
+            "thought_result_ref": self.thought_result_ref,
+            "goal_proposal_id": self.goal_proposal_id,
+            "evidence_refs": json.dumps(self.evidence_refs),
+            "policy_decision_refs": json.dumps(self.policy_decision_refs),
+            "subject_person_id": self.subject_person_id,
+            "viewer_scope": self.viewer_scope,
+            "shareability": self.shareability,
+            "capability_allowlist": json.dumps(self.capability_allowlist),
+            "goal_fingerprint": self.goal_fingerprint,
+            "created_at": self.created_at, "updated_at": self.updated_at,
+        }
+
+    @classmethod
+    def from_row(cls, r: Dict[str, Any]) -> "Project":
+        def _list(name: str) -> List[str]:
+            try:
+                value = json.loads(r.get(name) or "[]")
+                return [str(item) for item in value] if isinstance(value, list) else []
+            except Exception:
+                return []
+
+        return cls(
+            id=r["id"], title=r.get("title", "") or "",
+            objective=r.get("objective", "") or "",
+            source=r.get("source", "owner") or "owner",
+            status=r.get("status", "planning") or "planning",
+            outcome=r.get("outcome", "pending") or "pending",
+            entity_ids=_list("entity_ids"),
+            reason=r.get("reason", "") or "",
+            replans=int(r.get("replans") or 0),
+            next_review_at=float(r.get("next_review_at") or 0.0),
+            concern_id=r.get("concern_id", "") or "",
+            source_event_refs=_list("source_event_refs"),
+            thought_job_id=r.get("thought_job_id", "") or "",
+            thought_result_ref=r.get("thought_result_ref", "") or "",
+            goal_proposal_id=r.get("goal_proposal_id", "") or "",
+            evidence_refs=_list("evidence_refs"),
+            policy_decision_refs=_list("policy_decision_refs"),
+            subject_person_id=r.get("subject_person_id", "") or "",
+            viewer_scope=r.get("viewer_scope", "owner") or "owner",
+            shareability=r.get("shareability", "owner_private") or "owner_private",
+            capability_allowlist=_list("capability_allowlist"),
+            goal_fingerprint=r.get("goal_fingerprint", "") or "",
+            created_at=float(r.get("created_at") or time.time()),
+            updated_at=float(r.get("updated_at") or time.time()),
+        )
+
+
+@dataclass
+class Step:
+    project_id: str
+    ordinal: int
+    description: str
+    action_kind: str = "analyze"
+    depends_on: List[int] = field(default_factory=list)   # ordinals within project
+    status: str = "pending"
+    attempts: int = 0
+    result: str = ""
+    work_order_ref: str = ""          # immutable project-ledger WorkOrder row
+    work_order_digest: str = ""       # exact authority envelope digest
+    work_order_issued_at: float = 0.0  # stable first-dispatch authority time
+    result_ref: str = ""              # logical ExecutionResultV1 ledger row
+    boundary_subject: str = ""          # extra subject text for the guard
+    confidence: float = 0.6             # planner-stated (charter contract)
+    id: str = field(default_factory=lambda: f"step-{uuid.uuid4().hex[:12]}")
+    created_at: float = field(default_factory=time.time)
+    updated_at: float = field(default_factory=time.time)
+
+    def to_row(self) -> Dict[str, Any]:
+        return {
+            "id": self.id, "project_id": self.project_id, "ordinal": self.ordinal,
+            "description": self.description, "action_kind": self.action_kind,
+            "depends_on": json.dumps(self.depends_on), "status": self.status,
+            "attempts": self.attempts, "result": self.result,
+            "work_order_ref": self.work_order_ref,
+            "work_order_digest": self.work_order_digest,
+            "work_order_issued_at": self.work_order_issued_at,
+            "result_ref": self.result_ref,
+            "boundary_subject": self.boundary_subject,
+            "confidence": self.confidence,
+            "created_at": self.created_at, "updated_at": self.updated_at,
+        }
+
+    @classmethod
+    def from_row(cls, r: Dict[str, Any]) -> "Step":
+        try:
+            deps = json.loads(r.get("depends_on") or "[]")
+            deps = [int(d) for d in deps] if isinstance(deps, list) else []
+        except Exception:
+            deps = []
+        return cls(
+            id=r["id"], project_id=r.get("project_id", "") or "",
+            ordinal=int(r.get("ordinal") or 0),
+            description=r.get("description", "") or "",
+            action_kind=r.get("action_kind", "analyze") or "analyze",
+            depends_on=deps, status=r.get("status", "pending") or "pending",
+            attempts=int(r.get("attempts") or 0),
+            result=r.get("result", "") or "",
+            work_order_ref=r.get("work_order_ref", "") or "",
+            work_order_digest=r.get("work_order_digest", "") or "",
+            work_order_issued_at=float(r.get("work_order_issued_at") or 0.0),
+            result_ref=r.get("result_ref", "") or "",
+            boundary_subject=r.get("boundary_subject", "") or "",
+            confidence=float(r.get("confidence", 0.6) or 0.6),
+            created_at=float(r.get("created_at") or time.time()),
+            updated_at=float(r.get("updated_at") or time.time()),
+        )

apsimo/projects/planner.py

@@ -0,0 +1,181 @@
+"""Project planner: objective -> validated, dependency-ordered Steps.
+
+One LLM planning pass returns a STRICT JSON step list; deterministic code
+re-validates everything (same discipline as directed intake): unknown
+action_kinds are dropped, dependency cycles are broken, unknown/self deps are
+stripped, and the step count is capped. The LLM proposes; code decides.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any, Dict, List, Optional
+
+from apsimo.projects.models import ACTION_KINDS, Step, projects_max_steps
+
+logger = logging.getLogger(__name__)
+
+# The system prompt composes through the shared cognition charter
+# (cognition/charter.py, role "planner"): shared doctrine + planner rules +
+# injected self-model/boundaries/skills. This block only supplies the
+# action-kind vocabulary the role rules reference.
+_ACTION_KINDS_CONTEXT = """\
+Allowed action kinds (choose the weakest kind that does the job):
+- "analyze": reason over information the system already has (memory, world
+  model, configured read-only repo mirrors).
+- "research": gather new information (web search, background research).
+- "internal": internal bookkeeping (record findings, update memory).
+- "directed": delegate scoped repo/code work to the directed-action pipeline
+  (it has its own approval gates). Use ONLY for work on the owner's
+  designated repositories.
+- "deliver": report a finding/milestone to the owner."""
+
+
+def _parse_array(content: str) -> List[dict]:
+    text = (content or "").strip()
+    if "```" in text:
+        m = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
+        if m:
+            text = m.group(1).strip()
+    if not text.startswith("["):
+        m = re.search(r"\[.*\]", text, re.DOTALL)
+        if not m:
+            return []
+        text = m.group(0)
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        logger.warning("project planner output was not valid JSON; dropping")
+        return []
+    return [d for d in data if isinstance(d, dict)] if isinstance(data, list) else []
+
+
+def validate_steps(raw: List[dict], project_id: str = "",
+                   max_steps: Optional[int] = None) -> List[Step]:
+    """Deterministic re-validation of an LLM-proposed step list.
+
+    Drops steps with unknown action_kind, strips unknown/self dependencies,
+    breaks dependency cycles (a dep that is not strictly ordinal-decreasing
+    after topological sorting is dropped), caps the count, and renumbers
+    ordinals contiguously.
+    """
+    cap = max_steps or projects_max_steps()
+
+    # 1. Shape + kind filter.
+    cleaned: List[Dict[str, Any]] = []
+    for d in raw or []:
+        desc = str(d.get("description", "")).strip()
+        kind = str(d.get("action_kind", "")).strip().lower()
+        if not desc or kind not in ACTION_KINDS:
+            continue
+        try:
+            ordinal = int(d.get("ordinal", len(cleaned) + 1))
+        except (TypeError, ValueError):
+            ordinal = len(cleaned) + 1
+        deps = []
+        for dep in (d.get("depends_on") or []):
+            try:
+                deps.append(int(dep))
+            except (TypeError, ValueError):
+                continue
+        try:
+            confidence = max(0.0, min(1.0, float(d.get("confidence", 0.6))))
+        except (TypeError, ValueError):
+            confidence = 0.6
+        cleaned.append({"ordinal": ordinal, "description": desc[:500],
+                        "action_kind": kind, "depends_on": deps,
+                        "confidence": confidence})
+
+    if not cleaned:
+        return []
+
+    # 2. Deduplicate ordinals (keep first) and drop unknown/self deps.
+    by_ord: Dict[int, Dict[str, Any]] = {}
+    for d in cleaned:
+        if d["ordinal"] not in by_ord:
+            by_ord[d["ordinal"]] = d
+    known = set(by_ord)
+    for d in by_ord.values():
+        d["depends_on"] = [x for x in d["depends_on"]
+                           if x in known and x != d["ordinal"]]
+
+    # 3. Topological sort (Kahn); cycle-breaking: when no node is free, cut
+    #    the remaining node with the fewest deps loose (drop its deps).
+    remaining = dict(by_ord)
+    ordered: List[Dict[str, Any]] = []
+    placed: set = set()
+    while remaining:
+        free = [o for o, d in remaining.items()
+                if all(dep in placed for dep in d["depends_on"])]
+        if not free:
+            victim = min(remaining.values(),
+                         key=lambda d: (len(d["depends_on"]), d["ordinal"]))
+            logger.info("project planner: breaking dependency cycle at step %s",
+                        victim["ordinal"])
+            victim["depends_on"] = [dep for dep in victim["depends_on"]
+                                    if dep in placed]
+            free = [victim["ordinal"]]
+        for o in sorted(free):
+            ordered.append(remaining.pop(o))
+            placed.add(o)
+
+    # 4. Cap, then drop deps pointing at capped-away steps.
+    ordered = ordered[:cap]
+    kept = {d["ordinal"] for d in ordered}
+    for d in ordered:
+        d["depends_on"] = [x for x in d["depends_on"] if x in kept]
+
+    # 5. Renumber contiguously (deps remapped).
+    remap = {d["ordinal"]: i + 1 for i, d in enumerate(ordered)}
+    steps: List[Step] = []
+    for d in ordered:
+        steps.append(Step(
+            project_id=project_id,
+            ordinal=remap[d["ordinal"]],
+            description=d["description"],
+            action_kind=d["action_kind"],
+            depends_on=sorted(remap[x] for x in d["depends_on"]),
+            confidence=d.get("confidence", 0.6),
+        ))
+    return steps
+
+
+async def plan_project(
+    router: Any,
+    objective: str,
+    *,
+    project_id: str = "",
+    context: str = "",
+    skills_block: str = "",
+    self_brief: str = "",
+    boundaries: str = "",
+    max_steps: Optional[int] = None,
+) -> List[Step]:
+    """One LLM planning pass -> validated Steps. Empty list on any failure."""
+    if router is None or not (objective or "").strip():
+        return []
+    cap = max_steps or projects_max_steps()
+    user_parts = [f"OBJECTIVE:\n{objective.strip()[:1500]}"]
+    if context:
+        user_parts.append(f"CONTEXT:\n{str(context)[:2000]}")
+    try:
+        from apsimo.cognition.charter import build_system_prompt
+        system = build_system_prompt(
+            "planner",
+            self_brief=(self_brief or None),
+            boundaries=(boundaries or None),
+            skills=(skills_block or None),
+            extra=_ACTION_KINDS_CONTEXT,
+            max_steps=cap)
+        response = await router.complete(
+            [{"role": "system", "content": system},
+             {"role": "user", "content": "\n\n".join(user_parts)}],
+            context={"task": "project_planning"})
+        content = getattr(response, "content", "") or ""
+    except Exception as exc:
+        logger.warning("project planning LLM call failed: %s", exc)
+        return []
+    return validate_steps(_parse_array(content), project_id=project_id,
+                          max_steps=cap)