apsimo 1.3.0__py3-none-any.whl

apsimo/connectors/manager.py

@@ -0,0 +1,218 @@
+"""ConnectorManager -- registers pull connectors and runs the ingest phase
+(cognition item 2, Phase C).
+
+Per due connector: poll -> for each Observation, OBSERVE-boundary check the
+subject (a "don't look at X" blackout suppresses ingest of X) -> record to the
+observation store (feeds the initiative engine) -> feed the world-model
+populator (which upserts entities/relationships under its OWN shadow-first
+gate; belief maintenance rides the populator's inline property-audit hook, so
+changed facts are reconciled without a second pass here).
+
+Mode (COLONY_CONNECTORS_MODE, default off):
+  off    -> no polling.
+  shadow -> CALIBRATION: poll + normalize + LOG the output and the entities it
+            WOULD populate, writing nothing (inspect a source before trusting
+            it, same discipline as the world-model populator shadow).
+  live   -> record observations + feed the populator.
+
+Per-connector cadence and enable/credentials are env-only
+(COLONY_CONNECTOR_<NAME>_*). Default off; enable per connector.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import asyncio
+import time
+from typing import Any, Dict, List, Optional
+
+from apsimo.connectors.base import Connector, Observation
+
+logger = logging.getLogger(__name__)
+
+
+def connectors_enabled() -> bool:
+    return os.environ.get(
+        "COLONY_CONNECTORS_ENABLED", "false").strip().lower() in (
+            "1", "true", "yes", "on")
+
+
+def connectors_mode() -> str:
+    from apsimo.util.autonomy_preset import resolve
+    return resolve("COLONY_CONNECTORS_MODE", ("off", "shadow", "live"), "off")
+
+
+class ConnectorManager:
+    def __init__(self, *, observation_store: Any = None, populator: Any = None,
+                 directive_manager: Any = None, self_model: Any = None) -> None:
+        self._obs = observation_store
+        self._populator = populator
+        self._directives = directive_manager
+        self._self_model = self_model
+        self._connectors: List[Connector] = []
+        self._last: Dict[str, Dict[str, Any]] = {}
+
+    # -- registration -----------------------------------------------------
+    def register(self, connector: Connector) -> None:
+        self._connectors.append(connector)
+
+    def register_default_connectors(self) -> int:
+        """Register every reference connector that is enabled.
+
+        Multi-account: a connector's ACCOUNTS key (env
+        COLONY_CONNECTOR_<NAME>_ACCOUNTS or secret connector/<name>/accounts,
+        comma-separated) expands into one instance per account, each reading
+        its own connector/<name>_<account>/* namespace. A listed account is
+        enabled by default; without ACCOUNTS the single classic instance is
+        used."""
+        from apsimo.connectors.base import ConnectorConfig
+        from apsimo.connectors.imap_email import IMAPEmailConnector
+        from apsimo.connectors.caldav_calendar import CalendarConnector
+        from apsimo.connectors.fs_documents import FSDocumentsConnector
+        from apsimo.connectors.webhook_pull import WebhookPullConnector
+        for cls in (IMAPEmailConnector, CalendarConnector,
+                    FSDocumentsConnector, WebhookPullConnector):
+            try:
+                accounts = [a.strip() for a in
+                            ConnectorConfig(cls.name).get("ACCOUNTS", "")
+                            .replace(";", ",").split(",") if a.strip()]
+                if accounts:
+                    instances = []
+                    for a in accounts:
+                        c = cls(account=a)
+                        c._default_enabled = True
+                        instances.append(c)
+                else:
+                    instances = [cls()]
+                for c in instances:
+                    if c.enabled:
+                        self.register(c)
+            except Exception:
+                logger.debug("connector %s init failed", cls.__name__,
+                             exc_info=True)
+        return len(self._connectors)
+
+    # -- boundary gate ----------------------------------------------------
+    def _boundary_ok(self, obs: Observation) -> bool:
+        """OBSERVE-capability check: a subject under a perception blackout is
+        suppressed. Reads survive an ACT-level "leave X alone"."""
+        if self._directives is None:
+            return True
+        try:
+            from apsimo.directives import Action
+            subject = " ".join([e.name for e in obs.entities]
+                               + [str(obs.payload.get("subject", ""))]).strip()
+            return self._directives.check(Action(
+                kind="observe", text=subject or obs.domain,
+                target=subject)).allowed
+        except Exception:
+            # Fail CLOSED: a broken boundary check must not convert into
+            # permission to ingest — the observation is only delayed until
+            # the directive engine answers again.
+            logger.warning("connector boundary check failed; withholding "
+                           "observation this cycle", exc_info=True)
+            return False
+
+    # -- the ingest phase -------------------------------------------------
+    async def poll_due(self, now: Optional[float] = None) -> Dict[str, Any]:
+        now = now if now is not None else time.time()
+        mode = connectors_mode()
+        report: Dict[str, Any] = {"mode": mode, "connectors": [],
+                                  "observations": 0, "populated": 0,
+                                  "skipped_boundary": 0}
+        if mode == "off" or not self._connectors:
+            return report
+
+        for c in self._connectors:
+            if not c.enabled or not c.due(now):
+                continue
+            try:
+                # poll() implementations do blocking I/O (urllib/IMAP/CalDAV
+                # with timeouts up to 30s) — off the event loop, or one hung
+                # endpoint freezes the whole autonomy tick.
+                observations = await asyncio.to_thread(c.poll)
+            except Exception:
+                logger.debug("connector %s poll failed", c.name, exc_info=True)
+                observations = []
+            c.mark_polled(now)
+
+            kept: List[Observation] = []
+            for obs in observations:
+                if not self._boundary_ok(obs):
+                    report["skipped_boundary"] += 1
+                    continue
+                kept.append(obs)
+
+            counts = await self._ingest(c, kept, mode)
+            self._last[c.name] = {
+                "at": now, "domain": c.domain, "count": len(kept),
+                "populated": counts["populated"]}
+            report["connectors"].append({
+                "name": c.name, "domain": c.domain,
+                "observations": len(kept), **counts})
+            report["observations"] += len(kept)
+            report["populated"] += counts["populated"]
+
+        if report["observations"] or report["skipped_boundary"]:
+            self._journal(report)
+        return report
+
+    async def _ingest(self, connector: Connector, observations: List[Observation],
+                      mode: str) -> Dict[str, int]:
+        populated = 0
+        if mode == "shadow":
+            # Log the normalized output + would-populate entities; write nothing.
+            for obs in observations:
+                logger.info("connector[shadow] %s: %s id=%s entities=%s",
+                            connector.name, obs.domain, obs.external_id,
+                            [e.name for e in obs.entities][:6])
+            return {"recorded": 0, "populated": 0}
+
+        # live: record observations, then feed the populator.
+        recorded = 0
+        if self._obs is not None and observations:
+            try:
+                rows = [o.to_store_row() for o in observations]
+                recorded = self._obs.record_batch(
+                    connector.domain, rows, reported_by=f"connector:{connector.name}")
+            except Exception:
+                logger.debug("observation record failed", exc_info=True)
+        if self._populator is not None:
+            for obs in observations:
+                if not obs.text:
+                    continue
+                try:
+                    await self._populator.populate_from_text(
+                        obs.text, source_id=f"connector:{connector.name}:{obs.external_id}")
+                    populated += 1
+                except Exception:
+                    logger.debug("populate failed", exc_info=True)
+        return {"recorded": recorded, "populated": populated}
+
+    # -- observability ----------------------------------------------------
+    def status(self) -> Dict[str, Any]:
+        return {
+            "mode": connectors_mode(),
+            "enabled": connectors_enabled(),
+            "connectors": [
+                {"name": c.name, "domain": c.domain, "enabled": c.enabled,
+                 "poll_secs": c.poll_secs, "last": self._last.get(c.name)}
+                for c in self._connectors],
+        }
+
+    def _journal(self, report: Dict[str, Any]) -> None:
+        journal = getattr(self._self_model, "journal", None)
+        if journal is None:
+            return
+        try:
+            journal.record(
+                "connectors",
+                f"ingested {report['observations']} observation(s) from "
+                f"{len(report['connectors'])} connector(s)",
+                reasoning=f"mode={report['mode']}, "
+                          f"skipped_boundary={report['skipped_boundary']}",
+                decision="acted" if report["mode"] == "live" else "held",
+                reversibility="reversible")
+        except Exception:
+            logger.debug("connector journal failed", exc_info=True)

apsimo/connectors/webhook_pull.py

@@ -0,0 +1,88 @@
+"""Generic JSON-pull connector (read-only): GETs a metrics/business endpoint
+and normalizes selected fields into "metrics" observations + a Project/Product
+entity carrying those metrics.
+
+This is a PULL connector (the sidecar fetches); push-style ingress belongs to
+the host framework's webhook adapter and is deliberately not built here. Config
+env-only (COLONY_CONNECTOR_WEBHOOK_*): URL, AUTH_HEADER + AUTH_VALUE, FIELD_MAP
+(JSON dotted-path map), ID_FIELD, ENTITY_NAME, ENTITY_KIND.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import Any, Dict, List
+
+from apsimo.connectors.base import Connector, EntityHint, Observation
+
+logger = logging.getLogger(__name__)
+
+
+def _dig(obj: Any, path: str) -> Any:
+    """Resolve a dotted path (a.b.0.c) against nested dict/list JSON."""
+    cur = obj
+    for part in path.split("."):
+        if isinstance(cur, dict):
+            cur = cur.get(part)
+        elif isinstance(cur, list):
+            try:
+                cur = cur[int(part)]
+            except (ValueError, IndexError):
+                return None
+        else:
+            return None
+    return cur
+
+
+class WebhookPullConnector(Connector):
+    name = "webhook"
+    domain = "metrics"
+    default_poll_secs = 3600
+
+    def normalize(self, data: Any, *, field_map: Dict[str, str],
+                  entity_name: str, entity_kind: str = "project",
+                  id_field: str = "") -> List[Observation]:
+        """Pure: map configured fields out of a JSON body into one metrics obs."""
+        metrics = {out_key: _dig(data, src) for out_key, src in field_map.items()}
+        metrics = {k: v for k, v in metrics.items() if v is not None}
+        ext_id = str(_dig(data, id_field) if id_field else entity_name) or entity_name
+        kv = ", ".join(f"{k}={v}" for k, v in metrics.items())
+        text = f"Metrics for {entity_name}: {kv}" if kv else f"Metrics for {entity_name}"
+        entities = [EntityHint(kind=entity_kind, name=entity_name)] if entity_name else []
+        return [Observation(
+            domain=self.domain, external_id=str(ext_id)[:200], ts=time.time(),
+            payload={"entity": entity_name, "metrics": metrics},
+            entities=entities, text=text)]
+
+    def _fetch(self) -> Any:
+        import urllib.request
+        url = self.config.get("URL")
+        if not url:
+            return None
+        req = urllib.request.Request(url, method="GET")
+        header = self.config.get("AUTH_HEADER")
+        value = self.config.get("AUTH_VALUE")
+        if header and value:
+            req.add_header(header, value)
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return json.loads(resp.read().decode("utf-8", "replace"))
+
+    def poll(self) -> List[Observation]:
+        try:
+            data = self._fetch()
+            if data is None:
+                return []
+            try:
+                field_map = json.loads(self.config.get("FIELD_MAP", "{}"))
+            except json.JSONDecodeError:
+                field_map = {}
+            return self.normalize(
+                data, field_map=field_map or {},
+                entity_name=self.config.get("ENTITY_NAME", "metrics"),
+                entity_kind=self.config.get("ENTITY_KIND", "project"),
+                id_field=self.config.get("ID_FIELD", ""))
+        except Exception:
+            logger.debug("webhook pull failed", exc_info=True)
+            return []

apsimo/contacts/__init__.py

@@ -0,0 +1,33 @@
+"""Colony Contacts Management System.
+
+Provides contact lifecycle management: storage, import, and privacy controls.
+All contacts are Person nodes in the world model.
+
+Quick start::
+
+    from apsimo.contacts import ContactStore
+    from apsimo.contacts.store import SQLiteContactStore
+    from apsimo.contacts.config import ContactsConfig
+
+    config = ContactsConfig(sqlite_path=":memory:")
+    async with SQLiteContactStore(config) as store:
+        contact = await store.create(display_name="Alice Chen", trust_tier="trusted")
+        await store.add_handle(contact.contact_id, "email", "alice@example.com")
+"""
+
+from .store import ContactStore, SQLiteContactStore
+from .importer import ContactImporter, SQLiteContactImporter
+from .models import Contact, ContactHandle, MergeProposal, MergeAuditRecord
+from .config import ContactsConfig
+
+__all__ = [
+    "ContactStore",
+    "ContactImporter",
+    "SQLiteContactStore",
+    "SQLiteContactImporter",
+    "Contact",
+    "ContactHandle",
+    "MergeProposal",
+    "MergeAuditRecord",
+    "ContactsConfig",
+]