apsimo 1.3.0__py3-none-any.whl

apsimo/connectors/base.py

@@ -0,0 +1,152 @@
+"""Connector framework base types (cognition item 2, Phase C).
+
+A connector is a READ-ONLY sense: it polls an external source on its own
+cadence and normalizes what it sees into Observations. Everything downstream
+(the observation store that feeds the initiative engine, the world-model
+populator, belief maintenance) consumes the same normalized shape, so a new
+source is just a new poll+normalize -- no bespoke ingest path.
+
+All configuration comes from the environment (per-connector
+COLONY_CONNECTOR_<NAME>_<KEY>); no credentials ever live in code or tests.
+Push-style ingress is intentionally NOT built here: the host framework's
+webhook adapter already handles HMAC-authenticated inbound POSTs. These are the
+pull-style connectors (IMAP, calendar, filesystem, metrics pull).
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class EntityHint:
+    """A structured entity a connector recognized in an observation."""
+    kind: str                       # person | company | event | document | project | ...
+    name: str
+    external_ids: Dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class Observation:
+    """The normalized unit every connector emits."""
+    domain: str                     # email | calendar | document | metrics | ...
+    external_id: str                # stable id within the domain (dedup key)
+    ts: float                       # source event time (epoch seconds)
+    payload: Dict[str, Any] = field(default_factory=dict)
+    entities: List[EntityHint] = field(default_factory=list)
+    text: str = ""                  # natural-language render for world extraction
+
+    def to_store_row(self) -> Dict[str, Any]:
+        """Shape for ObservationStore.record_batch (one snapshot per entity)."""
+        return {"entity_id": self.external_id,
+                "payload": {**self.payload, "_entities": [
+                    {"kind": e.kind, "name": e.name, "external_ids": e.external_ids}
+                    for e in self.entities], "_text": self.text},
+                "observed_at": _iso(self.ts)}
+
+
+def _iso(ts: float):
+    from datetime import datetime, timezone
+    try:
+        return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat()
+    except (ValueError, OverflowError, OSError):
+        return datetime.now(timezone.utc).isoformat()
+
+
+def _slug(value: str) -> str:
+    """Lowercase [a-z0-9_] account slug — it becomes part of env var names
+    and secrets paths, so anything else is dropped."""
+    return "".join(c for c in (value or "").lower() if c.isalnum() or c == "_")
+
+
+class ConnectorConfig:
+    """Config for one connector, resolved env-first then secrets-store.
+
+    Precedence: COLONY_CONNECTOR_<NAME>_<KEY> env (an explicit deployment
+    override always wins) → the Colony secrets store at
+    ``connector/<name>/<key>`` → default. The secrets fallback means a
+    credential (IMAP password, ICS URL) is entered ONCE via the secrets
+    API/CLI, lives encrypted in the store, and survives service redeploys
+    without ever touching a plist or unit file."""
+
+    def __init__(self, name: str) -> None:
+        self._name = name.lower()
+        self._prefix = f"COLONY_CONNECTOR_{name.upper()}_"
+
+    def get(self, key: str, default: str = "") -> str:
+        env = os.environ.get(self._prefix + key.upper())
+        if env:
+            return env
+        try:
+            from apsimo.api.routers import host as _h
+            mgr = getattr(_h, "_secrets_manager", None)
+            if mgr is not None:
+                v = mgr.get(f"connector/{self._name}/{key.lower()}")
+                if v is not None:
+                    return v
+        except Exception:
+            pass
+        return default
+
+    def get_int(self, key: str, default: int) -> int:
+        try:
+            return int(self.get(key, str(default)))
+        except (TypeError, ValueError):
+            return default
+
+    def get_bool(self, key: str, default: bool = False) -> bool:
+        v = self.get(key, "1" if default else "0").strip().lower()
+        return v in ("1", "true", "yes", "on")
+
+
+class Connector(ABC):
+    """Read-only pull connector. Subclasses implement fetch + normalize."""
+
+    name: str = "connector"
+    domain: str = "generic"
+    default_poll_secs: int = 900
+
+    def __init__(self, account: str = "") -> None:
+        # An account-scoped instance ("imap" + account "aevonix") reads its
+        # own namespace: secrets connector/imap_aevonix/* and env
+        # COLONY_CONNECTOR_IMAP_AEVONIX_*. Several instances of one connector
+        # class run side by side, one per account.
+        self.account = _slug(account)
+        if self.account:
+            self.name = f"{self.name}_{self.account}"
+        self.config = ConnectorConfig(self.name)
+        self._last_poll: float = 0.0
+
+    # -- lifecycle knobs (env/secrets-driven) -----------------------------
+    @property
+    def enabled(self) -> bool:
+        # An account listed in the connector's ACCOUNTS key is intended ON;
+        # it can still be turned off explicitly with its own enabled=false.
+        return self.config.get_bool(
+            "ENABLED", getattr(self, "_default_enabled", False))
+
+    @property
+    def poll_secs(self) -> int:
+        return self.config.get_int("POLL_SECS", self.default_poll_secs)
+
+    def due(self, now: Optional[float] = None) -> bool:
+        now = now if now is not None else time.time()
+        return (now - self._last_poll) >= self.poll_secs
+
+    def mark_polled(self, now: Optional[float] = None) -> None:
+        self._last_poll = now if now is not None else time.time()
+
+    # -- the contract -----------------------------------------------------
+    @abstractmethod
+    def poll(self) -> List[Observation]:
+        """Fetch from the source and return normalized Observations.
+
+        Must be side-effect free beyond reading. Never raises past its own
+        try/except; returns [] on any failure so one bad source cannot stall
+        the manager.
+        """
+        ...

apsimo/connectors/caldav_calendar.py

@@ -0,0 +1,125 @@
+"""Calendar connector (read-only): pulls an iCalendar (.ics) feed and
+normalizes VEVENTs into "calendar" observations + Event/person entities.
+
+An ICS feed URL (Google/CalDAV/Exchange all expose one) is the
+dependency-light pull path; a full CalDAV REPORT or an OAuth API token is the
+deployment option, documented in the handoff. Credentials env-only
+(COLONY_CONNECTOR_CALENDAR_*). Read-only: never creates or edits events.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from datetime import datetime, timezone
+from typing import Any, Dict, List
+
+from apsimo.connectors.base import Connector, EntityHint, Observation
+
+logger = logging.getLogger(__name__)
+
+_UNFOLD = re.compile(r"\r?\n[ \t]")
+
+
+def _parse_ics(ics_text: str) -> List[Dict[str, Any]]:
+    """Minimal VEVENT parser (stdlib): returns per-event property dicts."""
+    text = _UNFOLD.sub("", ics_text or "")  # unfold continued lines
+    events: List[Dict[str, Any]] = []
+    cur: Dict[str, Any] = {}
+    in_event = False
+    for line in text.splitlines():
+        if line.startswith("BEGIN:VEVENT"):
+            in_event, cur = True, {"attendees": []}
+            continue
+        if line.startswith("END:VEVENT"):
+            if in_event:
+                events.append(cur)
+            in_event = False
+            continue
+        if not in_event or ":" not in line:
+            continue
+        name, value = line.split(":", 1)
+        key = name.split(";", 1)[0].upper()
+        if key == "ATTENDEE":
+            cn = re.search(r"CN=([^;:]+)", name)
+            cur["attendees"].append(cn.group(1) if cn else value.replace("mailto:", ""))
+        elif key in ("SUMMARY", "UID", "LOCATION", "DTSTART", "DTEND",
+                     "ORGANIZER", "DESCRIPTION"):
+            cur[key.lower()] = value
+            if key == "ORGANIZER":
+                cn = re.search(r"CN=([^;:]+)", name)
+                if cn:
+                    cur["organizer_name"] = cn.group(1)
+    return events
+
+
+def _ics_ts(value: str) -> float:
+    if not value:
+        return time.time()
+    v = value.strip().rstrip("Z")
+    for fmt in ("%Y%m%dT%H%M%S", "%Y%m%d"):
+        try:
+            return datetime.strptime(v, fmt).replace(tzinfo=timezone.utc).timestamp()
+        except ValueError:
+            continue
+    return time.time()
+
+
+class CalendarConnector(Connector):
+    name = "calendar"
+    domain = "calendar"
+    default_poll_secs = 1800
+
+    def normalize(self, events: List[Dict[str, Any]]) -> List[Observation]:
+        out: List[Observation] = []
+        for ev in events:
+            summary = ev.get("summary", "").strip()
+            uid = ev.get("uid", "") or summary
+            location = ev.get("location", "").strip()
+            attendees = [a.strip() for a in ev.get("attendees", []) if a.strip()]
+            organizer = ev.get("organizer_name", "").strip()
+            ts = _ics_ts(ev.get("dtstart", ""))
+            entities: List[EntityHint] = []
+            if summary:
+                entities.append(EntityHint(kind="event", name=summary,
+                                           external_ids={"uid": uid}))
+            for person in ([organizer] if organizer else []) + attendees:
+                if person and "@" not in person and len(person.split()) <= 4:
+                    entities.append(EntityHint(kind="person", name=person))
+            if location and not location.startswith("http"):
+                entities.append(EntityHint(kind="location", name=location))
+            who = ", ".join([organizer] + attendees).strip(", ")
+            text = (f"Calendar event '{summary}'"
+                    + (f" at {location}" if location else "")
+                    + (f" with {who}" if who else "")).strip()
+            out.append(Observation(
+                domain=self.domain, external_id=str(uid)[:200], ts=ts,
+                payload={"summary": summary, "location": location,
+                         "attendees": attendees, "organizer": organizer,
+                         "start": ev.get("dtstart", ""), "end": ev.get("dtend", "")},
+                entities=entities, text=text))
+        return out
+
+    def _fetch(self) -> List[Dict[str, Any]]:
+        import urllib.request
+        url = self.config.get("ICS_URL")
+        if not url:
+            return []
+        req = urllib.request.Request(url, method="GET")
+        user = self.config.get("USER")
+        password = self.config.get("PASSWORD")
+        if user and password:
+            import base64
+            token = base64.b64encode(f"{user}:{password}".encode()).decode()
+            req.add_header("Authorization", f"Basic {token}")
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return _parse_ics(resp.read().decode("utf-8", "replace"))
+
+    def poll(self) -> List[Observation]:
+        try:
+            limit = self.config.get_int("MAX", 50)
+            return self.normalize(self._fetch()[:limit])
+        except Exception:
+            logger.debug("calendar poll failed", exc_info=True)
+            return []

apsimo/connectors/fs_documents.py

@@ -0,0 +1,85 @@
+"""Filesystem documents connector (read-only): watches a folder for new or
+changed files and normalizes them into "document" observations + entity hints.
+
+Config env-only (COLONY_CONNECTOR_FS_*): PATH (folder), EXTENSIONS (csv),
+MAX. Only reads file metadata + a short text snippet (for the world-model
+extractor); never writes or deletes.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+from typing import Any, Dict, List
+
+from apsimo.connectors.base import Connector, EntityHint, Observation
+
+logger = logging.getLogger(__name__)
+
+_TEXT_EXTS = {".txt", ".md", ".rst", ".csv", ".json", ".log"}
+
+
+class FSDocumentsConnector(Connector):
+    name = "fs"
+    domain = "document"
+    default_poll_secs = 600
+
+    def normalize(self, files: List[Dict[str, Any]]) -> List[Observation]:
+        out: List[Observation] = []
+        for f in files:
+            path = f.get("path", "")
+            fname = f.get("name", "") or os.path.basename(path)
+            snippet = (f.get("snippet", "") or "")[:400]
+            eid = hashlib.sha1(path.encode("utf-8")).hexdigest()[:16] if path else fname
+            text = f"Document '{fname}'" + (f": {snippet}" if snippet else "")
+            out.append(Observation(
+                domain=self.domain, external_id=eid,
+                ts=float(f.get("mtime", 0.0)),
+                payload={"path": path, "name": fname, "size": f.get("size", 0),
+                         "mtime": f.get("mtime", 0.0), "snippet": snippet},
+                entities=[EntityHint(kind="document", name=fname,
+                                     external_ids={"path": path})],
+                text=text))
+        return out
+
+    def _fetch(self) -> List[Dict[str, Any]]:
+        folder = self.config.get("PATH")
+        if not folder or not os.path.isdir(folder):
+            return []
+        exts = {e.strip().lower() if e.strip().startswith(".") else "." + e.strip().lower()
+                for e in self.config.get("EXTENSIONS", "txt,md,pdf,docx").split(",")
+                if e.strip()}
+        limit = self.config.get_int("MAX", 50)
+        since = self._last_poll
+        found: List[Dict[str, Any]] = []
+        for root, _dirs, names in os.walk(folder):
+            for n in names:
+                ext = os.path.splitext(n)[1].lower()
+                if exts and ext not in exts:
+                    continue
+                p = os.path.join(root, n)
+                try:
+                    st = os.stat(p)
+                except OSError:
+                    continue
+                if st.st_mtime <= since:
+                    continue
+                snippet = ""
+                if ext in _TEXT_EXTS:
+                    try:
+                        with open(p, "r", errors="replace") as fh:
+                            snippet = fh.read(600)
+                    except Exception:
+                        pass
+                found.append({"path": p, "name": n, "size": st.st_size,
+                              "mtime": st.st_mtime, "snippet": snippet})
+        found.sort(key=lambda d: d["mtime"], reverse=True)
+        return found[:limit]
+
+    def poll(self) -> List[Observation]:
+        try:
+            return self.normalize(self._fetch())
+        except Exception:
+            logger.debug("fs poll failed", exc_info=True)
+            return []

apsimo/connectors/imap_email.py

@@ -0,0 +1,138 @@
+"""IMAP email connector (read-only). Reference connector for item 2.
+
+Reads recent message envelopes (headers + a short snippet) over IMAP and
+normalizes them into "email" observations plus person/company entity hints.
+Credentials are env-only (COLONY_CONNECTOR_IMAP_*). Never sends, never
+deletes, never marks read (uses BODY.PEEK).
+"""
+
+from __future__ import annotations
+
+import email
+import logging
+import re
+from email.header import decode_header
+from email.utils import parseaddr, parsedate_to_datetime
+from typing import Any, Dict, List
+
+from apsimo.connectors.base import Connector, EntityHint, Observation
+
+logger = logging.getLogger(__name__)
+
+# Consumer providers whose domain is NOT a company signal.
+_GENERIC_DOMAINS = frozenset({
+    "gmail.com", "googlemail.com", "yahoo.com", "outlook.com", "hotmail.com",
+    "icloud.com", "me.com", "aol.com", "proton.me", "protonmail.com", "gmx.com",
+})
+
+
+def _decode(raw: Any) -> str:
+    if raw is None:
+        return ""
+    if isinstance(raw, bytes):
+        raw = raw.decode("utf-8", "replace")
+    try:
+        parts = decode_header(raw)
+        return "".join(
+            (p.decode(enc or "utf-8", "replace") if isinstance(p, bytes) else p)
+            for p, enc in parts)
+    except Exception:
+        return str(raw)
+
+
+def _company_from_email(addr: str) -> str:
+    dom = (addr.rsplit("@", 1)[-1] if "@" in addr else "").lower()
+    if not dom or dom in _GENERIC_DOMAINS:
+        return ""
+    base = dom.rsplit(".", 1)[0].split(".")[-1]  # acme from mail.acme.co.uk-ish
+    return base.replace("-", " ").title() if base else ""
+
+
+class IMAPEmailConnector(Connector):
+    name = "imap"
+    domain = "email"
+    default_poll_secs = 900
+
+    def normalize(self, messages: List[Dict[str, Any]]) -> List[Observation]:
+        """Pure: [{message_id, from, to, subject, date, snippet}] -> Observations."""
+        out: List[Observation] = []
+        for m in messages:
+            from_name, from_addr = parseaddr(m.get("from", ""))
+            from_name = _decode(from_name) or from_addr
+            subject = _decode(m.get("subject", ""))
+            snippet = (m.get("snippet", "") or "")[:400]
+            ts = self._ts(m.get("date"))
+            entities: List[EntityHint] = []
+            if from_name and "@" not in from_name and len(from_name.split()) <= 4:
+                entities.append(EntityHint(kind="person", name=from_name,
+                                           external_ids={"email": from_addr}))
+            company = _company_from_email(from_addr)
+            if company:
+                entities.append(EntityHint(kind="company", name=company,
+                                           external_ids={"domain": from_addr.rsplit('@', 1)[-1]}))
+            text = (f"Email from {from_name} <{from_addr}> "
+                    f"re: {subject}. {snippet}").strip()
+            out.append(Observation(
+                domain=self.domain,
+                external_id=str(m.get("message_id") or f"{from_addr}:{subject}")[:200],
+                ts=ts,
+                payload={"from": from_addr, "from_name": from_name,
+                         "to": m.get("to", ""), "subject": subject,
+                         "snippet": snippet,
+                         "account": self.config.get("USER", self.account)},
+                entities=entities, text=text))
+        return out
+
+    @staticmethod
+    def _ts(date_str: Any) -> float:
+        import time
+        if not date_str:
+            return time.time()
+        try:
+            return parsedate_to_datetime(str(date_str)).timestamp()
+        except Exception:
+            return time.time()
+
+    def _fetch(self) -> List[Dict[str, Any]]:
+        import imaplib
+        host = self.config.get("HOST")
+        user = self.config.get("USER")
+        password = self.config.get("PASSWORD")
+        if not (host and user and password):
+            return []
+        port = self.config.get_int("PORT", 993)
+        mailbox = self.config.get("MAILBOX", "INBOX")
+        limit = self.config.get_int("MAX", 20)
+        msgs: List[Dict[str, Any]] = []
+        imap = imaplib.IMAP4_SSL(host, port)
+        try:
+            imap.login(user, password)
+            imap.select(mailbox, readonly=True)
+            typ, data = imap.search(None, "ALL")
+            if typ != "OK":
+                return []
+            ids = data[0].split()[-limit:]
+            for mid in reversed(ids):
+                typ, msg_data = imap.fetch(
+                    mid, "(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID FROM TO SUBJECT DATE)])")
+                if typ != "OK" or not msg_data or not msg_data[0]:
+                    continue
+                hdr = email.message_from_bytes(msg_data[0][1])
+                msgs.append({
+                    "message_id": hdr.get("Message-ID", ""),
+                    "from": hdr.get("From", ""), "to": hdr.get("To", ""),
+                    "subject": hdr.get("Subject", ""), "date": hdr.get("Date", ""),
+                    "snippet": ""})
+        finally:
+            try:
+                imap.logout()
+            except Exception:
+                pass
+        return msgs
+
+    def poll(self) -> List[Observation]:
+        try:
+            return self.normalize(self._fetch())
+        except Exception:
+            logger.debug("imap poll failed", exc_info=True)
+            return []