apsimo 1.3.0__py3-none-any.whl

apsimo/channels/phone_gateways.py

@@ -0,0 +1,42 @@
+"""Shared phone gateway resolution.
+
+Replaces the three hardcoded _PHONE_GATEWAYS tuples scattered across
+contacts/store.py, identity/resolver.py, and contacts/world_bridge.py.
+
+Reads from the channel store when available; falls back to a built-in
+set so the system works before any channels are registered.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+_BUILTIN_PHONE_GATEWAYS = frozenset({
+    "imessage", "sms", "signal", "whatsapp",
+})
+
+_channel_store_ref: Optional[object] = None
+
+
+def set_channel_store_ref(store) -> None:
+    global _channel_store_ref
+    _channel_store_ref = store
+
+
+def get_phone_gateways() -> frozenset[str]:
+    """Return the set of gateways that use phone-number identity.
+
+    When a channel store is available, returns the union of registered
+    channels with phone_identity_unification=True and the built-in set.
+    Without a channel store, returns only the built-in set.
+    """
+    if _channel_store_ref is not None:
+        try:
+            registered = _channel_store_ref.get_phone_gateways()
+            return frozenset(registered | _BUILTIN_PHONE_GATEWAYS)
+        except Exception:
+            logger.debug("Channel store query failed, using built-in set", exc_info=True)
+    return _BUILTIN_PHONE_GATEWAYS

apsimo/channels/presence.py

@@ -0,0 +1,188 @@
+"""ConversationPresenceStore — who has been seen in which conversation (L1.1).
+
+The conversation participant registry the leveled cross-contact tom2 system
+(docs-level: TOM2 levels) is built on. Every attributed turn records
+``(conversation_key, contact_id, resolution method, group_id)``; downstream
+consumers (the environment-risk classifier, subject-presence exclusion) read a
+windowed census of a conversation.
+
+Properties:
+
+* **Passive.** Recording is fed from the turns/sync attribution chokepoint
+  (after the ParticipantResolver has decided WHO the turn came from) and
+  changes nothing about turn processing. ``COLONY_CONV_PRESENCE`` (default
+  on) can disable recording entirely.
+* **Identity only.** Rows carry contact ids, resolution methods and opaque
+  conversation keys — never message content.
+* **System excluded.** The reserved machine sentinel (``system``) is never
+  recorded: machine turns must not shape a conversation's human census.
+* **Reads fail closed.** Read methods PROPAGATE storage errors instead of
+  returning an empty (i.e. "nobody here") census — an empty answer from a
+  broken store would read as a SAFE signal to the risk classifier. Callers
+  must catch and treat any error as "unknown, assume hostile".
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import sqlite3
+import threading
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, List, Optional
+
+from apsimo.identity.participants import SYSTEM_CONTACT_ID
+
+logger = logging.getLogger(__name__)
+
+#: Resolution methods considered STRONG identity evidence: the contact was
+#: matched by a verified handle or arrived pre-resolved as a canonical contact
+#: id. Scoped-name matches (unverified link proposals), shadow contacts and
+#: client-claimed ids are NOT strong.
+STRONG_METHODS = frozenset({"handle", "contact_id"})
+
+
+def conv_presence_enabled() -> bool:
+    """COLONY_CONV_PRESENCE (default on): passive presence recording."""
+    return os.environ.get("COLONY_CONV_PRESENCE", "on").strip().lower() not in (
+        "off", "0", "false", "no")
+
+
+def _now() -> datetime:
+    return datetime.now(tz=timezone.utc)
+
+
+class ConversationPresenceStore:
+    """SQLite-backed registry of who was seen in which conversation."""
+
+    def __init__(self, db_path: Optional[str] = None) -> None:
+        self._lock = threading.RLock()
+        self._conn = sqlite3.connect(db_path or ":memory:",
+                                     check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        with self._lock:
+            self._conn.execute("""
+                CREATE TABLE IF NOT EXISTS conversation_presence (
+                    conversation_key TEXT NOT NULL,
+                    contact_id       TEXT NOT NULL,
+                    method           TEXT NOT NULL DEFAULT '',
+                    group_id         TEXT NOT NULL DEFAULT '',
+                    first_seen_at    TEXT NOT NULL,
+                    last_seen_at     TEXT NOT NULL,
+                    turns            INTEGER NOT NULL DEFAULT 0,
+                    PRIMARY KEY (conversation_key, contact_id)
+                )""")
+            self._conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_conv_presence_contact "
+                "ON conversation_presence(contact_id)")
+            self._conn.commit()
+
+    # -- writes -----------------------------------------------------------
+    def record(self, conversation_key: str, contact_id: str, *,
+               method: str = "", group_id: str = "") -> bool:
+        """Record one sighting of ``contact_id`` in ``conversation_key``.
+
+        Returns True when a row was written/refreshed. The system sentinel
+        and empty ids are silently skipped (never an error — the chokepoint
+        must not care); the COLONY_CONV_PRESENCE gate turns the whole write
+        into a no-op. ``method`` stores the LATEST resolution method for the
+        pair: a weak latest sighting correctly downgrades the row, and a row
+        can only read as strong when the most recent turn actually resolved
+        strongly (by verified handle / canonical contact id).
+        """
+        if not conv_presence_enabled():
+            return False
+        conversation_key = str(conversation_key or "").strip()
+        contact_id = str(contact_id or "").strip()
+        if not conversation_key or not contact_id:
+            return False
+        if contact_id == SYSTEM_CONTACT_ID:
+            return False
+        now = _now().isoformat()
+        method = str(method or "").strip().lower()
+        group_id = str(group_id or "").strip()
+        with self._lock:
+            self._conn.execute(
+                """INSERT INTO conversation_presence
+                   (conversation_key, contact_id, method, group_id,
+                    first_seen_at, last_seen_at, turns)
+                   VALUES (?,?,?,?,?,?,1)
+                   ON CONFLICT(conversation_key, contact_id) DO UPDATE SET
+                    method=excluded.method,
+                    group_id=CASE WHEN excluded.group_id != ''
+                                  THEN excluded.group_id ELSE group_id END,
+                    last_seen_at=excluded.last_seen_at,
+                    turns=turns+1""",
+                (conversation_key, contact_id, method, group_id, now, now))
+            self._conn.commit()
+        return True
+
+    # -- reads (fail closed: errors PROPAGATE) ------------------------------
+    def census(self, conversation_key: str,
+               window_hours: float = 48.0) -> List[Dict[str, Any]]:
+        """Participants seen in this conversation within the window.
+
+        Raises on any storage error — callers MUST treat an exception as
+        "census unknown" (hostile), never as "empty room".
+        """
+        cutoff = (_now() - timedelta(hours=float(window_hours))).isoformat()
+        with self._lock:
+            rows = self._conn.execute(
+                "SELECT * FROM conversation_presence "
+                "WHERE conversation_key=? AND last_seen_at >= ? "
+                "ORDER BY last_seen_at DESC",
+                (str(conversation_key or ""), cutoff)).fetchall()
+        return [dict(r) for r in rows]
+
+    def is_present(self, conversation_key: str, contact_id: str,
+                   window_hours: float = 48.0) -> bool:
+        """True when the contact was seen in the conversation within the
+        window. Raises on storage errors (see census)."""
+        cutoff = (_now() - timedelta(hours=float(window_hours))).isoformat()
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT 1 FROM conversation_presence "
+                "WHERE conversation_key=? AND contact_id=? AND last_seen_at >= ?",
+                (str(conversation_key or ""), str(contact_id or ""),
+                 cutoff)).fetchone()
+        return row is not None
+
+    def cooccurred(self, contact_a: str, contact_b: str,
+                   within_days: float = 30.0) -> bool:
+        """True when both contacts were seen in at least ONE common
+        conversation within the window (mutual-knowledge evidence: they
+        demonstrably know of each other). Raises on storage errors."""
+        a = str(contact_a or "").strip()
+        b = str(contact_b or "").strip()
+        if not a or not b or a == b:
+            return False
+        cutoff = (_now() - timedelta(days=float(within_days))).isoformat()
+        with self._lock:
+            row = self._conn.execute(
+                """SELECT 1 FROM conversation_presence pa
+                   JOIN conversation_presence pb
+                     ON pa.conversation_key = pb.conversation_key
+                   WHERE pa.contact_id=? AND pb.contact_id=?
+                     AND pa.last_seen_at >= ? AND pb.last_seen_at >= ?
+                   LIMIT 1""", (a, b, cutoff, cutoff)).fetchone()
+        return row is not None
+
+    def counts(self) -> Dict[str, Any]:
+        """Aggregate observability (numbers only)."""
+        with self._lock:
+            convs = self._conn.execute(
+                "SELECT COUNT(DISTINCT conversation_key) n "
+                "FROM conversation_presence").fetchone()
+            contacts = self._conn.execute(
+                "SELECT COUNT(DISTINCT contact_id) n "
+                "FROM conversation_presence").fetchone()
+            rows = self._conn.execute(
+                "SELECT COUNT(*) n FROM conversation_presence").fetchone()
+        return {"rows": int(rows["n"]), "conversations": int(convs["n"]),
+                "contacts": int(contacts["n"])}
+
+    def close(self) -> None:
+        try:
+            self._conn.close()
+        except Exception:
+            pass

apsimo/channels/router.py

@@ -0,0 +1,235 @@
+"""Channel registration API -- /v1/channels/ endpoints."""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import os
+from typing import Optional
+from urllib.parse import urlparse
+
+from fastapi import APIRouter, Header, HTTPException, status
+from pydantic import BaseModel
+
+from apsimo.channels.manifest import ChannelManifest
+from apsimo.channels.store import ChannelStore
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/v1/channels", tags=["channels"])
+
+_channel_store: Optional[ChannelStore] = None
+
+
+def set_channel_store(store: ChannelStore) -> None:
+    global _channel_store
+    _channel_store = store
+
+
+def get_channel_store() -> Optional[ChannelStore]:
+    return _channel_store
+
+
+def _require_store() -> ChannelStore:
+    if _channel_store is None:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="Channel store not initialized",
+        )
+    return _channel_store
+
+
+# ── Request / Response models ────────────────────────────────────────────
+
+
+class RegisterResponse(BaseModel):
+    channel_key: str
+    registered_at: str
+    channel_token: str
+
+
+class ChannelInfo(BaseModel):
+    channel_key: str
+    display_name: str
+    gateway_family: str
+    status: str
+    registered_at: str
+    last_seen_at: Optional[str] = None
+    supports_media: bool = False
+    supports_reactions: bool = False
+    supports_voice: bool = False
+    supports_rich_text: bool = False
+    max_message_length: Optional[int] = None
+    phone_identity_unification: bool = False
+    provides_channel_id: bool = False
+    delivery_protocol: str = "hermes"
+    delivery_aliases: list[str] = []
+    home_chat_id: Optional[str] = None
+
+
+# ── Webhook validation ───────────────────────────────────────────────────
+
+
+_PRIVATE_NETWORKS = [
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+    ipaddress.ip_network("169.254.0.0/16"),
+    ipaddress.ip_network("127.0.0.0/8"),
+    ipaddress.ip_network("::1/128"),
+    ipaddress.ip_network("fd00::/8"),
+    ipaddress.ip_network("fe80::/10"),
+]
+
+
+def _validate_webhook(url: str) -> None:
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Webhook URL must be http or https, got '{parsed.scheme}'",
+        )
+    if not parsed.hostname:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Webhook URL has no hostname",
+        )
+
+    allow_private = os.environ.get(
+        "COLONY_ALLOW_PRIVATE_WEBHOOKS", ""
+    ).lower() in ("true", "1", "yes")
+
+    if not allow_private:
+        try:
+            addr = ipaddress.ip_address(parsed.hostname)
+            if any(addr in net for net in _PRIVATE_NETWORKS):
+                raise HTTPException(
+                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                    detail=(
+                        f"Webhook URL resolves to private address {addr}. "
+                        "Set COLONY_ALLOW_PRIVATE_WEBHOOKS=true to allow."
+                    ),
+                )
+        except ValueError:
+            pass
+
+
+# ── Endpoints ────────────────────────────────────────────────────────────
+
+
+@router.post("/register", status_code=status.HTTP_201_CREATED)
+async def register_channel(
+    manifest: ChannelManifest,
+    x_channel_token: Optional[str] = Header(None),
+) -> RegisterResponse:
+    store = _require_store()
+
+    if manifest.delivery_webhook:
+        _validate_webhook(manifest.delivery_webhook)
+
+    try:
+        registered = store.register(manifest, channel_token=x_channel_token)
+    except ValueError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT, detail=str(exc)
+        )
+
+    return RegisterResponse(
+        channel_key=registered.channel_key,
+        registered_at=registered.registered_at,
+        channel_token=registered.channel_token,
+    )
+
+
+@router.get("", response_model=list[ChannelInfo])
+async def list_channels() -> list[ChannelInfo]:
+    store = _require_store()
+    return [_to_info(ch) for ch in store.list_all()]
+
+
+@router.get("/{channel_key}", response_model=ChannelInfo)
+async def get_channel(channel_key: str) -> ChannelInfo:
+    store = _require_store()
+    ch = store.get(channel_key)
+    if ch is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Channel '{channel_key}' not found",
+        )
+    return _to_info(ch)
+
+
+@router.put("/{channel_key}", response_model=ChannelInfo)
+async def update_channel(
+    channel_key: str,
+    manifest: ChannelManifest,
+    x_channel_token: str = Header(...),
+) -> ChannelInfo:
+    store = _require_store()
+
+    if manifest.channel_key != channel_key:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="channel_key in body must match URL path",
+        )
+    if manifest.delivery_webhook:
+        _validate_webhook(manifest.delivery_webhook)
+
+    if not store.verify_token(channel_key, x_channel_token):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid channel token",
+        )
+
+    try:
+        registered = store.register(manifest, channel_token=x_channel_token)
+    except ValueError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT, detail=str(exc)
+        )
+
+    return _to_info(registered)
+
+
+@router.delete("/{channel_key}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_channel(
+    channel_key: str,
+    x_channel_token: str = Header(...),
+) -> None:
+    store = _require_store()
+
+    if not store.verify_token(channel_key, x_channel_token):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid channel token",
+        )
+    if not store.revoke(channel_key):
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Channel '{channel_key}' not found",
+        )
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────
+
+
+def _to_info(ch) -> ChannelInfo:
+    m = ch.manifest
+    return ChannelInfo(
+        channel_key=ch.channel_key,
+        display_name=ch.display_name,
+        gateway_family=ch.gateway_family,
+        status=ch.status,
+        registered_at=ch.registered_at,
+        last_seen_at=ch.last_seen_at,
+        supports_media=m.supports_media,
+        supports_reactions=m.supports_reactions,
+        supports_voice=m.supports_voice,
+        supports_rich_text=m.supports_rich_text,
+        max_message_length=m.max_message_length,
+        phone_identity_unification=m.phone_identity_unification,
+        provides_channel_id=m.provides_channel_id,
+        delivery_protocol=m.delivery_protocol,
+        delivery_aliases=m.delivery_aliases,
+        home_chat_id=m.home_chat_id,
+    )

apsimo/channels/store.py

@@ -0,0 +1,231 @@
+"""Channel registration store -- backed by colony-channels.db.
+
+Tracks which channels are registered with this Colony instance, their
+capabilities, and authentication tokens.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import logging
+import secrets
+import sqlite3
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from apsimo.channels.manifest import ChannelManifest
+from apsimo.migrations import run_migrations_sync
+
+logger = logging.getLogger(__name__)
+
+_MIGRATIONS_DIR = Path(__file__).parent / "migrations"
+
+
+@dataclass
+class RegisteredChannel:
+    channel_key: str
+    display_name: str
+    gateway_family: str
+    manifest: ChannelManifest
+    channel_token: str
+    registered_at: str
+    last_seen_at: Optional[str]
+    status: str
+
+
+class ChannelStore:
+    """SQLite-backed channel registration store."""
+
+    def __init__(self, db_path: str = ":memory:") -> None:
+        self._db_path = db_path
+        self._conn: Optional[sqlite3.Connection] = None
+
+    def connect(self) -> None:
+        if self._db_path != ":memory:":
+            Path(self._db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(self._db_path, check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA foreign_keys=ON")
+        run_migrations_sync(self._conn, _MIGRATIONS_DIR)
+
+    def close(self) -> None:
+        if self._conn:
+            self._conn.close()
+            self._conn = None
+
+    def _require_conn(self) -> sqlite3.Connection:
+        if not self._conn:
+            raise RuntimeError("ChannelStore not connected")
+        return self._conn
+
+    # ── Registration ─────────────────────────────────────────────────────
+
+    def register(
+        self,
+        manifest: ChannelManifest,
+        *,
+        channel_token: Optional[str] = None,
+    ) -> RegisteredChannel:
+        """Register a new channel or update an existing one.
+
+        If the channel_key already exists and the provided token matches,
+        the registration is updated (upsert).  If no token is provided for
+        a new registration, one is generated.
+
+        Raises ValueError on token mismatch (409 at the API layer).
+        """
+        conn = self._require_conn()
+        now = datetime.now(timezone.utc).isoformat()
+
+        existing = self.get(manifest.channel_key)
+
+        if existing is not None:
+            if not channel_token or not self.verify_token(
+                manifest.channel_key, channel_token
+            ):
+                raise ValueError(
+                    f"Channel '{manifest.channel_key}' already registered; "
+                    "provide the correct channel_token to update"
+                )
+            conn.execute(
+                """UPDATE channels
+                   SET display_name = ?, gateway_family = ?,
+                       manifest_json = ?, last_seen_at = ?
+                 WHERE channel_key = ?""",
+                (
+                    manifest.display_name,
+                    manifest.gateway_family,
+                    manifest.model_dump_json(),
+                    now,
+                    manifest.channel_key,
+                ),
+            )
+            conn.commit()
+            return self.get(manifest.channel_key)  # type: ignore[return-value]
+
+        token = channel_token or _generate_token()
+        token_hash = _hash_token(token)
+
+        conn.execute(
+            """INSERT INTO channels
+               (channel_key, display_name, gateway_family,
+                manifest_json, channel_token, registered_at, last_seen_at, status)
+               VALUES (?, ?, ?, ?, ?, ?, ?, 'active')""",
+            (
+                manifest.channel_key,
+                manifest.display_name,
+                manifest.gateway_family,
+                manifest.model_dump_json(),
+                token_hash,
+                now,
+                now,
+            ),
+        )
+        conn.commit()
+
+        registered = self.get(manifest.channel_key)
+        assert registered is not None
+        registered.channel_token = token
+        return registered
+
+    # ── Queries ──────────────────────────────────────────────────────────
+
+    def get(self, channel_key: str) -> Optional[RegisteredChannel]:
+        conn = self._require_conn()
+        cur = conn.execute(
+            "SELECT * FROM channels WHERE channel_key = ?", (channel_key,)
+        )
+        row = cur.fetchone()
+        return _row_to_channel(row) if row else None
+
+    def list_active(self) -> list[RegisteredChannel]:
+        conn = self._require_conn()
+        cur = conn.execute(
+            "SELECT * FROM channels WHERE status = 'active' ORDER BY channel_key"
+        )
+        return [_row_to_channel(row) for row in cur.fetchall()]
+
+    def list_all(self) -> list[RegisteredChannel]:
+        conn = self._require_conn()
+        cur = conn.execute("SELECT * FROM channels ORDER BY channel_key")
+        return [_row_to_channel(row) for row in cur.fetchall()]
+
+    def get_phone_gateways(self) -> set[str]:
+        """Return channel_keys with phone_identity_unification enabled."""
+        return {
+            ch.channel_key
+            for ch in self.list_active()
+            if ch.manifest.phone_identity_unification
+        }
+
+    # ── Mutations ────────────────────────────────────────────────────────
+
+    def touch(self, channel_key: str) -> None:
+        """Update last_seen_at for the given channel."""
+        conn = self._require_conn()
+        now = datetime.now(timezone.utc).isoformat()
+        conn.execute(
+            "UPDATE channels SET last_seen_at = ? WHERE channel_key = ?",
+            (now, channel_key),
+        )
+        conn.commit()
+
+    def revoke(self, channel_key: str) -> bool:
+        conn = self._require_conn()
+        cur = conn.execute(
+            "UPDATE channels SET status = 'revoked' WHERE channel_key = ?",
+            (channel_key,),
+        )
+        conn.commit()
+        return cur.rowcount > 0
+
+    def delete(self, channel_key: str) -> bool:
+        conn = self._require_conn()
+        cur = conn.execute(
+            "DELETE FROM channels WHERE channel_key = ?", (channel_key,)
+        )
+        conn.commit()
+        return cur.rowcount > 0
+
+    # ── Auth ─────────────────────────────────────────────────────────────
+
+    def verify_token(self, channel_key: str, token: str) -> bool:
+        conn = self._require_conn()
+        cur = conn.execute(
+            "SELECT channel_token FROM channels WHERE channel_key = ?",
+            (channel_key,),
+        )
+        row = cur.fetchone()
+        if not row:
+            return False
+        return hmac.compare_digest(row["channel_token"], _hash_token(token))
+
+
+# ── Helpers ──────────────────────────────────────────────────────────────
+
+
+def _generate_token() -> str:
+    return "ch_" + secrets.token_urlsafe(32)
+
+
+def _hash_token(token: str) -> str:
+    return hashlib.sha256(token.encode()).hexdigest()
+
+
+def _row_to_channel(row: sqlite3.Row) -> RegisteredChannel:
+    manifest = ChannelManifest.model_validate_json(row["manifest_json"])
+    return RegisteredChannel(
+        channel_key=row["channel_key"],
+        display_name=row["display_name"],
+        gateway_family=row["gateway_family"],
+        manifest=manifest,
+        channel_token=row["channel_token"],
+        registered_at=row["registered_at"],
+        last_seen_at=row["last_seen_at"],
+        status=row["status"],
+    )