cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/teams/__init__.py

@@ -0,0 +1,656 @@
+"""
+Microsoft Teams connector for CortexDB.
+
+Ingests channel messages, chat messages, threaded replies, meeting transcripts,
+and file shares from Microsoft Teams into CortexDB as Episodes.  Uses the
+Microsoft Graph API v1.0 via ``httpx`` for HTTP access, and ``msal`` for
+Azure AD OAuth2 client-credentials authentication.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="teams")
+_GRAPH_BASE = "https://graph.microsoft.com/v1.0"
+_PAGE_SIZE = 50
+
+# Microsoft Graph throttling: respect Retry-After, default 60 req/min.
+_DEFAULT_RETRY_DELAY = 5
+_RATE_LIMIT_DELAY = 1.0
+
+
+class TeamsConnector(CortexConnector):
+    """Connector that syncs Microsoft Teams messages into CortexDB.
+
+    Uses Azure AD application (client credentials) flow to authenticate
+    against the Microsoft Graph API.  Requires an Azure AD app registration
+    with the following *application* permissions:
+
+    - ``ChannelMessage.Read.All``
+    - ``Chat.Read.All``
+    - ``User.Read.All``
+    - ``TeamMember.Read.All``
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    azure_tenant_id:
+        Azure AD tenant (directory) ID.
+    client_id:
+        Azure AD application (client) ID.
+    client_secret:
+        Azure AD client secret.
+    team_ids:
+        List of Teams team IDs to sync.  Required for channel message
+        ingestion.
+    include_chats:
+        Whether to ingest 1:1 and group chat messages.
+    namespace:
+        CortexDB namespace for ingested episodes.
+    tenant_id:
+        CortexDB tenant identifier.
+    backfill_days:
+        Number of days of history to fetch on the first sync.
+    poll_interval_sec:
+        Seconds between polling cycles (used by the worker loop, not
+        by ``fetch_events`` directly).
+    """
+
+    connector_name: str = "teams"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        azure_tenant_id: str,
+        client_id: str,
+        client_secret: str,
+        team_ids: list[str] | None = None,
+        include_chats: bool = False,
+        namespace: str = "teams",
+        tenant_id: str = "default",
+        backfill_days: int = 30,
+        poll_interval_sec: int = 30,
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        self.azure_tenant_id = azure_tenant_id
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.team_ids = team_ids or []
+        self.include_chats = include_chats
+        self.namespace = namespace
+        self.backfill_days = backfill_days
+        self.poll_interval_sec = poll_interval_sec
+
+        self._access_token: str | None = None
+        self._token_expires_at: datetime = datetime.min.replace(tzinfo=timezone.utc)
+        self._user_cache: dict[str, dict[str, Any]] = {}
+
+    # -- authentication ------------------------------------------------------
+
+    async def _ensure_token(self) -> str:
+        """Acquire or refresh an Azure AD access token via MSAL."""
+        now = datetime.now(timezone.utc)
+        if self._access_token and now < self._token_expires_at:
+            return self._access_token
+
+        import msal
+
+        app = msal.ConfidentialClientApplication(
+            client_id=self.client_id,
+            client_credential=self.client_secret,
+            authority=f"https://login.microsoftonline.com/{self.azure_tenant_id}",
+        )
+        result = app.acquire_token_for_client(
+            scopes=["https://graph.microsoft.com/.default"]
+        )
+        if "access_token" not in result:
+            error = result.get("error_description", result.get("error", "unknown"))
+            raise RuntimeError(f"Azure AD token acquisition failed: {error}")
+
+        self._access_token = result["access_token"]
+        # MSAL returns expires_in in seconds; buffer 60 s for safety.
+        expires_in = int(result.get("expires_in", 3600))
+        self._token_expires_at = now + timedelta(seconds=expires_in - 60)
+        return self._access_token
+
+    # -- Graph API helpers ---------------------------------------------------
+
+    async def _graph_get(
+        self,
+        client: Any,
+        url: str,
+        params: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Perform a GET against Microsoft Graph with retry-after handling."""
+        token = await self._ensure_token()
+        headers = {"Authorization": f"Bearer {token}"}
+
+        for attempt in range(5):
+            resp = await client.get(url, headers=headers, params=params, timeout=30.0)
+
+            if resp.status_code == 429:
+                retry_after = int(resp.headers.get("Retry-After", _DEFAULT_RETRY_DELAY))
+                logger.warning(
+                    "Graph API rate-limited; sleeping %d s (attempt %d)",
+                    retry_after,
+                    attempt + 1,
+                )
+                await asyncio.sleep(retry_after)
+                continue
+
+            if resp.status_code == 401:
+                # Token may have expired mid-sync; force refresh.
+                self._access_token = None
+                token = await self._ensure_token()
+                headers = {"Authorization": f"Bearer {token}"}
+                continue
+
+            resp.raise_for_status()
+            return resp.json()
+
+        raise RuntimeError(f"Graph API request failed after 5 retries: {url}")
+
+    async def _graph_get_all_pages(
+        self,
+        client: Any,
+        url: str,
+        params: dict[str, Any] | None = None,
+        value_key: str = "value",
+    ) -> list[dict[str, Any]]:
+        """Paginate through all ``@odata.nextLink`` pages."""
+        items: list[dict[str, Any]] = []
+        next_url: str | None = url
+        current_params = params
+
+        while next_url:
+            data = await self._graph_get(client, next_url, params=current_params)
+            items.extend(data.get(value_key, []))
+            next_url = data.get("@odata.nextLink")
+            # After the first page the nextLink includes query params already.
+            current_params = None
+            await asyncio.sleep(_RATE_LIMIT_DELAY)
+
+        return items
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None,
+    ) -> list[RawEvent]:
+        """Fetch channel messages, chat messages, and meeting data from Teams."""
+        import httpx
+
+        if since is None:
+            since = datetime.now(timezone.utc) - timedelta(days=self.backfill_days)
+
+        since_iso = since.strftime("%Y-%m-%dT%H:%M:%SZ")
+        events: list[RawEvent] = []
+
+        async with httpx.AsyncClient() as client:
+            # --- Channel messages ---
+            for team_id in self.team_ids:
+                channel_events = await self._fetch_team_channels(
+                    client, team_id, since_iso
+                )
+                events.extend(channel_events)
+
+            # --- 1:1 and group chats ---
+            if self.include_chats:
+                chat_events = await self._fetch_chats(client, since_iso)
+                events.extend(chat_events)
+
+        return events
+
+    async def _fetch_team_channels(
+        self,
+        client: Any,
+        team_id: str,
+        since_iso: str,
+    ) -> list[RawEvent]:
+        """Fetch messages from all channels in a team."""
+        events: list[RawEvent] = []
+        channels_url = f"{_GRAPH_BASE}/teams/{team_id}/channels"
+        channels = await self._graph_get_all_pages(client, channels_url)
+
+        for channel in channels:
+            channel_id = channel["id"]
+            channel_name = channel.get("displayName", channel_id)
+            membership_type = channel.get("membershipType", "standard")
+
+            messages_url = (
+                f"{_GRAPH_BASE}/teams/{team_id}/channels/{channel_id}/messages"
+            )
+            params = {
+                "$top": str(_PAGE_SIZE),
+                "$filter": f"lastModifiedDateTime gt {since_iso}",
+            }
+
+            try:
+                messages = await self._graph_get_all_pages(
+                    client, messages_url, params=params
+                )
+            except Exception:
+                logger.warning(
+                    "Failed to fetch messages for channel %s/%s; skipping",
+                    team_id,
+                    channel_id,
+                )
+                continue
+
+            for msg in messages:
+                raw = self._message_to_raw_event(
+                    msg,
+                    context_type="channel",
+                    team_id=team_id,
+                    channel_id=channel_id,
+                    channel_name=channel_name,
+                    membership_type=membership_type,
+                )
+                events.append(raw)
+
+                # Fetch threaded replies.
+                reply_events = await self._fetch_replies(
+                    client, team_id, channel_id, msg["id"],
+                    channel_name, membership_type, since_iso,
+                )
+                events.extend(reply_events)
+
+        return events
+
+    async def _fetch_replies(
+        self,
+        client: Any,
+        team_id: str,
+        channel_id: str,
+        message_id: str,
+        channel_name: str,
+        membership_type: str,
+        since_iso: str,
+    ) -> list[RawEvent]:
+        """Fetch replies to a channel message."""
+        replies_url = (
+            f"{_GRAPH_BASE}/teams/{team_id}/channels/{channel_id}"
+            f"/messages/{message_id}/replies"
+        )
+        params = {"$top": str(_PAGE_SIZE)}
+
+        try:
+            replies = await self._graph_get_all_pages(client, replies_url, params=params)
+        except Exception:
+            logger.debug("Could not fetch replies for message %s", message_id)
+            return []
+
+        events: list[RawEvent] = []
+        for reply in replies:
+            created = reply.get("createdDateTime", "")
+            if created and created < since_iso:
+                continue
+            raw = self._message_to_raw_event(
+                reply,
+                context_type="channel_reply",
+                team_id=team_id,
+                channel_id=channel_id,
+                channel_name=channel_name,
+                membership_type=membership_type,
+                parent_message_id=message_id,
+            )
+            events.append(raw)
+        return events
+
+    async def _fetch_chats(
+        self,
+        client: Any,
+        since_iso: str,
+    ) -> list[RawEvent]:
+        """Fetch messages from 1:1 and group chats."""
+        events: list[RawEvent] = []
+        chats_url = f"{_GRAPH_BASE}/chats"
+        params = {"$top": str(_PAGE_SIZE), "$expand": "members"}
+
+        try:
+            chats = await self._graph_get_all_pages(client, chats_url, params=params)
+        except Exception:
+            logger.warning("Failed to fetch chats; skipping chat ingestion")
+            return events
+
+        for chat in chats:
+            chat_id = chat["id"]
+            chat_type = chat.get("chatType", "oneOnOne")
+            members = [
+                m.get("displayName", "")
+                for m in chat.get("members", [])
+            ]
+            member_ids = [
+                m.get("userId", "")
+                for m in chat.get("members", [])
+                if m.get("userId")
+            ]
+
+            messages_url = f"{_GRAPH_BASE}/chats/{chat_id}/messages"
+            params_msgs = {
+                "$top": str(_PAGE_SIZE),
+                "$filter": f"lastModifiedDateTime gt {since_iso}",
+            }
+
+            try:
+                messages = await self._graph_get_all_pages(
+                    client, messages_url, params=params_msgs
+                )
+            except Exception:
+                logger.debug("Could not fetch messages for chat %s", chat_id)
+                continue
+
+            for msg in messages:
+                raw = self._message_to_raw_event(
+                    msg,
+                    context_type="chat",
+                    chat_id=chat_id,
+                    chat_type=chat_type,
+                    chat_members=members,
+                    chat_member_ids=member_ids,
+                )
+                events.append(raw)
+
+        return events
+
+    # -- raw event construction ----------------------------------------------
+
+    def _message_to_raw_event(
+        self,
+        msg: dict[str, Any],
+        *,
+        context_type: str,
+        team_id: str | None = None,
+        channel_id: str | None = None,
+        channel_name: str | None = None,
+        membership_type: str | None = None,
+        parent_message_id: str | None = None,
+        chat_id: str | None = None,
+        chat_type: str | None = None,
+        chat_members: list[str] | None = None,
+        chat_member_ids: list[str] | None = None,
+    ) -> RawEvent:
+        """Transform a Graph API message object into a ``RawEvent``."""
+        msg_id = msg.get("id", "")
+        created = msg.get("createdDateTime", "")
+        timestamp = self._parse_datetime(created)
+
+        # Detect file attachments -> DOCUMENT type marker.
+        attachments = msg.get("attachments", [])
+        has_file = any(
+            a.get("contentType", "").startswith("reference")
+            or a.get("contentType") == "application/vnd.microsoft.card.adaptive"
+            for a in attachments
+        )
+
+        # Detect meeting-related messages.
+        event_detail = msg.get("eventDetail")
+        is_meeting = event_detail is not None and "call" in str(
+            event_detail.get("@odata.type", "")
+        ).lower()
+
+        data: dict[str, Any] = {
+            **msg,
+            "_context_type": context_type,
+            "_team_id": team_id,
+            "_channel_id": channel_id,
+            "_channel_name": channel_name,
+            "_membership_type": membership_type,
+            "_parent_message_id": parent_message_id,
+            "_chat_id": chat_id,
+            "_chat_type": chat_type,
+            "_chat_members": chat_members or [],
+            "_chat_member_ids": chat_member_ids or [],
+            "_has_file": has_file,
+            "_is_meeting": is_meeting,
+        }
+
+        # Build external_id for idempotency.
+        if context_type == "chat":
+            external_id = f"teams:chat:{chat_id}:{msg_id}"
+        else:
+            external_id = f"teams:channel:{team_id}:{channel_id}:{msg_id}"
+
+        # Determine permissions.
+        is_private = (
+            context_type == "chat"
+            or membership_type == "private"
+        )
+        allowed = tuple(chat_member_ids or []) if context_type == "chat" else ()
+
+        return RawEvent(
+            source="teams",
+            external_id=external_id,
+            timestamp=timestamp,
+            data=data,
+            permissions={
+                "is_private": is_private,
+                "allowed_principals": allowed,
+                "membership_type": membership_type or chat_type or "standard",
+            },
+        )
+
+    # -- CortexConnector interface: normalize --------------------------------
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert a Teams message ``RawEvent`` into a CortexDB ``Episode``."""
+        data = raw.data
+
+        # Determine episode type.
+        if data.get("_is_meeting"):
+            episode_type = EpisodeType.MEETING
+        elif data.get("_has_file"):
+            episode_type = EpisodeType.DOCUMENT
+        else:
+            episode_type = EpisodeType.MESSAGE
+
+        # Actor.
+        from_field = data.get("from", {}) or {}
+        user_field = from_field.get("user", {}) or {}
+        actor = Actor(
+            id=user_field.get("id", "unknown"),
+            name=user_field.get("displayName", "unknown"),
+            email=user_field.get("userPrincipalName"),
+        )
+
+        # Content: prefer plaintext body.
+        body = data.get("body", {}) or {}
+        content = body.get("content", "")
+        if body.get("contentType") == "html":
+            # Strip HTML tags for plain text storage.
+            content = self._strip_html(content)
+
+        # Entities.
+        entities = self._extract_entities(data)
+
+        # Tags.
+        context_type = data.get("_context_type", "channel")
+        tags = ["teams", context_type]
+        if episode_type == EpisodeType.MEETING:
+            tags.append("meeting")
+        if data.get("_has_file"):
+            tags.append("file_share")
+        if data.get("importance", "normal") == "high":
+            tags.append("high_importance")
+
+        # Thread ID: group channel replies under the parent message.
+        team_id = data.get("_team_id")
+        channel_id = data.get("_channel_id")
+        parent_msg = data.get("_parent_message_id")
+        chat_id = data.get("_chat_id")
+
+        if context_type == "channel_reply" and parent_msg:
+            thread_id = f"teams:{team_id}:{channel_id}:{parent_msg}"
+        elif context_type == "chat" and chat_id:
+            thread_id = f"teams:chat:{chat_id}"
+        else:
+            thread_id = None
+
+        parent_id = None
+        if context_type == "channel_reply" and parent_msg:
+            parent_id = f"teams:channel:{team_id}:{channel_id}:{parent_msg}"
+
+        # Metadata.
+        metadata: dict[str, Any] = {
+            "context_type": context_type,
+            "message_type": data.get("messageType", "message"),
+            "importance": data.get("importance", "normal"),
+        }
+        if team_id:
+            metadata["team_id"] = team_id
+        if channel_id:
+            metadata["channel_id"] = channel_id
+            metadata["channel_name"] = data.get("_channel_name", "")
+        if chat_id:
+            metadata["chat_id"] = chat_id
+            metadata["chat_type"] = data.get("_chat_type", "")
+        if data.get("attachments"):
+            metadata["attachment_count"] = len(data["attachments"])
+        mentions = data.get("mentions", [])
+        if mentions:
+            metadata["mentions"] = [
+                m.get("mentioned", {}).get("user", {}).get("displayName", "")
+                for m in mentions
+            ]
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=episode_type,
+            occurred_at=raw.timestamp,
+            content=content,
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=entities,
+            tags=tags,
+            metadata=metadata,
+            thread_id=thread_id,
+            parent_id=parent_id,
+            idempotency_key=raw.external_id,
+        )
+
+    # -- CortexConnector interface: map_permissions --------------------------
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map Teams channel/chat privacy to CortexDB visibility.
+
+        - Standard (public) channels -> ``ORGANIZATION``
+        - Private channels -> ``RESTRICTED`` (org-visible but flagged)
+        - 1:1 / group chats -> ``PRIVATE`` with member allow-list
+        """
+        perms = raw.permissions
+        if perms.get("is_private"):
+            principals = tuple(str(p) for p in perms.get("allowed_principals", ()))
+            if principals:
+                return Visibility(
+                    level=VisibilityLevel.PRIVATE,
+                    allowed_principals=principals,
+                )
+            return Visibility(level=VisibilityLevel.RESTRICTED)
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- helpers -------------------------------------------------------------
+
+    def _extract_entities(self, data: dict[str, Any]) -> list[EntityRef]:
+        """Extract entity references from a Teams message."""
+        entities: list[EntityRef] = []
+
+        team_id = data.get("_team_id")
+        if team_id:
+            entities.append(
+                EntityRef(
+                    entity_type="team",
+                    entity_id=team_id,
+                    display_name=f"team:{team_id}",
+                )
+            )
+
+        channel_id = data.get("_channel_id")
+        if channel_id:
+            entities.append(
+                EntityRef(
+                    entity_type="channel",
+                    entity_id=channel_id,
+                    display_name=data.get("_channel_name", channel_id),
+                )
+            )
+
+        chat_id = data.get("_chat_id")
+        if chat_id:
+            entities.append(
+                EntityRef(
+                    entity_type="chat",
+                    entity_id=chat_id,
+                    display_name=f"chat:{data.get('_chat_type', 'oneOnOne')}",
+                )
+            )
+
+        # Extract mentioned users.
+        for mention in data.get("mentions", []):
+            mentioned_user = mention.get("mentioned", {}).get("user", {})
+            if mentioned_user.get("id"):
+                entities.append(
+                    EntityRef(
+                        entity_type="user",
+                        entity_id=mentioned_user["id"],
+                        display_name=mentioned_user.get("displayName", ""),
+                    )
+                )
+
+        # Extract actors from chat members.
+        for member_id in data.get("_chat_member_ids", []):
+            entities.append(
+                EntityRef(
+                    entity_type="user",
+                    entity_id=member_id,
+                    display_name="",
+                )
+            )
+
+        return entities
+
+    @staticmethod
+    def _parse_datetime(iso_str: str) -> datetime:
+        """Parse an ISO 8601 datetime string from the Graph API."""
+        if not iso_str:
+            return datetime.now(timezone.utc)
+        # Graph API returns e.g. "2026-03-15T10:30:00.000Z"
+        cleaned = iso_str.replace("Z", "+00:00")
+        try:
+            return datetime.fromisoformat(cleaned)
+        except ValueError:
+            logger.debug("Could not parse datetime %r; using now()", iso_str)
+            return datetime.now(timezone.utc)
+
+    @staticmethod
+    def _strip_html(html: str) -> str:
+        """Naively strip HTML tags for plain-text extraction."""
+        import re
+
+        text = re.sub(r"<br\s*/?>", "\n", html)
+        text = re.sub(r"<[^>]+>", "", text)
+        text = text.strip()
+        return text