contextbase-plugin-microsoft-mail 0.2.6__py3-none-any.whl

plugin_microsoft_mail/sources/sync.py

@@ -0,0 +1,375 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+import dlt
+from shared_plugins.naming import (
+    dlt_resource_name,
+    dlt_source_name,
+    plugin_id_from_module,
+)
+from shared_plugins.resources import ctx_dlt_resource
+
+from ..models.ctx import (
+    MAIL_FOLDER_COLUMN_DESCRIPTIONS,
+    MESSAGE_COLUMN_DESCRIPTIONS,
+    MailFolderRow,
+    MessageRow,
+)
+from ..models.translators import (
+    mail_folder_rows_to_ctx_models,
+    message_rows_to_ctx_models,
+)
+from ..utils.client import (
+    SyncGraphMailClient,
+    graph_object_to_payload,
+)
+
+# Known unhandled correctness gaps:
+#
+# 1. Stale messages from removed/hidden folders. `apply_mail_folder_delta_rows`
+#    drops folder IDs from the active set when a folder is @removed or
+#    transitions to isHidden, and `messages` only iterates the active set.
+#    Existing message rows (and their attachment_content rows) under the dropped
+#    folder remain in the destination forever — no tombstone path covers this
+#    transition.
+#
+# 2. Attachment same-count replacements (lives in sources/attachments.py). The
+#    candidate query compares counts of materializable attachments vs. existing
+#    attachment_content rows; a message changing [att-1] -> [att-2] looks like
+#    1 == 1 and never re-materializes. The stale file row persists, the new
+#    attachment is missing.
+
+MAIL_FOLDER_DELTA_CURSOR_URL_KEY = "cursor_url"
+ACTIVE_MAIL_FOLDERS_KEY = "active_folders_by_id"
+MESSAGE_DELTA_CURSOR_URLS_BY_FOLDER_ID_KEY = "cursor_urls_by_folder_id"
+
+PLUGIN_ID = plugin_id_from_module(__file__)
+JOB = "sync"
+DELTA_PAGE_SIZE = 100
+DELTA_PREFER_HEADER = f'IdType="ImmutableId", odata.maxpagesize={DELTA_PAGE_SIZE}'
+MAIL_FOLDER_DELTA_QUERY_PARAMS = {
+    "$select": [
+        "id",
+        "displayName",
+        "parentFolderId",
+        "childFolderCount",
+        "totalItemCount",
+        "unreadItemCount",
+        "isHidden",
+    ],
+}
+MESSAGE_DELTA_ORDERBY = ["receivedDateTime desc"]
+INCLUDE_HIDDEN_MAIL_FOLDERS = False
+
+
+@dataclass(frozen=True)
+class MailFolderDeltaDrainResult:
+    mail_folder_rows: list[dict[str, Any]]
+    active_mail_folders_by_id: dict[str, dict[str, Any]]
+    cursor_url: str
+
+
+@dataclass(frozen=True)
+class DeltaPage:
+    rows: list[Any]
+    cursor_url: str
+
+
+def is_delta_cursor_url(cursor_url: str) -> bool:
+    query = parse_qs(urlparse(cursor_url).query)
+    return "$deltatoken" in query or "deltatoken" in query
+
+
+def drain_delta_pages(
+    *,
+    initial_cursor_url: str | None,
+    fetch_page: Callable[[str | None], Any],
+    rows_from_page: Callable[[Any], list[Any]],
+) -> Iterator[DeltaPage]:
+    cursor_url = initial_cursor_url
+
+    while True:
+        response = fetch_page(cursor_url)
+        rows = rows_from_page(response)
+
+        delta_link = getattr(response, "odata_delta_link", None)
+        if delta_link:
+            yield DeltaPage(rows=rows, cursor_url=delta_link)
+            return
+
+        next_link = getattr(response, "odata_next_link", None)
+        if not next_link:
+            raise RuntimeError(
+                "Graph delta response did not include @odata.nextLink or "
+                "@odata.deltaLink"
+            )
+
+        yield DeltaPage(rows=rows, cursor_url=next_link)
+        cursor_url = next_link
+
+
+def message_delta_cursor_urls_for_active_mail_folders(
+    previous_message_cursor_urls_by_folder_id: dict[str, str],
+    active_mail_folder_ids: list[str],
+) -> dict[str, str]:
+    active_mail_folder_id_set = set(active_mail_folder_ids)
+    return {
+        folder_id: cursor_url
+        for folder_id, cursor_url in previous_message_cursor_urls_by_folder_id.items()
+        if folder_id in active_mail_folder_id_set
+    }
+
+
+def message_delta_cursor_urls_with_cursor_for_folder(
+    message_cursor_urls_by_folder_id: dict[str, str],
+    *,
+    folder_id: str,
+    cursor_url: str,
+) -> dict[str, str]:
+    return {
+        **message_cursor_urls_by_folder_id,
+        folder_id: cursor_url,
+    }
+
+
+def should_include_mail_folder_payload(payload: dict[str, Any]) -> bool:
+    if payload.get("@removed") is not None:
+        return False
+    if not INCLUDE_HIDDEN_MAIL_FOLDERS and payload.get("isHidden"):
+        return False
+    return True
+
+
+def mail_folder_delta_page_rows(response: Any) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for folder in getattr(response, "value", None) or []:
+        row = graph_object_to_payload(folder)
+        if not row.get("id"):
+            raise RuntimeError(
+                f"Graph mail-folder delta row missing id: keys={sorted(row)}"
+            )
+        rows.append(row)
+    return rows
+
+
+def apply_mail_folder_delta_rows(
+    *,
+    previous_active_mail_folders_by_id: dict[str, dict[str, Any]],
+    mail_folder_rows: list[dict[str, Any]],
+) -> dict[str, dict[str, Any]]:
+    active_mail_folders_by_id = dict(previous_active_mail_folders_by_id)
+
+    for row in mail_folder_rows:
+        folder_id = str(row["id"])
+        if should_include_mail_folder_payload(row):
+            active_mail_folders_by_id[folder_id] = {
+                "id": folder_id,
+                "display_name": row.get("displayName"),
+                "parent_folder_id": row.get("parentFolderId"),
+                "is_hidden": row.get("isHidden"),
+            }
+        else:
+            active_mail_folders_by_id.pop(folder_id, None)
+
+    return active_mail_folders_by_id
+
+
+def drain_mail_folder_delta(
+    *,
+    client: SyncGraphMailClient,
+    previous_cursor_url: str | None,
+    previous_active_mail_folders_by_id: dict[str, dict[str, Any]],
+) -> MailFolderDeltaDrainResult:
+    mail_folder_rows: list[dict[str, Any]] = []
+    drained_cursor_url: str | None = None
+
+    for page in drain_delta_pages(
+        initial_cursor_url=previous_cursor_url,
+        fetch_page=lambda cursor_url: client.get_folder_delta_page(
+            delta_url=cursor_url,
+            query_params=MAIL_FOLDER_DELTA_QUERY_PARAMS,
+            prefer_header=DELTA_PREFER_HEADER,
+        ),
+        rows_from_page=mail_folder_delta_page_rows,
+    ):
+        mail_folder_rows.extend(page.rows)
+        drained_cursor_url = page.cursor_url
+
+    if drained_cursor_url is None or not is_delta_cursor_url(drained_cursor_url):
+        raise RuntimeError("mail folder delta drain did not finish with a delta cursor")
+
+    return MailFolderDeltaDrainResult(
+        mail_folder_rows=mail_folder_rows,
+        active_mail_folders_by_id=apply_mail_folder_delta_rows(
+            previous_active_mail_folders_by_id=previous_active_mail_folders_by_id,
+            mail_folder_rows=mail_folder_rows,
+        ),
+        cursor_url=drained_cursor_url,
+    )
+
+
+def message_delta_page_rows(
+    *,
+    response: Any,
+    folder_id: str,
+) -> list[Any]:
+    # Don't pre-validate parent_folder_id — @removed rows carry only id +
+    # @removed; the translator detects them via additional_data and emits a
+    # tombstone using folder_id from context. Live rows with no
+    # parent_folder_id fail pydantic validation at MessageRow.parent_folder_id.
+    rows: list[Any] = []
+    for message in getattr(response, "value", None) or []:
+        message_id = getattr(message, "id", None)
+        if not message_id:
+            raise RuntimeError(
+                f"Graph message delta row missing id (folder_id={folder_id!r})"
+            )
+        rows.append(message)
+    return rows
+
+
+def drain_message_delta_pages_for_folder(
+    *,
+    client: SyncGraphMailClient,
+    folder_id: str,
+    previous_cursor_url: str | None,
+    initial_message_delta_top: int | None,
+) -> Iterator[DeltaPage]:
+    yield from drain_delta_pages(
+        initial_cursor_url=previous_cursor_url,
+        fetch_page=lambda cursor_url: client.get_message_delta_page(
+            folder_id=folder_id,
+            delta_url=cursor_url,
+            query_params={
+                "$orderby": MESSAGE_DELTA_ORDERBY,
+                "$top": initial_message_delta_top,
+                "$expand": [
+                    "attachments($select=id,name,contentType,size,isInline,lastModifiedDateTime)"
+                ],
+            },
+            prefer_header=DELTA_PREFER_HEADER,
+        ),
+        rows_from_page=lambda response: message_delta_page_rows(
+            response=response,
+            folder_id=folder_id,
+        ),
+    )
+
+
+@dlt.source(name=dlt_source_name(PLUGIN_ID, JOB))
+def microsoft_mail_source(
+    binding_id: str,
+    *,
+    client: SyncGraphMailClient,
+    initial_message_delta_top: int | None = None,
+) -> tuple[Any, ...]:
+    folder_drain_cache: MailFolderDeltaDrainResult | None = None
+
+    def get_mail_folder_delta_snapshot() -> MailFolderDeltaDrainResult:
+        nonlocal folder_drain_cache
+        if folder_drain_cache is not None:
+            return folder_drain_cache
+
+        source_state = dlt.current.source_state()
+        previous_cursor_url = source_state.get(MAIL_FOLDER_DELTA_CURSOR_URL_KEY)
+        previous_active_mail_folders_by_id = dict(
+            source_state.get(ACTIVE_MAIL_FOLDERS_KEY) or {}
+        )
+
+        result = drain_mail_folder_delta(
+            client=client,
+            previous_cursor_url=previous_cursor_url,
+            previous_active_mail_folders_by_id=previous_active_mail_folders_by_id,
+        )
+
+        source_state[MAIL_FOLDER_DELTA_CURSOR_URL_KEY] = result.cursor_url
+        source_state[ACTIVE_MAIL_FOLDERS_KEY] = result.active_mail_folders_by_id
+
+        folder_drain_cache = result
+        return result
+
+    @ctx_dlt_resource(
+        name=dlt_resource_name("mail_folders"),
+        write_disposition="merge",
+        primary_key=("_ctx_binding_id", "id"),
+        columns={
+            **MAIL_FOLDER_COLUMN_DESCRIPTIONS,
+            "_ctx_deleted": {"hard_delete": True},
+        },
+    )
+    def mail_folders() -> Iterator[MailFolderRow]:
+        snapshot = get_mail_folder_delta_snapshot()
+        yield from mail_folder_rows_to_ctx_models(
+            binding_id,
+            snapshot.mail_folder_rows,
+        )
+
+    @ctx_dlt_resource(
+        name=dlt_resource_name("messages"),
+        write_disposition="merge",
+        primary_key=("_ctx_binding_id", "id", "parent_folder_id"),
+        columns={
+            **MESSAGE_COLUMN_DESCRIPTIONS,
+            "_ctx_deleted": {"hard_delete": True},
+        },
+    )
+    def messages() -> Iterator[MessageRow]:
+        snapshot = get_mail_folder_delta_snapshot()
+        active_mail_folder_ids = sorted(snapshot.active_mail_folders_by_id)
+
+        source_state = dlt.current.source_state()
+        previous_message_cursor_urls_by_folder_id = (
+            message_delta_cursor_urls_for_active_mail_folders(
+                dict(
+                    source_state.get(MESSAGE_DELTA_CURSOR_URLS_BY_FOLDER_ID_KEY) or {}
+                ),
+                active_mail_folder_ids,
+            )
+        )
+        updated_message_cursor_urls_by_folder_id = dict(
+            previous_message_cursor_urls_by_folder_id,
+        )
+
+        for folder_id in active_mail_folder_ids:
+            previous_message_cursor_url = previous_message_cursor_urls_by_folder_id.get(
+                folder_id
+            )
+            message_cursor_url: str | None = previous_message_cursor_url
+
+            for page in drain_message_delta_pages_for_folder(
+                client=client,
+                folder_id=folder_id,
+                previous_cursor_url=previous_message_cursor_url,
+                initial_message_delta_top=initial_message_delta_top,
+            ):
+                message_cursor_url = page.cursor_url
+                yield from message_rows_to_ctx_models(
+                    binding_id,
+                    page.rows,
+                    folder_id=folder_id,
+                )
+
+            if message_cursor_url is None or not is_delta_cursor_url(
+                message_cursor_url
+            ):
+                raise RuntimeError(
+                    "message delta drain did not finish with a delta cursor"
+                )
+
+            updated_message_cursor_urls_by_folder_id = (
+                message_delta_cursor_urls_with_cursor_for_folder(
+                    updated_message_cursor_urls_by_folder_id,
+                    folder_id=folder_id,
+                    cursor_url=message_cursor_url,
+                )
+            )
+
+        source_state[MESSAGE_DELTA_CURSOR_URLS_BY_FOLDER_ID_KEY] = (
+            updated_message_cursor_urls_by_folder_id
+        )
+
+    return (mail_folders, messages)

plugin_microsoft_mail/utils/__init__.py

@@ -0,0 +1 @@
+"""Utilities for Microsoft Mail Graph and DLT spikes."""

plugin_microsoft_mail/utils/attachments.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import hashlib
+from base64 import b64decode
+from collections.abc import Mapping, Sequence
+from pathlib import PurePosixPath
+from typing import Any
+
+from shared_plugins.scratch import replace_scratch_dir_files
+
+from ..models.ctx import AttachmentContentRow
+from ..models.translators import attachment_content_row_from_graph_payload
+
+REFERENCE_ATTACHMENT_ODATA_TYPE = "#microsoft.graph.referenceAttachment"
+ITEM_ATTACHMENT_ODATA_TYPE = "#microsoft.graph.itemAttachment"
+FILE_ATTACHMENT_ODATA_TYPE = "#microsoft.graph.fileAttachment"
+KNOWN_ATTACHMENT_ODATA_TYPES = frozenset(
+    {
+        FILE_ATTACHMENT_ODATA_TYPE,
+        REFERENCE_ATTACHMENT_ODATA_TYPE,
+        ITEM_ATTACHMENT_ODATA_TYPE,
+    }
+)
+
+
+def _hash_path_segment(*parts: str) -> str:
+    hash_input = "\n".join(parts)
+    return hashlib.sha256(hash_input.encode("utf-8")).hexdigest()
+
+
+def _build_deterministic_file_name(
+    *,
+    attachment_id: str,
+    name: str | None,
+) -> str:
+    digest = _hash_path_segment(attachment_id, name or "")
+    suffix = ""
+    if name:
+        suffix = "".join(PurePosixPath(name.strip()).suffixes)
+    return f"{digest}{suffix}"
+
+
+def materialize_attachment_payloads(
+    *,
+    binding_id: str,
+    message_id: str,
+    attachment_payloads: Sequence[Mapping[str, Any]],
+) -> list[AttachmentContentRow]:
+    """Decode each payload's `contentBytes`, write all of a message's
+    attachments into the per-message scratch directory atomically, and return
+    one AttachmentContentRow per payload in input order.
+
+    All payloads must include `id` and `contentBytes` (base64). Reference and
+    item attachments are excluded upstream — this helper is for materializable
+    file attachments only.
+
+    Writes are batched into a single `replace_scratch_dir_files` call because
+    that helper atomically REPLACES the entire `relative_dir`; per-attachment
+    calls would clobber each other within the same message.
+    """
+    if len(attachment_payloads) == 0:
+        return []
+
+    file_name_by_attachment_id: dict[str, str] = {}
+    files: dict[str, bytes] = {}
+    for payload in attachment_payloads:
+        attachment_id = payload.get("id")
+        if not isinstance(attachment_id, str):
+            raise RuntimeError(
+                "attachment payload missing id "
+                f"message_id={message_id} keys={sorted(payload)}"
+            )
+        content_bytes_b64 = payload.get("contentBytes")
+        if not isinstance(content_bytes_b64, str):
+            raise RuntimeError(
+                "attachment payload missing contentBytes "
+                f"message_id={message_id} attachment_id={attachment_id}"
+            )
+
+        file_name = _build_deterministic_file_name(
+            attachment_id=attachment_id,
+            name=(
+                payload.get("name") if isinstance(payload.get("name"), str) else None
+            ),
+        )
+        file_name_by_attachment_id[attachment_id] = file_name
+        files[file_name] = b64decode(content_bytes_b64)
+
+    path_by_file_name = replace_scratch_dir_files(
+        binding_id=binding_id,
+        relative_dir=f"attachments/{_hash_path_segment(message_id)}",
+        files=files,
+    )
+
+    rows: list[AttachmentContentRow] = []
+    for payload in attachment_payloads:
+        attachment_id = payload["id"]
+        file_name = file_name_by_attachment_id[attachment_id]
+        rows.append(
+            attachment_content_row_from_graph_payload(
+                binding_id=binding_id,
+                message_id=message_id,
+                attachment_payload=payload,
+                file_path=path_by_file_name[file_name],
+            )
+        )
+    return rows