contextbase-plugin-microsoft-mail 0.2.6__py3-none-any.whl

plugin_microsoft_mail/models/translators.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator, Mapping
+from typing import Any
+
+from ..utils.client import graph_object_to_payload
+from .ctx import (
+    AttachmentContentRow,
+    MailFolderRow,
+    MessageRow,
+)
+
+
+def _payload(item: object) -> dict[str, Any]:
+    return graph_object_to_payload(item)
+
+
+def _list_value(payload: Mapping[str, Any], key: str) -> list[Any]:
+    value = payload.get(key)
+    if value is None:
+        return []
+    if not isinstance(value, list):
+        raise TypeError(
+            f"translator: field {key!r} must be a list, got {type(value).__name__}"
+        )
+    return list(value)
+
+
+def _dict_value(payload: Mapping[str, Any], key: str) -> dict[str, Any] | None:
+    value = payload.get(key)
+    if value is None:
+        return None
+    if not isinstance(value, dict):
+        raise TypeError(
+            f"translator: field {key!r} must be a dict, got {type(value).__name__}"
+        )
+    return dict(value)
+
+
+def mail_folder_rows_to_ctx_models(
+    binding_id: str,
+    rows: Iterable[object],
+) -> Iterator[MailFolderRow]:
+    """Translate Graph delta mail-folder rows to MailFolderRow instances.
+
+    `@removed` rows from Graph carry only `id` + the `@removed` marker; they
+    become tombstones (`ctx_deleted=True`) so dlt's `hard_delete` deletes the
+    matching row at merge time. Live rows pass through with full field set.
+    """
+    for row in rows:
+        payload = _payload(row)
+        additional_data = _dict_value(payload, "additional_data") or {}
+        if "@removed" in additional_data or "@removed" in payload:
+            yield MailFolderRow(
+                ctx_binding_id=binding_id,
+                id=payload.get("id"),
+                ctx_deleted=True,
+            )
+            continue
+
+        yield MailFolderRow(
+            ctx_binding_id=binding_id,
+            id=payload.get("id"),
+            odata_type=payload.get("@odata.type"),
+            additional_data=additional_data,
+            child_folder_count=payload.get("childFolderCount"),
+            child_folders=_list_value(payload, "childFolders"),
+            display_name=payload.get("displayName"),
+            is_hidden=payload.get("isHidden"),
+            message_rules=_list_value(payload, "messageRules"),
+            messages=_list_value(payload, "messages"),
+            multi_value_extended_properties=_list_value(
+                payload,
+                "multiValueExtendedProperties",
+            ),
+            parent_folder_id=payload.get("parentFolderId"),
+            single_value_extended_properties=_list_value(
+                payload,
+                "singleValueExtendedProperties",
+            ),
+            total_item_count=payload.get("totalItemCount"),
+            unread_item_count=payload.get("unreadItemCount"),
+        )
+
+
+def message_rows_to_ctx_models(
+    binding_id: str,
+    rows: Iterable[object],
+    *,
+    folder_id: str | None = None,
+) -> Iterator[MessageRow]:
+    """Translate Graph delta message rows to MessageRow instances.
+
+    `folder_id` is the folder whose delta produced these rows. It's used to
+    populate `parent_folder_id` on tombstone rows (`@removed` entries from Graph
+    contain only `id` + `@removed` and don't carry the folder).
+    """
+    for row in rows:
+        payload = _payload(row)
+        additional_data = _dict_value(payload, "additional_data") or {}
+        if "@removed" in additional_data or "@removed" in payload:
+            if folder_id is None:
+                raise RuntimeError(
+                    "@removed message row received without folder_id context "
+                    f"message_id={payload.get('id')!r}"
+                )
+            yield MessageRow(
+                ctx_binding_id=binding_id,
+                id=payload.get("id"),
+                parent_folder_id=folder_id,
+                ctx_deleted=True,
+            )
+            continue
+
+        yield MessageRow(
+            ctx_binding_id=binding_id,
+            ctx_source_updated_at=payload.get("lastModifiedDateTime"),
+            id=payload.get("id"),
+            odata_type=payload.get("@odata.type"),
+            etag=payload.get("@odata.etag"),
+            additional_data=additional_data,
+            attachments=_list_value(payload, "attachments"),
+            bcc_recipients=_list_value(payload, "bccRecipients"),
+            body=_dict_value(payload, "body"),
+            body_preview=payload.get("bodyPreview"),
+            categories=_list_value(payload, "categories"),
+            cc_recipients=_list_value(payload, "ccRecipients"),
+            change_key=payload.get("changeKey"),
+            conversation_id=payload.get("conversationId"),
+            conversation_index=payload.get("conversationIndex"),
+            created_date_time=payload.get("createdDateTime"),
+            extensions=_list_value(payload, "extensions"),
+            flag=_dict_value(payload, "flag"),
+            from_=_dict_value(payload, "from"),
+            has_attachments=payload.get("hasAttachments"),
+            importance=payload.get("importance"),
+            inference_classification=payload.get("inferenceClassification"),
+            internet_message_headers=_list_value(
+                payload,
+                "internetMessageHeaders",
+            ),
+            internet_message_id=payload.get("internetMessageId"),
+            is_delivery_receipt_requested=payload.get("isDeliveryReceiptRequested"),
+            is_draft=payload.get("isDraft"),
+            is_read=payload.get("isRead"),
+            is_read_receipt_requested=payload.get("isReadReceiptRequested"),
+            last_modified_date_time=payload.get("lastModifiedDateTime"),
+            multi_value_extended_properties=_list_value(
+                payload,
+                "multiValueExtendedProperties",
+            ),
+            parent_folder_id=payload.get("parentFolderId"),
+            received_date_time=payload.get("receivedDateTime"),
+            reply_to=_list_value(payload, "replyTo"),
+            sender=_dict_value(payload, "sender"),
+            sent_date_time=payload.get("sentDateTime"),
+            single_value_extended_properties=_list_value(
+                payload,
+                "singleValueExtendedProperties",
+            ),
+            subject=payload.get("subject"),
+            to_recipients=_list_value(payload, "toRecipients"),
+            unique_body=_dict_value(payload, "uniqueBody"),
+            web_link=payload.get("webLink"),
+        )
+
+
+def attachment_content_row_from_graph_payload(
+    *,
+    binding_id: str,
+    message_id: str,
+    attachment_payload: Mapping[str, Any],
+    file_path: str,
+) -> AttachmentContentRow:
+    """Build an AttachmentContentRow from a Graph attachment object and a
+    locally-materialized file path."""
+    last_modified = attachment_payload.get("lastModifiedDateTime")
+    return AttachmentContentRow(
+        ctx_binding_id=binding_id,
+        ctx_source_updated_at=last_modified,
+        message_id=message_id,
+        attachment_id=attachment_payload.get("id"),
+        odata_type=attachment_payload.get("@odata.type"),
+        media_content_type=attachment_payload.get("@odata.mediaContentType"),
+        name=attachment_payload.get("name"),
+        content_type=attachment_payload.get("contentType"),
+        size=attachment_payload.get("size"),
+        is_inline=attachment_payload.get("isInline"),
+        content_id=attachment_payload.get("contentId"),
+        content_location=attachment_payload.get("contentLocation"),
+        last_modified_date_time=last_modified,
+        file_path=file_path,
+    )

plugin_microsoft_mail/plugin.json

@@ -0,0 +1,7 @@
+{
+	"auth": {
+		"type": "client_credentials"
+	},
+	"mode": "dagster",
+	"plugin_id": "microsoft_mail"
+}

plugin_microsoft_mail/sources/__init__.py

@@ -0,0 +1 @@
+"""DLT sources for Microsoft Mail."""

plugin_microsoft_mail/sources/attachments.py

@@ -0,0 +1,407 @@
+from __future__ import annotations
+
+import logging
+import time
+import uuid
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from typing import Any, Literal, Self
+
+import dlt
+from dlt.destinations.sql_client import SqlClientBase
+from pydantic import model_validator
+from shared_plugins.dlt import destination_has_table
+from shared_plugins.models import IdStr, StrictModel
+from shared_plugins.naming import (
+    dlt_resource_name,
+    dlt_source_name,
+    plugin_id_from_module,
+)
+from shared_plugins.resources import ctx_dlt_resource
+
+from ..models.ctx import (
+    ATTACHMENT_CONTENT_COLUMN_DESCRIPTIONS,
+    AttachmentContentRow,
+)
+from ..utils.attachments import (
+    FILE_ATTACHMENT_ODATA_TYPE,
+    KNOWN_ATTACHMENT_ODATA_TYPES,
+    materialize_attachment_payloads,
+)
+from ..utils.client import (
+    SyncGraphMailClient,
+    graph_object_to_payload,
+)
+
+PLUGIN_ID = plugin_id_from_module(__file__)
+JOB = "attachment_content"
+SOURCE_NAME = dlt_source_name(PLUGIN_ID, JOB)
+LOGGER = logging.getLogger(__name__)
+DEFAULT_CANDIDATE_LIMIT = 200
+ATTACHMENT_PREFER_HEADER = 'IdType="ImmutableId"'
+
+
+@dataclass(frozen=True)
+class Candidate:
+    action: str  # 'materialize' or 'orphan'
+    message_id: str
+    attachments: list[dict[str, Any]] | None = None  # only for 'materialize'
+
+
+class AttachmentCandidateProjection(StrictModel):
+    action: Literal["materialize", "orphan"]
+    message_id: IdStr
+    attachments: list[dict[str, Any]] | None = None
+
+    @model_validator(mode="after")
+    def _validate_payload_for_action(self) -> Self:
+        if self.action == "materialize" and self.attachments is None:
+            raise ValueError(
+                "materialize attachment candidate requires attachments payload"
+            )
+        if self.action == "orphan" and self.attachments is not None:
+            raise ValueError(
+                "orphan attachment candidate must not include attachments payload"
+            )
+        return self
+
+    def to_candidate(self) -> Candidate:
+        return Candidate(
+            action=self.action,
+            message_id=self.message_id,
+            attachments=(
+                [dict(attachment) for attachment in self.attachments]
+                if self.attachments is not None
+                else None
+            ),
+        )
+
+
+def parse_attachment_candidate(row: Mapping[str, Any]) -> Candidate:
+    return AttachmentCandidateProjection.model_validate(dict(row)).to_candidate()
+
+
+def build_materialize_arm_query() -> str:
+    return """
+WITH latest_message AS (
+  SELECT DISTINCT ON (_ctx_binding_id, id)
+    _ctx_binding_id,
+    id,
+    attachments
+  FROM messages
+  WHERE _ctx_binding_id = %s
+  ORDER BY _ctx_binding_id, id, last_modified_date_time DESC NULLS LAST
+)
+SELECT
+  'materialize'::text AS action,
+  m.id AS message_id,
+  m.attachments AS attachments
+FROM latest_message AS m
+WHERE (
+    SELECT count(*)
+    FROM jsonb_array_elements(COALESCE(m.attachments, '[]'::jsonb)) AS a
+    WHERE COALESCE(a->>'@odata.type', '') NOT IN (
+      '#microsoft.graph.referenceAttachment',
+      '#microsoft.graph.itemAttachment'
+    )
+  ) <> (
+    SELECT count(*)
+    FROM attachment_content AS c
+    WHERE c._ctx_binding_id = m._ctx_binding_id
+      AND c.message_id = m.id
+  )
+""".strip()
+
+
+def build_orphan_arm_query() -> str:
+    return """
+SELECT
+  'orphan'::text AS action,
+  c.message_id,
+  NULL::jsonb AS attachments
+FROM attachment_content AS c
+LEFT JOIN messages AS m
+  ON m._ctx_binding_id = c._ctx_binding_id
+ AND m.id = c.message_id
+WHERE c._ctx_binding_id = %s
+  AND m.id IS NULL
+GROUP BY c.message_id
+""".strip()
+
+
+def build_bootstrap_materialize_arm_query() -> str:
+    """Materialize-arm variant for the case where `attachment_content` table
+    doesn't exist yet (first run after a clean reseed). Hardcodes the
+    existing-count subquery to 0 instead of selecting from `attachment_content`,
+    so the candidate query doesn't fail on a missing relation.
+    """
+    return """
+WITH latest_message AS (
+  SELECT DISTINCT ON (_ctx_binding_id, id)
+    _ctx_binding_id,
+    id,
+    attachments
+  FROM messages
+  WHERE _ctx_binding_id = %s
+  ORDER BY _ctx_binding_id, id, last_modified_date_time DESC NULLS LAST
+)
+SELECT
+  'materialize'::text AS action,
+  m.id AS message_id,
+  m.attachments AS attachments
+FROM latest_message AS m
+WHERE (
+    SELECT count(*)
+    FROM jsonb_array_elements(m.attachments) AS a
+    WHERE COALESCE(a->>'@odata.type', '') NOT IN (
+      '#microsoft.graph.referenceAttachment',
+      '#microsoft.graph.itemAttachment'
+    )
+  ) > 0
+""".strip()
+
+
+def build_combined_candidate_query(*, limit: int) -> str:
+    materialize = build_materialize_arm_query()
+    orphan = build_orphan_arm_query()
+    return f"""
+WITH candidates AS (
+{materialize}
+UNION ALL
+{orphan}
+)
+SELECT action, message_id, attachments
+FROM candidates
+ORDER BY message_id ASC
+LIMIT %s
+""".strip()
+
+
+def iter_candidates(
+    sql_client: SqlClientBase[Any],
+    *,
+    binding_id: str,
+    limit: int,
+) -> list[Candidate]:
+    if not destination_has_table(sql_client, "messages"):
+        raise RuntimeError(
+            "messages table does not exist yet; run microsoft-mail-dlt-sync first"
+        )
+
+    if not destination_has_table(sql_client, "attachment_content"):
+        # Bootstrap: orphan arm has no rows AND we can't reference
+        # attachment_content yet. Use the bootstrap materialize-arm SQL.
+        query = (
+            build_bootstrap_materialize_arm_query()
+            + "\nORDER BY message_id ASC\nLIMIT %s"
+        )
+        params = (binding_id, limit)
+    else:
+        query = build_combined_candidate_query(limit=limit)
+        params = (binding_id, binding_id, limit)
+
+    candidates: list[Candidate] = []
+    with sql_client.execute_query(query, *params) as cursor:
+        if cursor.description is None:
+            return candidates
+        columns = [c[0] for c in cursor.description]
+        for row in cursor.fetchall():
+            raw = dict(zip(columns, row))
+            candidates.append(parse_attachment_candidate(raw))
+    return candidates
+
+
+def _materializable_attachments(
+    attachments: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """Filter to file attachments. Reference and item attachments produce no
+    row in `attachment_content`. An unknown `@odata.type` is treated as a loud
+    error per spec §8 — Graph's attachment subtype set is closed."""
+    materializable: list[dict[str, Any]] = []
+    for a in attachments:
+        odata_type = a.get("@odata.type")
+        if odata_type not in KNOWN_ATTACHMENT_ODATA_TYPES:
+            raise RuntimeError(
+                f"unknown attachment @odata.type {odata_type!r}; "
+                f"expected one of {sorted(KNOWN_ATTACHMENT_ODATA_TYPES)}"
+            )
+        if odata_type == FILE_ATTACHMENT_ODATA_TYPE:
+            materializable.append(a)
+    return materializable
+
+
+def _full_attachment_payload(obj: Any) -> dict[str, Any]:
+    """Build the materialization payload for a Graph FileAttachment object.
+
+    Kiota's Python deserializer stores `FileAttachment.content_bytes` as the
+    ASCII bytes of the base64 string from Graph's wire JSON — not as the
+    decoded file bytes. Routing it through `graph_object_to_payload` (which
+    uses Kiota's JSON writer) base64-encodes those ASCII bytes a second time,
+    so a single `b64decode` downstream would produce the base64 string back
+    instead of the real file bytes. Override `contentBytes` with the original
+    base64 string so the single `b64decode` in `materialize_attachment_payloads`
+    yields real bytes. (Verified empirically — see scratch/smoke_real_graph.py.)
+    """
+    payload = graph_object_to_payload(obj)
+    kiota_content_bytes = getattr(obj, "content_bytes", None)
+    if isinstance(kiota_content_bytes, (bytes, bytearray)):
+        payload["contentBytes"] = bytes(kiota_content_bytes).decode("ascii")
+    return payload
+
+
+def fetch_and_emit_for_message(
+    *,
+    binding_id: str,
+    client: SyncGraphMailClient,
+    candidate: Candidate,
+) -> Iterator[AttachmentContentRow]:
+    if candidate.attachments is None:
+        raise RuntimeError(
+            f"materialize candidate without attachments payload message_id={candidate.message_id}"
+        )
+
+    full_payloads: list[dict[str, Any]] = []
+    for raw_attachment in _materializable_attachments(candidate.attachments):
+        attachment_id = raw_attachment.get("id")
+        if not isinstance(attachment_id, str):
+            raise RuntimeError(
+                "attachment in messages.attachments has no id "
+                f"message_id={candidate.message_id} payload_keys={sorted(raw_attachment)}"
+            )
+        full = client.get_attachment_full(
+            message_id=candidate.message_id,
+            attachment_id=attachment_id,
+            prefer_header=ATTACHMENT_PREFER_HEADER,
+        )
+        full_payloads.append(_full_attachment_payload(full))
+
+    rows = materialize_attachment_payloads(
+        binding_id=binding_id,
+        message_id=candidate.message_id,
+        attachment_payloads=full_payloads,
+    )
+    yield from rows
+
+
+TOMBSTONE_ATTACHMENT_ID_PREFIX = "_ctx_tombstone:"
+
+
+def _generate_tombstone_attachment_id() -> str:
+    """Build a unique sentinel `attachment_id` for a tombstone row.
+
+    The prefix marks intent; the random uuid4 suffix prevents any conceivable
+    collision with a real Graph attachment id and guarantees uniqueness
+    across multiple tombstones in the same merge.
+    """
+    return f"{TOMBSTONE_ATTACHMENT_ID_PREFIX}{uuid.uuid4()}"
+
+
+def emit_orphan_tombstone(
+    *,
+    binding_id: str,
+    candidate: Candidate,
+) -> AttachmentContentRow:
+    """Build a tombstone row that dlt will use to delete all attachment_content
+    rows matching merge_key=(_ctx_binding_id, message_id).
+
+    The PK includes `attachment_id` (Postgres requires PK columns NOT NULL),
+    so the tombstone carries a sentinel id with the form
+    `_ctx_tombstone:<uuid4>`. dlt's `hard_delete` on `_ctx_deleted` deletes
+    the sentinel-bearing row at the end of the merge, so the sentinel never
+    persists in the destination."""
+    return AttachmentContentRow(
+        ctx_binding_id=binding_id,
+        message_id=candidate.message_id,
+        attachment_id=_generate_tombstone_attachment_id(),
+        ctx_deleted=True,
+    )
+
+
+@dlt.source(name=SOURCE_NAME)
+def microsoft_mail_attachment_source(
+    binding_id: str,
+    *,
+    client: SyncGraphMailClient,
+) -> tuple[Any, ...]:
+    @ctx_dlt_resource(
+        name=dlt_resource_name("attachment_content"),
+        write_disposition={"disposition": "merge", "strategy": "delete-insert"},
+        primary_key=("_ctx_binding_id", "message_id", "attachment_id"),
+        merge_key=("_ctx_binding_id", "message_id"),
+        columns={
+            **ATTACHMENT_CONTENT_COLUMN_DESCRIPTIONS,
+            # file_path is required on live rows but NULL on tombstones
+            # (ctx_deleted=True). dlt infers schema from first-observed data,
+            # so we declare it nullable upfront to match the pydantic Optional
+            # type. attachment_id stays NOT NULL because it's part of the PK
+            # — tombstones carry a sentinel value (TOMBSTONE_ATTACHMENT_ID).
+            "file_path": {"nullable": True},
+            "_ctx_deleted": {"data_type": "bool", "hard_delete": True},
+        },
+    )
+    def attachment_content() -> Iterator[AttachmentContentRow]:
+        # `schema_name=SOURCE_NAME` avoids dlt's `Schema(pipeline_name)`
+        # fallback inside `_get_schema_or_create`. The pipeline_name for a
+        # per-binding attachment_content pipeline routinely exceeds dlt's
+        # 64-char Schema-name limit, and the source schema is what we
+        # actually want here. `run_dlt_pipeline` pre-registers an empty
+        # schema with this name on cold start so the lookup succeeds.
+        with dlt.current.pipeline().sql_client(
+            schema_name=SOURCE_NAME,
+        ) as sql_client:
+            candidates = iter_candidates(
+                sql_client,
+                binding_id=binding_id,
+                limit=DEFAULT_CANDIDATE_LIMIT,
+            )
+
+        run_started = time.monotonic()
+        materialized_messages = 0
+        materialized_rows = 0
+        orphaned_messages = 0
+
+        for candidate in candidates:
+            if candidate.action == "orphan":
+                orphaned_messages += 1
+                yield emit_orphan_tombstone(binding_id=binding_id, candidate=candidate)
+                continue
+
+            if candidate.action != "materialize":
+                raise RuntimeError(f"unknown candidate action {candidate.action!r}")
+
+            # Pre-filter: when the materialize arm fires for a message
+            # whose materializable count is zero (e.g. all file attachments
+            # were removed but reference attachments remain, or the
+            # attachments array is empty entirely), there's nothing to
+            # fetch. Emit a tombstone instead — dlt's delete-insert merge
+            # is a no-op when zero rows are emitted for a merge_key, so
+            # leftover rows would persist without the tombstone.
+            materializable = _materializable_attachments(candidate.attachments or [])
+            if not materializable:
+                orphaned_messages += 1
+                yield emit_orphan_tombstone(binding_id=binding_id, candidate=candidate)
+                continue
+
+            rows = list(
+                fetch_and_emit_for_message(
+                    binding_id=binding_id,
+                    client=client,
+                    candidate=candidate,
+                )
+            )
+            materialized_messages += 1
+            materialized_rows += len(rows)
+            yield from rows
+
+        elapsed = time.monotonic() - run_started
+        LOGGER.info(
+            "microsoft_mail.attachment_content.run_complete "
+            "candidates=%d materialized_messages=%d rows=%d orphaned_messages=%d elapsed=%.3fs",
+            len(candidates),
+            materialized_messages,
+            materialized_rows,
+            orphaned_messages,
+            elapsed,
+        )
+
+    return (attachment_content,)