contextbase-plugin-microsoft-mail 0.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contextbase_plugin_microsoft_mail-0.0.0a1.dist-info/METADATA +15 -0
- contextbase_plugin_microsoft_mail-0.0.0a1.dist-info/RECORD +17 -0
- contextbase_plugin_microsoft_mail-0.0.0a1.dist-info/WHEEL +4 -0
- contextbase_plugin_microsoft_mail-0.0.0a1.dist-info/entry_points.txt +3 -0
- plugin_microsoft_mail/__init__.py +1 -0
- plugin_microsoft_mail/binding_config.py +14 -0
- plugin_microsoft_mail/component.py +193 -0
- plugin_microsoft_mail/models/__init__.py +1 -0
- plugin_microsoft_mail/models/ctx.py +378 -0
- plugin_microsoft_mail/models/translators.py +194 -0
- plugin_microsoft_mail/plugin.json +26 -0
- plugin_microsoft_mail/sources/__init__.py +1 -0
- plugin_microsoft_mail/sources/attachments.py +392 -0
- plugin_microsoft_mail/sources/sync.py +409 -0
- plugin_microsoft_mail/utils/__init__.py +1 -0
- plugin_microsoft_mail/utils/attachments.py +107 -0
- plugin_microsoft_mail/utils/client.py +178 -0
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterator
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import dlt
|
|
8
|
+
from shared_plugins.microsoft_graph import (
|
|
9
|
+
DeltaPage,
|
|
10
|
+
drain_delta_pages,
|
|
11
|
+
graph_object_to_payload,
|
|
12
|
+
is_delta_cursor_url,
|
|
13
|
+
)
|
|
14
|
+
from shared_plugins.naming import (
|
|
15
|
+
dlt_resource_name,
|
|
16
|
+
dlt_source_name,
|
|
17
|
+
plugin_id_from_module,
|
|
18
|
+
)
|
|
19
|
+
from shared_plugins.resources import ctx_dlt_resource
|
|
20
|
+
|
|
21
|
+
from ..models.ctx import (
|
|
22
|
+
MAIL_FOLDER_COLUMN_DESCRIPTIONS,
|
|
23
|
+
MESSAGE_COLUMN_DESCRIPTIONS,
|
|
24
|
+
MailFolderRow,
|
|
25
|
+
MessageRow,
|
|
26
|
+
)
|
|
27
|
+
from ..models.translators import (
|
|
28
|
+
mail_folder_rows_to_ctx_models,
|
|
29
|
+
message_rows_to_ctx_models,
|
|
30
|
+
)
|
|
31
|
+
from ..utils.client import SyncGraphMailClient
|
|
32
|
+
|
|
33
|
+
# Known unhandled correctness gaps:
|
|
34
|
+
#
|
|
35
|
+
# 1. Stale messages from removed/hidden folders. `apply_mail_folder_delta_rows`
|
|
36
|
+
# drops folder IDs from the active set when a folder is @removed or
|
|
37
|
+
# transitions to isHidden, and `messages` only iterates the active set.
|
|
38
|
+
# Existing message rows (and their attachment_content rows) under the dropped
|
|
39
|
+
# folder remain in the destination forever — no tombstone path covers this
|
|
40
|
+
# transition.
|
|
41
|
+
#
|
|
42
|
+
# 2. Attachment same-count replacements (lives in sources/attachments.py). The
|
|
43
|
+
# candidate query compares counts of materializable attachments vs. existing
|
|
44
|
+
# attachment_content rows; a message changing [att-1] -> [att-2] looks like
|
|
45
|
+
# 1 == 1 and never re-materializes. The stale file row persists, the new
|
|
46
|
+
# attachment is missing.
|
|
47
|
+
|
|
48
|
+
MAIL_FOLDER_DELTA_CURSOR_URL_KEY = "cursor_url"
|
|
49
|
+
ACTIVE_MAIL_FOLDERS_KEY = "active_folders_by_id"
|
|
50
|
+
MESSAGE_DELTA_CURSOR_URLS_BY_FOLDER_ID_KEY = "cursor_urls_by_folder_id"
|
|
51
|
+
|
|
52
|
+
PLUGIN_ID = plugin_id_from_module(__file__)
|
|
53
|
+
JOB = "sync"
|
|
54
|
+
DELTA_PAGE_SIZE = 100
|
|
55
|
+
DELTA_PREFER_HEADER = f'IdType="ImmutableId", odata.maxpagesize={DELTA_PAGE_SIZE}'
|
|
56
|
+
MAIL_FOLDER_DELTA_QUERY_PARAMS = {
|
|
57
|
+
"$select": [
|
|
58
|
+
"id",
|
|
59
|
+
"displayName",
|
|
60
|
+
"parentFolderId",
|
|
61
|
+
"childFolderCount",
|
|
62
|
+
"totalItemCount",
|
|
63
|
+
"unreadItemCount",
|
|
64
|
+
"isHidden",
|
|
65
|
+
],
|
|
66
|
+
}
|
|
67
|
+
MESSAGE_DELTA_ORDERBY = ["receivedDateTime desc"]
|
|
68
|
+
INCLUDE_HIDDEN_MAIL_FOLDERS = False
|
|
69
|
+
MESSAGE_ATTACHMENTS_EXPAND = (
|
|
70
|
+
"attachments($select=id,name,contentType,size,isInline,lastModifiedDateTime)"
|
|
71
|
+
)
|
|
72
|
+
# Re-fetch of a sparse read/unread change event (see
|
|
73
|
+
# `message_payloads_with_sparse_change_refetched`) pulls the full message back
|
|
74
|
+
# with the same attachment expansion and immutable ids the delta itself uses.
|
|
75
|
+
MESSAGE_FULL_PREFER_HEADER = 'IdType="ImmutableId"'
|
|
76
|
+
MESSAGE_FULL_QUERY_PARAMS: dict[str, Any] = {"$expand": [MESSAGE_ATTACHMENTS_EXPAND]}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass(frozen=True)
|
|
80
|
+
class MailFolderDeltaDrainResult:
|
|
81
|
+
mail_folder_rows: list[dict[str, Any]]
|
|
82
|
+
active_mail_folders_by_id: dict[str, dict[str, Any]]
|
|
83
|
+
cursor_url: str
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def message_delta_cursor_urls_for_active_mail_folders(
|
|
87
|
+
previous_message_cursor_urls_by_folder_id: dict[str, str],
|
|
88
|
+
active_mail_folder_ids: list[str],
|
|
89
|
+
) -> dict[str, str]:
|
|
90
|
+
active_mail_folder_id_set = set(active_mail_folder_ids)
|
|
91
|
+
return {
|
|
92
|
+
folder_id: cursor_url
|
|
93
|
+
for folder_id, cursor_url in previous_message_cursor_urls_by_folder_id.items()
|
|
94
|
+
if folder_id in active_mail_folder_id_set
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def message_delta_cursor_urls_with_cursor_for_folder(
|
|
99
|
+
message_cursor_urls_by_folder_id: dict[str, str],
|
|
100
|
+
*,
|
|
101
|
+
folder_id: str,
|
|
102
|
+
cursor_url: str,
|
|
103
|
+
) -> dict[str, str]:
|
|
104
|
+
return {
|
|
105
|
+
**message_cursor_urls_by_folder_id,
|
|
106
|
+
folder_id: cursor_url,
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def should_include_mail_folder_payload(payload: dict[str, Any]) -> bool:
|
|
111
|
+
if payload.get("@removed") is not None:
|
|
112
|
+
return False
|
|
113
|
+
if not INCLUDE_HIDDEN_MAIL_FOLDERS and payload.get("isHidden"):
|
|
114
|
+
return False
|
|
115
|
+
return True
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def mail_folder_delta_page_rows(response: Any) -> list[dict[str, Any]]:
|
|
119
|
+
rows: list[dict[str, Any]] = []
|
|
120
|
+
for folder in getattr(response, "value", None) or []:
|
|
121
|
+
row = graph_object_to_payload(folder)
|
|
122
|
+
if not row.get("id"):
|
|
123
|
+
raise RuntimeError(
|
|
124
|
+
f"Graph mail-folder delta row missing id: keys={sorted(row)}"
|
|
125
|
+
)
|
|
126
|
+
rows.append(row)
|
|
127
|
+
return rows
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def apply_mail_folder_delta_rows(
|
|
131
|
+
*,
|
|
132
|
+
previous_active_mail_folders_by_id: dict[str, dict[str, Any]],
|
|
133
|
+
mail_folder_rows: list[dict[str, Any]],
|
|
134
|
+
) -> dict[str, dict[str, Any]]:
|
|
135
|
+
active_mail_folders_by_id = dict(previous_active_mail_folders_by_id)
|
|
136
|
+
|
|
137
|
+
for row in mail_folder_rows:
|
|
138
|
+
folder_id = str(row["id"])
|
|
139
|
+
if should_include_mail_folder_payload(row):
|
|
140
|
+
active_mail_folders_by_id[folder_id] = {
|
|
141
|
+
"id": folder_id,
|
|
142
|
+
"display_name": row.get("displayName"),
|
|
143
|
+
"parent_folder_id": row.get("parentFolderId"),
|
|
144
|
+
"is_hidden": row.get("isHidden"),
|
|
145
|
+
}
|
|
146
|
+
else:
|
|
147
|
+
active_mail_folders_by_id.pop(folder_id, None)
|
|
148
|
+
|
|
149
|
+
return active_mail_folders_by_id
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def drain_mail_folder_delta(
|
|
153
|
+
*,
|
|
154
|
+
client: SyncGraphMailClient,
|
|
155
|
+
previous_cursor_url: str | None,
|
|
156
|
+
previous_active_mail_folders_by_id: dict[str, dict[str, Any]],
|
|
157
|
+
) -> MailFolderDeltaDrainResult:
|
|
158
|
+
mail_folder_rows: list[dict[str, Any]] = []
|
|
159
|
+
drained_cursor_url: str | None = None
|
|
160
|
+
|
|
161
|
+
for page in drain_delta_pages(
|
|
162
|
+
initial_cursor_url=previous_cursor_url,
|
|
163
|
+
fetch_page=lambda cursor_url: client.get_folder_delta_page(
|
|
164
|
+
delta_url=cursor_url,
|
|
165
|
+
query_params=MAIL_FOLDER_DELTA_QUERY_PARAMS,
|
|
166
|
+
prefer_header=DELTA_PREFER_HEADER,
|
|
167
|
+
),
|
|
168
|
+
rows_from_page=mail_folder_delta_page_rows,
|
|
169
|
+
):
|
|
170
|
+
mail_folder_rows.extend(page.rows)
|
|
171
|
+
drained_cursor_url = page.cursor_url
|
|
172
|
+
|
|
173
|
+
if drained_cursor_url is None or not is_delta_cursor_url(drained_cursor_url):
|
|
174
|
+
raise RuntimeError("mail folder delta drain did not finish with a delta cursor")
|
|
175
|
+
|
|
176
|
+
return MailFolderDeltaDrainResult(
|
|
177
|
+
mail_folder_rows=mail_folder_rows,
|
|
178
|
+
active_mail_folders_by_id=apply_mail_folder_delta_rows(
|
|
179
|
+
previous_active_mail_folders_by_id=previous_active_mail_folders_by_id,
|
|
180
|
+
mail_folder_rows=mail_folder_rows,
|
|
181
|
+
),
|
|
182
|
+
cursor_url=drained_cursor_url,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def message_delta_page_rows(
|
|
187
|
+
*,
|
|
188
|
+
response: Any,
|
|
189
|
+
folder_id: str,
|
|
190
|
+
) -> list[Any]:
|
|
191
|
+
# Don't pre-validate parent_folder_id — @removed rows carry only id +
|
|
192
|
+
# @removed; the translator detects them via additional_data and emits a
|
|
193
|
+
# tombstone using folder_id from context. Live rows with no
|
|
194
|
+
# parent_folder_id fail pydantic validation at MessageRow.parent_folder_id.
|
|
195
|
+
rows: list[Any] = []
|
|
196
|
+
for message in getattr(response, "value", None) or []:
|
|
197
|
+
message_id = getattr(message, "id", None)
|
|
198
|
+
if not message_id:
|
|
199
|
+
raise RuntimeError(
|
|
200
|
+
f"Graph message delta row missing id (folder_id={folder_id!r})"
|
|
201
|
+
)
|
|
202
|
+
rows.append(message)
|
|
203
|
+
return rows
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def drain_message_delta_pages_for_folder(
|
|
207
|
+
*,
|
|
208
|
+
client: SyncGraphMailClient,
|
|
209
|
+
folder_id: str,
|
|
210
|
+
previous_cursor_url: str | None,
|
|
211
|
+
initial_message_delta_top: int | None,
|
|
212
|
+
) -> Iterator[DeltaPage]:
|
|
213
|
+
yield from drain_delta_pages(
|
|
214
|
+
initial_cursor_url=previous_cursor_url,
|
|
215
|
+
fetch_page=lambda cursor_url: client.get_message_delta_page(
|
|
216
|
+
folder_id=folder_id,
|
|
217
|
+
delta_url=cursor_url,
|
|
218
|
+
query_params={
|
|
219
|
+
"$orderby": MESSAGE_DELTA_ORDERBY,
|
|
220
|
+
"$top": initial_message_delta_top,
|
|
221
|
+
"$expand": [MESSAGE_ATTACHMENTS_EXPAND],
|
|
222
|
+
},
|
|
223
|
+
prefer_header=DELTA_PREFER_HEADER,
|
|
224
|
+
),
|
|
225
|
+
rows_from_page=lambda response: message_delta_page_rows(
|
|
226
|
+
response=response,
|
|
227
|
+
folder_id=folder_id,
|
|
228
|
+
),
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def message_payloads_with_sparse_change_refetched(
|
|
233
|
+
*,
|
|
234
|
+
client: SyncGraphMailClient,
|
|
235
|
+
rows: list[Any],
|
|
236
|
+
folder_id: str,
|
|
237
|
+
) -> Iterator[dict[str, Any]]:
|
|
238
|
+
"""Yield Graph message payloads, re-fetching the full message for any
|
|
239
|
+
sparse read/unread change event.
|
|
240
|
+
|
|
241
|
+
Graph's per-folder `messages/delta` emits a sparse change object (only `id`
|
|
242
|
+
+ `isRead`) when a message's read state flips — a folder-collection-level
|
|
243
|
+
event, not a change to the message itself (see the `message: delta`
|
|
244
|
+
reference). dlt's merge replaces the whole row, so feeding that sparse row
|
|
245
|
+
to the `messages` resource would NULL every column except `is_read`. We
|
|
246
|
+
detect a sparse, non-`@removed` row by the absence of `receivedDateTime`
|
|
247
|
+
(every real message — including drafts — carries it) and re-fetch the full
|
|
248
|
+
message so the merge upserts a complete row with the updated read state.
|
|
249
|
+
|
|
250
|
+
The re-fetched row is scoped to `folder_id` — the folder whose delta
|
|
251
|
+
surfaced this change — rather than the mailbox-wide GET's *current*
|
|
252
|
+
`parentFolderId`. A message can move folders between the delta event and
|
|
253
|
+
this (lazy) re-fetch; trusting the re-fetched `parentFolderId` would key the
|
|
254
|
+
row to a different folder than the one being drained, duplicating the
|
|
255
|
+
message under two `parent_folder_id`s. Pinning it to `folder_id` keeps the
|
|
256
|
+
row on the draining folder's primary key — the same scoping the `@removed`
|
|
257
|
+
tombstone path uses — so a concurrent move is reconciled by that folder's
|
|
258
|
+
later `@removed` entry. (For an unmoved message the two are identical.)
|
|
259
|
+
|
|
260
|
+
`@removed` tombstones and already-complete rows pass through untouched. A
|
|
261
|
+
re-fetch failure (including a 404 — the message vanished between the delta
|
|
262
|
+
event and this call) is NOT swallowed: it propagates, failing the run
|
|
263
|
+
without advancing the per-folder cursor, so the next run retries. A genuine
|
|
264
|
+
read-then-delete then surfaces as `@removed` and tombstones the row; a
|
|
265
|
+
transient 404 recovers on retry with the read-state update intact.
|
|
266
|
+
"""
|
|
267
|
+
for row in rows:
|
|
268
|
+
payload = graph_object_to_payload(row)
|
|
269
|
+
additional_data = payload.get("additional_data") or {}
|
|
270
|
+
is_removed = "@removed" in additional_data or "@removed" in payload
|
|
271
|
+
if is_removed or payload.get("receivedDateTime") is not None:
|
|
272
|
+
yield payload
|
|
273
|
+
continue
|
|
274
|
+
full = client.get_message_full(
|
|
275
|
+
message_id=payload["id"],
|
|
276
|
+
query_params=MESSAGE_FULL_QUERY_PARAMS,
|
|
277
|
+
prefer_header=MESSAGE_FULL_PREFER_HEADER,
|
|
278
|
+
)
|
|
279
|
+
full_payload = graph_object_to_payload(full)
|
|
280
|
+
# A single-message GET carries an `@odata.context` envelope annotation
|
|
281
|
+
# (the $metadata URL, with the mailbox address) that delta items don't.
|
|
282
|
+
# Drop it from additional_data so a re-fetched row matches a delta row's
|
|
283
|
+
# instead of persisting a useless URL. (The Kiota serializer also lifts
|
|
284
|
+
# @odata.context to a top-level key; the translator maps only known
|
|
285
|
+
# fields, so that copy is dropped before persistence either way.)
|
|
286
|
+
full_additional_data = full_payload.get("additional_data")
|
|
287
|
+
if isinstance(full_additional_data, dict):
|
|
288
|
+
full_additional_data.pop("@odata.context", None)
|
|
289
|
+
full_payload["parentFolderId"] = folder_id
|
|
290
|
+
yield full_payload
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
@dlt.source(name=dlt_source_name(PLUGIN_ID, JOB))
|
|
294
|
+
def microsoft_mail_source(
|
|
295
|
+
binding_id: str,
|
|
296
|
+
*,
|
|
297
|
+
client: SyncGraphMailClient,
|
|
298
|
+
initial_message_delta_top: int | None = None,
|
|
299
|
+
) -> tuple[Any, ...]:
|
|
300
|
+
folder_drain_cache: MailFolderDeltaDrainResult | None = None
|
|
301
|
+
|
|
302
|
+
def get_mail_folder_delta_snapshot() -> MailFolderDeltaDrainResult:
|
|
303
|
+
nonlocal folder_drain_cache
|
|
304
|
+
if folder_drain_cache is not None:
|
|
305
|
+
return folder_drain_cache
|
|
306
|
+
|
|
307
|
+
source_state = dlt.current.source_state()
|
|
308
|
+
previous_cursor_url = source_state.get(MAIL_FOLDER_DELTA_CURSOR_URL_KEY)
|
|
309
|
+
previous_active_mail_folders_by_id = dict(
|
|
310
|
+
source_state.get(ACTIVE_MAIL_FOLDERS_KEY) or {}
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
result = drain_mail_folder_delta(
|
|
314
|
+
client=client,
|
|
315
|
+
previous_cursor_url=previous_cursor_url,
|
|
316
|
+
previous_active_mail_folders_by_id=previous_active_mail_folders_by_id,
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
source_state[MAIL_FOLDER_DELTA_CURSOR_URL_KEY] = result.cursor_url
|
|
320
|
+
source_state[ACTIVE_MAIL_FOLDERS_KEY] = result.active_mail_folders_by_id
|
|
321
|
+
|
|
322
|
+
folder_drain_cache = result
|
|
323
|
+
return result
|
|
324
|
+
|
|
325
|
+
@ctx_dlt_resource(
|
|
326
|
+
name=dlt_resource_name("mail_folders"),
|
|
327
|
+
write_disposition="merge",
|
|
328
|
+
primary_key=("_ctx_binding_id", "id"),
|
|
329
|
+
columns={
|
|
330
|
+
**MAIL_FOLDER_COLUMN_DESCRIPTIONS,
|
|
331
|
+
"_ctx_deleted": {"hard_delete": True},
|
|
332
|
+
},
|
|
333
|
+
)
|
|
334
|
+
def mail_folders() -> Iterator[MailFolderRow]:
|
|
335
|
+
snapshot = get_mail_folder_delta_snapshot()
|
|
336
|
+
yield from mail_folder_rows_to_ctx_models(
|
|
337
|
+
binding_id,
|
|
338
|
+
snapshot.mail_folder_rows,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
@ctx_dlt_resource(
|
|
342
|
+
name=dlt_resource_name("messages"),
|
|
343
|
+
write_disposition="merge",
|
|
344
|
+
primary_key=("_ctx_binding_id", "id", "parent_folder_id"),
|
|
345
|
+
columns={
|
|
346
|
+
**MESSAGE_COLUMN_DESCRIPTIONS,
|
|
347
|
+
"_ctx_deleted": {"hard_delete": True},
|
|
348
|
+
},
|
|
349
|
+
)
|
|
350
|
+
def messages() -> Iterator[MessageRow]:
|
|
351
|
+
snapshot = get_mail_folder_delta_snapshot()
|
|
352
|
+
active_mail_folder_ids = sorted(snapshot.active_mail_folders_by_id)
|
|
353
|
+
|
|
354
|
+
source_state = dlt.current.source_state()
|
|
355
|
+
previous_message_cursor_urls_by_folder_id = (
|
|
356
|
+
message_delta_cursor_urls_for_active_mail_folders(
|
|
357
|
+
dict(
|
|
358
|
+
source_state.get(MESSAGE_DELTA_CURSOR_URLS_BY_FOLDER_ID_KEY) or {}
|
|
359
|
+
),
|
|
360
|
+
active_mail_folder_ids,
|
|
361
|
+
)
|
|
362
|
+
)
|
|
363
|
+
updated_message_cursor_urls_by_folder_id = dict(
|
|
364
|
+
previous_message_cursor_urls_by_folder_id,
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
for folder_id in active_mail_folder_ids:
|
|
368
|
+
previous_message_cursor_url = previous_message_cursor_urls_by_folder_id.get(
|
|
369
|
+
folder_id
|
|
370
|
+
)
|
|
371
|
+
message_cursor_url: str | None = previous_message_cursor_url
|
|
372
|
+
|
|
373
|
+
for page in drain_message_delta_pages_for_folder(
|
|
374
|
+
client=client,
|
|
375
|
+
folder_id=folder_id,
|
|
376
|
+
previous_cursor_url=previous_message_cursor_url,
|
|
377
|
+
initial_message_delta_top=initial_message_delta_top,
|
|
378
|
+
):
|
|
379
|
+
message_cursor_url = page.cursor_url
|
|
380
|
+
yield from message_rows_to_ctx_models(
|
|
381
|
+
binding_id,
|
|
382
|
+
message_payloads_with_sparse_change_refetched(
|
|
383
|
+
client=client,
|
|
384
|
+
rows=page.rows,
|
|
385
|
+
folder_id=folder_id,
|
|
386
|
+
),
|
|
387
|
+
folder_id=folder_id,
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
if message_cursor_url is None or not is_delta_cursor_url(
|
|
391
|
+
message_cursor_url
|
|
392
|
+
):
|
|
393
|
+
raise RuntimeError(
|
|
394
|
+
"message delta drain did not finish with a delta cursor"
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
updated_message_cursor_urls_by_folder_id = (
|
|
398
|
+
message_delta_cursor_urls_with_cursor_for_folder(
|
|
399
|
+
updated_message_cursor_urls_by_folder_id,
|
|
400
|
+
folder_id=folder_id,
|
|
401
|
+
cursor_url=message_cursor_url,
|
|
402
|
+
)
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
source_state[MESSAGE_DELTA_CURSOR_URLS_BY_FOLDER_ID_KEY] = (
|
|
406
|
+
updated_message_cursor_urls_by_folder_id
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
return (mail_folders, messages)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Utilities for Microsoft Mail Graph and DLT spikes."""
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
from base64 import b64decode
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from pathlib import PurePosixPath
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from shared_plugins.scratch import replace_scratch_dir_files
|
|
10
|
+
|
|
11
|
+
from ..models.ctx import AttachmentContentRow
|
|
12
|
+
from ..models.translators import attachment_content_row_from_graph_payload
|
|
13
|
+
|
|
14
|
+
REFERENCE_ATTACHMENT_ODATA_TYPE = "#microsoft.graph.referenceAttachment"
|
|
15
|
+
ITEM_ATTACHMENT_ODATA_TYPE = "#microsoft.graph.itemAttachment"
|
|
16
|
+
FILE_ATTACHMENT_ODATA_TYPE = "#microsoft.graph.fileAttachment"
|
|
17
|
+
KNOWN_ATTACHMENT_ODATA_TYPES = frozenset(
|
|
18
|
+
{
|
|
19
|
+
FILE_ATTACHMENT_ODATA_TYPE,
|
|
20
|
+
REFERENCE_ATTACHMENT_ODATA_TYPE,
|
|
21
|
+
ITEM_ATTACHMENT_ODATA_TYPE,
|
|
22
|
+
}
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _hash_path_segment(*parts: str) -> str:
|
|
27
|
+
hash_input = "\n".join(parts)
|
|
28
|
+
return hashlib.sha256(hash_input.encode("utf-8")).hexdigest()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _build_deterministic_file_name(
|
|
32
|
+
*,
|
|
33
|
+
attachment_id: str,
|
|
34
|
+
name: str | None,
|
|
35
|
+
) -> str:
|
|
36
|
+
digest = _hash_path_segment(attachment_id, name or "")
|
|
37
|
+
suffix = ""
|
|
38
|
+
if name:
|
|
39
|
+
suffix = "".join(PurePosixPath(name.strip()).suffixes)
|
|
40
|
+
return f"{digest}{suffix}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def materialize_attachment_payloads(
|
|
44
|
+
*,
|
|
45
|
+
binding_id: str,
|
|
46
|
+
message_id: str,
|
|
47
|
+
attachment_payloads: Sequence[Mapping[str, Any]],
|
|
48
|
+
) -> list[AttachmentContentRow]:
|
|
49
|
+
"""Decode each payload's `contentBytes`, write all of a message's
|
|
50
|
+
attachments into the per-message scratch directory atomically, and return
|
|
51
|
+
one AttachmentContentRow per payload in input order.
|
|
52
|
+
|
|
53
|
+
All payloads must include `id` and `contentBytes` (base64). Reference and
|
|
54
|
+
item attachments are excluded upstream — this helper is for materializable
|
|
55
|
+
file attachments only.
|
|
56
|
+
|
|
57
|
+
Writes are batched into a single `replace_scratch_dir_files` call because
|
|
58
|
+
that helper atomically REPLACES the entire `relative_dir`; per-attachment
|
|
59
|
+
calls would clobber each other within the same message.
|
|
60
|
+
"""
|
|
61
|
+
if len(attachment_payloads) == 0:
|
|
62
|
+
return []
|
|
63
|
+
|
|
64
|
+
file_name_by_attachment_id: dict[str, str] = {}
|
|
65
|
+
files: dict[str, bytes] = {}
|
|
66
|
+
for payload in attachment_payloads:
|
|
67
|
+
attachment_id = payload.get("id")
|
|
68
|
+
if not isinstance(attachment_id, str):
|
|
69
|
+
raise RuntimeError(
|
|
70
|
+
"attachment payload missing id "
|
|
71
|
+
f"message_id={message_id} keys={sorted(payload)}"
|
|
72
|
+
)
|
|
73
|
+
content_bytes_b64 = payload.get("contentBytes")
|
|
74
|
+
if not isinstance(content_bytes_b64, str):
|
|
75
|
+
raise RuntimeError(
|
|
76
|
+
"attachment payload missing contentBytes "
|
|
77
|
+
f"message_id={message_id} attachment_id={attachment_id}"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
file_name = _build_deterministic_file_name(
|
|
81
|
+
attachment_id=attachment_id,
|
|
82
|
+
name=(
|
|
83
|
+
payload.get("name") if isinstance(payload.get("name"), str) else None
|
|
84
|
+
),
|
|
85
|
+
)
|
|
86
|
+
file_name_by_attachment_id[attachment_id] = file_name
|
|
87
|
+
files[file_name] = b64decode(content_bytes_b64)
|
|
88
|
+
|
|
89
|
+
path_by_file_name = replace_scratch_dir_files(
|
|
90
|
+
binding_id=binding_id,
|
|
91
|
+
relative_dir=f"attachments/{_hash_path_segment(message_id)}",
|
|
92
|
+
files=files,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
rows: list[AttachmentContentRow] = []
|
|
96
|
+
for payload in attachment_payloads:
|
|
97
|
+
attachment_id = payload["id"]
|
|
98
|
+
file_name = file_name_by_attachment_id[attachment_id]
|
|
99
|
+
rows.append(
|
|
100
|
+
attachment_content_row_from_graph_payload(
|
|
101
|
+
binding_id=binding_id,
|
|
102
|
+
message_id=message_id,
|
|
103
|
+
attachment_payload=payload,
|
|
104
|
+
file_path=path_by_file_name[file_name],
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
return rows
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from asyncio import AbstractEventLoop
|
|
5
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from azure.core.credentials_async import AsyncTokenCredential
|
|
9
|
+
from msgraph import GraphServiceClient
|
|
10
|
+
from msgraph.graph_request_adapter import (
|
|
11
|
+
GraphRequestAdapter,
|
|
12
|
+
options as _GRAPH_DEFAULT_OPTIONS,
|
|
13
|
+
)
|
|
14
|
+
from msgraph_core import APIVersion
|
|
15
|
+
from shared_plugins.microsoft_graph import (
|
|
16
|
+
build_h11_request_adapter,
|
|
17
|
+
request_configuration,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class SyncGraphMailClient:
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
*,
|
|
25
|
+
credential_factory: Callable[[], AsyncTokenCredential],
|
|
26
|
+
mailbox_user_id: str,
|
|
27
|
+
):
|
|
28
|
+
self._credential_factory = credential_factory
|
|
29
|
+
self._mailbox_user_id = mailbox_user_id
|
|
30
|
+
self._loop: AbstractEventLoop | None = None
|
|
31
|
+
self._credential: AsyncTokenCredential | None = None
|
|
32
|
+
self._client: GraphServiceClient | None = None
|
|
33
|
+
|
|
34
|
+
def __enter__(self) -> SyncGraphMailClient:
|
|
35
|
+
self._loop = asyncio.new_event_loop()
|
|
36
|
+
self._loop.run_until_complete(self._open())
|
|
37
|
+
return self
|
|
38
|
+
|
|
39
|
+
def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
|
|
40
|
+
assert self._loop is not None
|
|
41
|
+
self._loop.run_until_complete(self._close())
|
|
42
|
+
self._loop.close()
|
|
43
|
+
|
|
44
|
+
async def _open(self) -> None:
|
|
45
|
+
# Build the credential INSIDE this client's event loop: azure.identity.aio
|
|
46
|
+
# credentials bind their transport to the running loop on first use, so the
|
|
47
|
+
# factory (which may build a ClientSecretCredential) must run here, not at
|
|
48
|
+
# component-build time.
|
|
49
|
+
self._credential = self._credential_factory()
|
|
50
|
+
self._client = GraphServiceClient(
|
|
51
|
+
request_adapter=build_h11_request_adapter(
|
|
52
|
+
self._credential,
|
|
53
|
+
api_version=APIVersion.v1,
|
|
54
|
+
adapter_cls=GraphRequestAdapter,
|
|
55
|
+
adapter_options=_GRAPH_DEFAULT_OPTIONS,
|
|
56
|
+
),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
async def _close(self) -> None:
|
|
60
|
+
if self._credential is not None:
|
|
61
|
+
await self._credential.close()
|
|
62
|
+
|
|
63
|
+
def _run(self, request: Awaitable[Any]) -> Any:
|
|
64
|
+
assert self._loop is not None
|
|
65
|
+
return self._loop.run_until_complete(request)
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def mailbox_user_id(self) -> str:
|
|
69
|
+
return self._mailbox_user_id
|
|
70
|
+
|
|
71
|
+
def get_folder_delta_page(
|
|
72
|
+
self,
|
|
73
|
+
*,
|
|
74
|
+
delta_url: str | None,
|
|
75
|
+
query_params: Mapping[str, object | None],
|
|
76
|
+
prefer_header: str | None,
|
|
77
|
+
) -> Any:
|
|
78
|
+
assert self._client is not None
|
|
79
|
+
builder = self._client.users.by_user_id(self.mailbox_user_id).mail_folders.delta
|
|
80
|
+
if delta_url:
|
|
81
|
+
builder = builder.with_url(delta_url)
|
|
82
|
+
|
|
83
|
+
return self._run(
|
|
84
|
+
builder.get(
|
|
85
|
+
request_configuration=request_configuration(
|
|
86
|
+
query_params=None if delta_url else query_params,
|
|
87
|
+
prefer_header=prefer_header,
|
|
88
|
+
),
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
def get_message_delta_page(
|
|
93
|
+
self,
|
|
94
|
+
*,
|
|
95
|
+
folder_id: str,
|
|
96
|
+
delta_url: str | None,
|
|
97
|
+
query_params: Mapping[str, object | None],
|
|
98
|
+
prefer_header: str | None,
|
|
99
|
+
) -> Any:
|
|
100
|
+
assert self._client is not None
|
|
101
|
+
builder = (
|
|
102
|
+
self._client.users.by_user_id(self.mailbox_user_id)
|
|
103
|
+
.mail_folders.by_mail_folder_id(folder_id)
|
|
104
|
+
.messages.delta
|
|
105
|
+
)
|
|
106
|
+
if delta_url:
|
|
107
|
+
builder = builder.with_url(delta_url)
|
|
108
|
+
|
|
109
|
+
return self._run(
|
|
110
|
+
builder.get(
|
|
111
|
+
request_configuration=request_configuration(
|
|
112
|
+
query_params=None if delta_url else query_params,
|
|
113
|
+
prefer_header=prefer_header,
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
def get_message_full(
|
|
119
|
+
self,
|
|
120
|
+
*,
|
|
121
|
+
message_id: str,
|
|
122
|
+
query_params: Mapping[str, object | None] | None = None,
|
|
123
|
+
prefer_header: str | None = None,
|
|
124
|
+
) -> Any:
|
|
125
|
+
"""GET /messages/{id} — the full message.
|
|
126
|
+
|
|
127
|
+
Used to re-materialize a complete row for a sparse read/unread delta
|
|
128
|
+
change event. Graph's per-folder `messages/delta` emits a sparse change
|
|
129
|
+
object (only `id` + `isRead`) when a message's read state flips — a
|
|
130
|
+
folder-collection-level event, not a content change. Merging that
|
|
131
|
+
sparse row would overwrite the complete row with NULLs in every column
|
|
132
|
+
except `is_read`, so we re-fetch the whole message instead.
|
|
133
|
+
|
|
134
|
+
Errors are not swallowed — including a 404. A 404 here means the message
|
|
135
|
+
vanished between the delta event and this re-fetch, and failing loud is
|
|
136
|
+
correct in every case: a genuine read-then-delete fails this run and
|
|
137
|
+
self-recovers on the next, where the folder's `@removed` delta entry
|
|
138
|
+
tombstones the row (a deleted item surfaces as `@removed` on a fresh
|
|
139
|
+
delta, not as a re-emitted read event, so the run does not wedge); a
|
|
140
|
+
transient/eventual-consistency 404 recovers on retry with the read-state
|
|
141
|
+
update intact; and a systematic 404 (e.g. a misissued immutable-id
|
|
142
|
+
header) is a real bug we want loud, not a silent skip of every
|
|
143
|
+
read-state update."""
|
|
144
|
+
assert self._client is not None
|
|
145
|
+
builder = self._client.users.by_user_id(
|
|
146
|
+
self.mailbox_user_id
|
|
147
|
+
).messages.by_message_id(message_id)
|
|
148
|
+
return self._run(
|
|
149
|
+
builder.get(
|
|
150
|
+
request_configuration=request_configuration(
|
|
151
|
+
query_params=query_params,
|
|
152
|
+
prefer_header=prefer_header,
|
|
153
|
+
),
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
def get_attachment_full(
|
|
158
|
+
self,
|
|
159
|
+
*,
|
|
160
|
+
message_id: str,
|
|
161
|
+
attachment_id: str,
|
|
162
|
+
prefer_header: str | None,
|
|
163
|
+
) -> Any:
|
|
164
|
+
"""GET /messages/{m}/attachments/{a} — returns the full attachment object
|
|
165
|
+
including contentBytes (for fileAttachment), contentId, contentLocation."""
|
|
166
|
+
assert self._client is not None
|
|
167
|
+
builder = (
|
|
168
|
+
self._client.users.by_user_id(self.mailbox_user_id)
|
|
169
|
+
.messages.by_message_id(message_id)
|
|
170
|
+
.attachments.by_attachment_id(attachment_id)
|
|
171
|
+
)
|
|
172
|
+
return self._run(
|
|
173
|
+
builder.get(
|
|
174
|
+
request_configuration=request_configuration(
|
|
175
|
+
prefer_header=prefer_header,
|
|
176
|
+
),
|
|
177
|
+
),
|
|
178
|
+
)
|