contextbase-plugin-gmail 0.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contextbase_plugin_gmail-0.2.6.dist-info/METADATA +13 -0
- contextbase_plugin_gmail-0.2.6.dist-info/RECORD +21 -0
- contextbase_plugin_gmail-0.2.6.dist-info/WHEEL +4 -0
- plugin_gmail/__init__.py +0 -0
- plugin_gmail/binding_config.py +13 -0
- plugin_gmail/component.py +269 -0
- plugin_gmail/defs/__init__.py +0 -0
- plugin_gmail/defs/defs.yaml +1 -0
- plugin_gmail/models/__init__.py +0 -0
- plugin_gmail/models/ctx.py +132 -0
- plugin_gmail/models/ingress.py +185 -0
- plugin_gmail/models/translators.py +470 -0
- plugin_gmail/models/types.py +12 -0
- plugin_gmail/plugin.json +9 -0
- plugin_gmail/sources/__init__.py +0 -0
- plugin_gmail/sources/attachments.py +307 -0
- plugin_gmail/sources/backfill.py +129 -0
- plugin_gmail/sources/history.py +160 -0
- plugin_gmail/utils/__init__.py +0 -0
- plugin_gmail/utils/attachments.py +251 -0
- plugin_gmail/utils/client.py +494 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: contextbase-plugin-gmail
|
|
3
|
+
Version: 0.2.6
|
|
4
|
+
Summary: Gmail plugin for ContextBase
|
|
5
|
+
Author: Alizain Feerasta
|
|
6
|
+
Author-email: Alizain Feerasta <alizain.feerasta@gmail.com>
|
|
7
|
+
Requires-Dist: contextbase-shared-plugins==0.2.6
|
|
8
|
+
Requires-Dist: dagster==1.12.14
|
|
9
|
+
Requires-Dist: dagster-dlt==0.28.14
|
|
10
|
+
Requires-Dist: dlt>=1.26.0
|
|
11
|
+
Requires-Dist: google-api-python-client>=2.185.0
|
|
12
|
+
Requires-Dist: pydantic>=2.12.0
|
|
13
|
+
Requires-Python: >=3.14, <3.15
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
plugin_gmail/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
plugin_gmail/binding_config.py,sha256=Gj9UfPxZPcJ5OJgRZFUOas7yOfr2YFHBEkE8xvo74w0,493
|
|
3
|
+
plugin_gmail/component.py,sha256=gFuOMbGOBEDYRNwE-d_OHykbvJIg3qkLpv6ctJij1tI,9334
|
|
4
|
+
plugin_gmail/defs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
plugin_gmail/defs/defs.yaml,sha256=T04TfRCWlD8cEeUlUUCGfMYeSI5ggCFVF49Ws5TmmNY,48
|
|
6
|
+
plugin_gmail/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
plugin_gmail/models/ctx.py,sha256=PyIzp1TiuI1lLL3xfHiI_xMupP7VCkzoIL2nfZ9bm1s,4666
|
|
8
|
+
plugin_gmail/models/ingress.py,sha256=DYvYQnimuiVPQZLdid0d8Hv3iq1djj8HvYxZOw_Kk-A,6493
|
|
9
|
+
plugin_gmail/models/translators.py,sha256=VMFmSL2A-K9LbunljE226yJRQ4J_KPGZMOZUOVTsmaQ,14781
|
|
10
|
+
plugin_gmail/models/types.py,sha256=ivgGsKdlSUaqag00IMpb3BphFbz3AGgmjjtfl4H-YbM,270
|
|
11
|
+
plugin_gmail/plugin.json,sha256=3xjoHA6z4isOwN8qU3X_5UG9pjZz-aVKqrte-d4HM9E,170
|
|
12
|
+
plugin_gmail/sources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
plugin_gmail/sources/attachments.py,sha256=q8lF_uObn3CuLYKxk9ZkdmK1_9Bbv68RG7ZkLzTldNw,9932
|
|
14
|
+
plugin_gmail/sources/backfill.py,sha256=tFLc-mdEtEpHvjKjEKpcgzk-UB2GU_ZAQ8shNuHmqXk,4122
|
|
15
|
+
plugin_gmail/sources/history.py,sha256=hc3WuQoZXHfhTS8uRAtttCZaoRxxpOFBZZ6md2XpsEU,4913
|
|
16
|
+
plugin_gmail/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
|
+
plugin_gmail/utils/attachments.py,sha256=eWXFZ0ZR7ToRvbyXwp2VlmOyRoikiz4hahSUl9xSVU4,8110
|
|
18
|
+
plugin_gmail/utils/client.py,sha256=H6a2LvJddoeDPkXKAc0fzMrf1sXBZjBAj2QVX6T1xF0,17774
|
|
19
|
+
contextbase_plugin_gmail-0.2.6.dist-info/WHEEL,sha256=i9aSRDivn5iP9LaR1BLQX2GNAuriQWPsFwbbWygTX2k,81
|
|
20
|
+
contextbase_plugin_gmail-0.2.6.dist-info/METADATA,sha256=CVQcJPIYyz21sxGcFraX1xsnMv7g9lrQwkmbZpTaEmM,447
|
|
21
|
+
contextbase_plugin_gmail-0.2.6.dist-info/RECORD,,
|
plugin_gmail/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from shared_plugins.bindings import BaseBindingConfigModel
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class GmailBindingConfig(BaseBindingConfigModel):
|
|
7
|
+
"""Gmail plugin binding configuration.
|
|
8
|
+
|
|
9
|
+
Gmail has no per-binding settings today. The empty model exists to
|
|
10
|
+
establish the parse site so that adding the first real field later is a
|
|
11
|
+
one-line change, and to replace ad hoc rejection guards with a
|
|
12
|
+
declarative unknown-keys check (inherited from ``BaseBindingConfigModel``).
|
|
13
|
+
"""
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import dagster as dg
|
|
2
|
+
from dagster import AssetExecutionContext
|
|
3
|
+
from dagster_dlt import DagsterDltResource
|
|
4
|
+
from shared_plugins.automation import non_overlapping_automation_condition
|
|
5
|
+
from shared_types.dagster_binding_plan import DagsterAllPlanBinding
|
|
6
|
+
from shared_plugins.bindings import (
|
|
7
|
+
parse_binding_config,
|
|
8
|
+
require_authenticated_account,
|
|
9
|
+
)
|
|
10
|
+
from shared_plugins.control_plane import ControlPlaneClient
|
|
11
|
+
from shared_plugins.dlt import resolve_partition_binding, run_dlt_pipeline
|
|
12
|
+
from shared_plugins.google_client.auth import (
|
|
13
|
+
build_google_service,
|
|
14
|
+
)
|
|
15
|
+
from shared_plugins.naming import (
|
|
16
|
+
dagster_asset_group_name,
|
|
17
|
+
dagster_asset_tags,
|
|
18
|
+
dagster_dlt_asset_key,
|
|
19
|
+
dagster_partition_def_name,
|
|
20
|
+
dagster_pool_name,
|
|
21
|
+
dlt_source_name,
|
|
22
|
+
plugin_id_from_module,
|
|
23
|
+
)
|
|
24
|
+
from shared_plugins.resources import DLT_RESOURCE
|
|
25
|
+
|
|
26
|
+
from .binding_config import GmailBindingConfig
|
|
27
|
+
from .sources.attachments import gmail_attachment_content_source
|
|
28
|
+
from .sources.backfill import gmail_backfill_source
|
|
29
|
+
from .sources.history import gmail_history_source
|
|
30
|
+
from .utils.client import GmailApiClient
|
|
31
|
+
|
|
32
|
+
PLUGIN_ID = plugin_id_from_module(__file__)
|
|
33
|
+
BACKFILL_JOB = "backfill"
|
|
34
|
+
HISTORY_JOB = "history"
|
|
35
|
+
ATTACHMENT_CONTENT_JOB = "attachment_content"
|
|
36
|
+
|
|
37
|
+
BACKFILL_SOURCE_NAME = dlt_source_name(PLUGIN_ID, BACKFILL_JOB)
|
|
38
|
+
HISTORY_SOURCE_NAME = dlt_source_name(PLUGIN_ID, HISTORY_JOB)
|
|
39
|
+
ATTACHMENT_CONTENT_SOURCE_NAME = dlt_source_name(PLUGIN_ID, ATTACHMENT_CONTENT_JOB)
|
|
40
|
+
|
|
41
|
+
BACKFILL_MESSAGES_ASSET_KEY = dagster_dlt_asset_key(BACKFILL_SOURCE_NAME, "messages")
|
|
42
|
+
HISTORY_EVENTS_ASSET_KEY = dagster_dlt_asset_key(HISTORY_SOURCE_NAME, "history_events")
|
|
43
|
+
HISTORY_MESSAGES_ASSET_KEY = dagster_dlt_asset_key(HISTORY_SOURCE_NAME, "messages")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _build_backfill_specs(
|
|
47
|
+
partitions_def: dg.PartitionsDefinition,
|
|
48
|
+
automation_condition: dg.AutomationCondition,
|
|
49
|
+
) -> list[dg.AssetSpec]:
|
|
50
|
+
shared = dict(
|
|
51
|
+
group_name=dagster_asset_group_name(PLUGIN_ID),
|
|
52
|
+
tags=dagster_asset_tags(PLUGIN_ID),
|
|
53
|
+
automation_condition=automation_condition,
|
|
54
|
+
partitions_def=partitions_def,
|
|
55
|
+
)
|
|
56
|
+
return [
|
|
57
|
+
dg.AssetSpec(
|
|
58
|
+
key=dagster_dlt_asset_key(BACKFILL_SOURCE_NAME, "profile"), **shared
|
|
59
|
+
),
|
|
60
|
+
dg.AssetSpec(
|
|
61
|
+
key=dagster_dlt_asset_key(BACKFILL_SOURCE_NAME, "labels"), **shared
|
|
62
|
+
),
|
|
63
|
+
dg.AssetSpec(key=BACKFILL_MESSAGES_ASSET_KEY, **shared),
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _build_history_specs(
|
|
68
|
+
partitions_def: dg.PartitionsDefinition,
|
|
69
|
+
automation_condition: dg.AutomationCondition,
|
|
70
|
+
) -> list[dg.AssetSpec]:
|
|
71
|
+
shared = dict(
|
|
72
|
+
group_name=dagster_asset_group_name(PLUGIN_ID),
|
|
73
|
+
tags=dagster_asset_tags(PLUGIN_ID),
|
|
74
|
+
automation_condition=automation_condition,
|
|
75
|
+
partitions_def=partitions_def,
|
|
76
|
+
)
|
|
77
|
+
return [
|
|
78
|
+
dg.AssetSpec(
|
|
79
|
+
key=dagster_dlt_asset_key(HISTORY_SOURCE_NAME, "profile"), **shared
|
|
80
|
+
),
|
|
81
|
+
dg.AssetSpec(
|
|
82
|
+
key=dagster_dlt_asset_key(HISTORY_SOURCE_NAME, "labels"), **shared
|
|
83
|
+
),
|
|
84
|
+
dg.AssetSpec(key=HISTORY_EVENTS_ASSET_KEY, **shared),
|
|
85
|
+
dg.AssetSpec(
|
|
86
|
+
key=HISTORY_MESSAGES_ASSET_KEY,
|
|
87
|
+
deps=[HISTORY_EVENTS_ASSET_KEY],
|
|
88
|
+
**shared,
|
|
89
|
+
),
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _build_attachment_content_specs(
|
|
94
|
+
partitions_def: dg.PartitionsDefinition,
|
|
95
|
+
automation_condition: dg.AutomationCondition,
|
|
96
|
+
) -> list[dg.AssetSpec]:
|
|
97
|
+
return [
|
|
98
|
+
dg.AssetSpec(
|
|
99
|
+
key=dagster_dlt_asset_key(ATTACHMENT_CONTENT_SOURCE_NAME, "attachments"),
|
|
100
|
+
group_name=dagster_asset_group_name(PLUGIN_ID),
|
|
101
|
+
tags=dagster_asset_tags(PLUGIN_ID),
|
|
102
|
+
automation_condition=automation_condition,
|
|
103
|
+
partitions_def=partitions_def,
|
|
104
|
+
deps=[BACKFILL_MESSAGES_ASSET_KEY, HISTORY_MESSAGES_ASSET_KEY],
|
|
105
|
+
),
|
|
106
|
+
]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _build_gmail_client(
|
|
110
|
+
binding: DagsterAllPlanBinding,
|
|
111
|
+
*,
|
|
112
|
+
control_plane: ControlPlaneClient,
|
|
113
|
+
) -> GmailApiClient:
|
|
114
|
+
authenticated_account = require_authenticated_account(binding)
|
|
115
|
+
return GmailApiClient(
|
|
116
|
+
service=build_google_service(
|
|
117
|
+
api_name="gmail",
|
|
118
|
+
api_version="v1",
|
|
119
|
+
auth=authenticated_account,
|
|
120
|
+
control_plane=control_plane,
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class GmailSyncComponent(dg.Component):
|
|
126
|
+
def build_defs(self, context: dg.ComponentLoadContext) -> dg.Definitions:
|
|
127
|
+
partitions_def = dg.DynamicPartitionsDefinition(
|
|
128
|
+
name=dagster_partition_def_name(PLUGIN_ID)
|
|
129
|
+
)
|
|
130
|
+
backfill_specs = _build_backfill_specs(
|
|
131
|
+
partitions_def,
|
|
132
|
+
non_overlapping_automation_condition(
|
|
133
|
+
dg.AutomationCondition.missing()
|
|
134
|
+
& ~dg.AutomationCondition.any_deps_missing()
|
|
135
|
+
),
|
|
136
|
+
)
|
|
137
|
+
history_specs = _build_history_specs(
|
|
138
|
+
partitions_def,
|
|
139
|
+
non_overlapping_automation_condition(
|
|
140
|
+
dg.AutomationCondition.on_cron("*/15 * * * *")
|
|
141
|
+
),
|
|
142
|
+
)
|
|
143
|
+
attachment_content_specs = _build_attachment_content_specs(
|
|
144
|
+
partitions_def,
|
|
145
|
+
non_overlapping_automation_condition(
|
|
146
|
+
dg.AutomationCondition.cron_tick_passed("*/10 * * * *")
|
|
147
|
+
),
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
@dg.multi_asset(
|
|
151
|
+
specs=backfill_specs,
|
|
152
|
+
can_subset=True,
|
|
153
|
+
name="gmail_backfill",
|
|
154
|
+
pool=dagster_pool_name(PLUGIN_ID),
|
|
155
|
+
)
|
|
156
|
+
def gmail_backfill_assets(
|
|
157
|
+
context: AssetExecutionContext,
|
|
158
|
+
dlt_resource: DagsterDltResource,
|
|
159
|
+
control_plane: dg.ResourceParam[ControlPlaneClient],
|
|
160
|
+
):
|
|
161
|
+
binding = resolve_partition_binding(
|
|
162
|
+
context=context,
|
|
163
|
+
control_plane=control_plane,
|
|
164
|
+
plugin_id=PLUGIN_ID,
|
|
165
|
+
)
|
|
166
|
+
binding_id = str(binding.binding_id)
|
|
167
|
+
parse_binding_config(binding, GmailBindingConfig)
|
|
168
|
+
client = _build_gmail_client(binding, control_plane=control_plane)
|
|
169
|
+
|
|
170
|
+
source = gmail_backfill_source(
|
|
171
|
+
binding_id,
|
|
172
|
+
client=client,
|
|
173
|
+
)
|
|
174
|
+
yield from run_dlt_pipeline(
|
|
175
|
+
context=context,
|
|
176
|
+
dlt_resource=dlt_resource,
|
|
177
|
+
source=source,
|
|
178
|
+
plugin_id=PLUGIN_ID,
|
|
179
|
+
binding_id=binding_id,
|
|
180
|
+
job_name=BACKFILL_JOB,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
@dg.multi_asset(
|
|
184
|
+
specs=history_specs,
|
|
185
|
+
can_subset=True,
|
|
186
|
+
name="gmail_history",
|
|
187
|
+
pool=dagster_pool_name(PLUGIN_ID),
|
|
188
|
+
)
|
|
189
|
+
def gmail_history_assets(
|
|
190
|
+
context: AssetExecutionContext,
|
|
191
|
+
dlt_resource: DagsterDltResource,
|
|
192
|
+
control_plane: dg.ResourceParam[ControlPlaneClient],
|
|
193
|
+
):
|
|
194
|
+
binding = resolve_partition_binding(
|
|
195
|
+
context=context,
|
|
196
|
+
control_plane=control_plane,
|
|
197
|
+
plugin_id=PLUGIN_ID,
|
|
198
|
+
)
|
|
199
|
+
binding_id = str(binding.binding_id)
|
|
200
|
+
parse_binding_config(binding, GmailBindingConfig)
|
|
201
|
+
client = _build_gmail_client(binding, control_plane=control_plane)
|
|
202
|
+
|
|
203
|
+
source = gmail_history_source(
|
|
204
|
+
binding_id,
|
|
205
|
+
client=client,
|
|
206
|
+
)
|
|
207
|
+
yield from run_dlt_pipeline(
|
|
208
|
+
context=context,
|
|
209
|
+
dlt_resource=dlt_resource,
|
|
210
|
+
source=source,
|
|
211
|
+
plugin_id=PLUGIN_ID,
|
|
212
|
+
binding_id=binding_id,
|
|
213
|
+
job_name=HISTORY_JOB,
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
@dg.multi_asset(
|
|
217
|
+
specs=attachment_content_specs,
|
|
218
|
+
name="gmail_attachment_content",
|
|
219
|
+
pool=dagster_pool_name(PLUGIN_ID),
|
|
220
|
+
)
|
|
221
|
+
def gmail_attachment_content_assets(
|
|
222
|
+
context: AssetExecutionContext,
|
|
223
|
+
dlt_resource: DagsterDltResource,
|
|
224
|
+
control_plane: dg.ResourceParam[ControlPlaneClient],
|
|
225
|
+
):
|
|
226
|
+
binding = resolve_partition_binding(
|
|
227
|
+
context=context,
|
|
228
|
+
control_plane=control_plane,
|
|
229
|
+
plugin_id=PLUGIN_ID,
|
|
230
|
+
)
|
|
231
|
+
binding_id = str(binding.binding_id)
|
|
232
|
+
parse_binding_config(binding, GmailBindingConfig)
|
|
233
|
+
client = _build_gmail_client(binding, control_plane=control_plane)
|
|
234
|
+
|
|
235
|
+
source = gmail_attachment_content_source(
|
|
236
|
+
binding_id,
|
|
237
|
+
client=client,
|
|
238
|
+
)
|
|
239
|
+
yield from run_dlt_pipeline(
|
|
240
|
+
context=context,
|
|
241
|
+
dlt_resource=dlt_resource,
|
|
242
|
+
source=source,
|
|
243
|
+
plugin_id=PLUGIN_ID,
|
|
244
|
+
binding_id=binding_id,
|
|
245
|
+
job_name=ATTACHMENT_CONTENT_JOB,
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
automation_sensor = dg.AutomationConditionSensorDefinition(
|
|
249
|
+
name="gmail_automation_sensor",
|
|
250
|
+
target=dg.AssetSelection.assets(
|
|
251
|
+
gmail_backfill_assets,
|
|
252
|
+
gmail_history_assets,
|
|
253
|
+
gmail_attachment_content_assets,
|
|
254
|
+
),
|
|
255
|
+
default_status=dg.DefaultSensorStatus.RUNNING,
|
|
256
|
+
minimum_interval_seconds=30,
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
return dg.Definitions(
|
|
260
|
+
assets=[
|
|
261
|
+
gmail_backfill_assets,
|
|
262
|
+
gmail_history_assets,
|
|
263
|
+
gmail_attachment_content_assets,
|
|
264
|
+
],
|
|
265
|
+
sensors=[automation_sensor],
|
|
266
|
+
resources={
|
|
267
|
+
"dlt_resource": DLT_RESOURCE,
|
|
268
|
+
},
|
|
269
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
type: plugin_gmail.component.GmailSyncComponent
|
|
File without changes
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import AwareDatetime, Field, field_validator
|
|
6
|
+
from shared_plugins.models import CtxModel, IdStr, NonNegativeInt, partialize
|
|
7
|
+
from shared_plugins.values import load_json_value, require_non_negative_int
|
|
8
|
+
|
|
9
|
+
from .ingress import GmailAttachmentIngress
|
|
10
|
+
from .types import HistoryId, MessageId, ThreadId
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ProfileRow(CtxModel):
|
|
14
|
+
email_address: str
|
|
15
|
+
messages_total: int | None = None
|
|
16
|
+
threads_total: int | None = None
|
|
17
|
+
history_id: HistoryId
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class LabelRow(CtxModel):
|
|
21
|
+
id: str
|
|
22
|
+
name: str
|
|
23
|
+
type: str | None = None
|
|
24
|
+
message_list_visibility: str | None = None
|
|
25
|
+
label_list_visibility: str | None = None
|
|
26
|
+
messages_total: int | None = None
|
|
27
|
+
messages_unread: int | None = None
|
|
28
|
+
threads_total: int | None = None
|
|
29
|
+
threads_unread: int | None = None
|
|
30
|
+
color: dict[str, str | None] | None = None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class HistoryEventRow(CtxModel):
|
|
34
|
+
id: HistoryId
|
|
35
|
+
messages: list[dict[str, Any]] = Field(default_factory=list)
|
|
36
|
+
messages_added: list[dict[str, Any]] = Field(default_factory=list)
|
|
37
|
+
messages_deleted: list[dict[str, Any]] = Field(default_factory=list)
|
|
38
|
+
labels_added: list[dict[str, Any]] = Field(default_factory=list)
|
|
39
|
+
labels_removed: list[dict[str, Any]] = Field(default_factory=list)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class MessageRow(CtxModel):
|
|
43
|
+
id: MessageId
|
|
44
|
+
thread_id: ThreadId
|
|
45
|
+
label_ids: list[str] = Field(default_factory=list)
|
|
46
|
+
snippet: str | None = None
|
|
47
|
+
history_id: HistoryId
|
|
48
|
+
internal_date: NonNegativeInt | None = None
|
|
49
|
+
size_estimate: NonNegativeInt | None = None
|
|
50
|
+
subject: str | None = None
|
|
51
|
+
from_address: str | None = None
|
|
52
|
+
to_addresses: str | None = None
|
|
53
|
+
cc_addresses: str | None = None
|
|
54
|
+
bcc_addresses: str | None = None
|
|
55
|
+
reply_to: str | None = None
|
|
56
|
+
message_id_header: str | None = None
|
|
57
|
+
in_reply_to: str | None = None
|
|
58
|
+
references_header: str | None = None
|
|
59
|
+
date: AwareDatetime | None = None
|
|
60
|
+
body_text: str | None = None
|
|
61
|
+
body_html: str | None = None
|
|
62
|
+
mime_type: str | None = None
|
|
63
|
+
classification_label_values: list[dict[str, Any]] = Field(default_factory=list)
|
|
64
|
+
attachment_count: NonNegativeInt = 0
|
|
65
|
+
attachments: list[dict[str, Any]] = Field(default_factory=list)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class AttachmentRow(CtxModel):
|
|
69
|
+
message_id: MessageId
|
|
70
|
+
part_id: IdStr
|
|
71
|
+
attachment_id: str | None = None
|
|
72
|
+
filename: str | None = None
|
|
73
|
+
mime_type: str | None = None
|
|
74
|
+
size: NonNegativeInt | None = None
|
|
75
|
+
content_disposition: str | None = None
|
|
76
|
+
content_id: str | None = None
|
|
77
|
+
file_path: str
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
MESSAGES_COLUMNS: dict[str, dict[str, str]] = {
|
|
81
|
+
"label_ids": {"data_type": "json"},
|
|
82
|
+
"id": {"description": "Gmail message ID. Primary key with _ctx_binding_id."},
|
|
83
|
+
"thread_id": {
|
|
84
|
+
"description": "Gmail thread ID. Group by this to get full conversations."
|
|
85
|
+
},
|
|
86
|
+
"snippet": {
|
|
87
|
+
"description": "Short plain-text preview (~200 chars). Use this instead of body_text for scanning/filtering."
|
|
88
|
+
},
|
|
89
|
+
"body_text": {
|
|
90
|
+
"description": "Full plain-text body including quoted reply chains. Replies contain all prior messages — use LEFT(body_text, N) and LIMIT. For reply-only content, look for 'On ... wrote:' or '___' separator markers."
|
|
91
|
+
},
|
|
92
|
+
"body_html": {
|
|
93
|
+
"description": 'Full HTML body. Quoted content is wrapped in <div class="gmail_quote"> or <blockquote>.'
|
|
94
|
+
},
|
|
95
|
+
"from_address": {
|
|
96
|
+
"description": "Sender as 'Display Name <email>' or bare email. Same person may appear with different display names or addresses."
|
|
97
|
+
},
|
|
98
|
+
"date": {"description": "Message date as timestamptz. Some spam has NULL dates."},
|
|
99
|
+
"subject": {
|
|
100
|
+
"description": "Subject line. Calendar invitations appear as 'Accepted:', 'Updated invitation:', etc."
|
|
101
|
+
},
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
MessageRowProjectionBase = partialize(
|
|
106
|
+
MessageRow,
|
|
107
|
+
name="MessageRowProjectionBase",
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class AttachmentCandidateProjection(MessageRowProjectionBase):
|
|
112
|
+
id: MessageId
|
|
113
|
+
attachment_count: NonNegativeInt
|
|
114
|
+
attachments: list[GmailAttachmentIngress]
|
|
115
|
+
existing_attachment_count: NonNegativeInt = 0
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def message_id(self) -> str:
|
|
119
|
+
return self.id
|
|
120
|
+
|
|
121
|
+
@field_validator("attachment_count", "existing_attachment_count", mode="before")
|
|
122
|
+
@classmethod
|
|
123
|
+
def _validate_required_non_negative_int(cls, value: object) -> int:
|
|
124
|
+
return require_non_negative_int(value)
|
|
125
|
+
|
|
126
|
+
@field_validator("attachments", mode="before")
|
|
127
|
+
@classmethod
|
|
128
|
+
def _coerce_attachments(cls, value: object) -> object:
|
|
129
|
+
loaded = load_json_value(value)
|
|
130
|
+
if not isinstance(loaded, list):
|
|
131
|
+
raise ValueError("must be a JSON array of attachment metadata")
|
|
132
|
+
return loaded
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import Field
|
|
6
|
+
from pydantic.types import NonNegativeInt as CoercedNonNegativeInt
|
|
7
|
+
from shared_plugins.models import IngressModel
|
|
8
|
+
|
|
9
|
+
from .types import LabelId, MessageId, NonNegativeInt, ThreadId
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class GmailMessagePartHeaderIngress(IngressModel):
|
|
13
|
+
name: str
|
|
14
|
+
value: str
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class GmailMessagePartBodyIngress(IngressModel):
|
|
18
|
+
attachment_id: str | None = Field(default=None, alias="attachmentId")
|
|
19
|
+
size: NonNegativeInt | None = None
|
|
20
|
+
data: str | None = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class GmailMessagePartIngress(IngressModel):
|
|
24
|
+
part_id: str | None = Field(default=None, alias="partId")
|
|
25
|
+
mime_type: str | None = Field(default=None, alias="mimeType")
|
|
26
|
+
filename: str | None = None
|
|
27
|
+
headers: list[GmailMessagePartHeaderIngress] = Field(default_factory=list)
|
|
28
|
+
body: GmailMessagePartBodyIngress | None = None
|
|
29
|
+
parts: list["GmailMessagePartIngress"] = Field(default_factory=list)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class GmailAttachmentIngress(IngressModel):
|
|
33
|
+
part_id: str = Field(min_length=1)
|
|
34
|
+
attachment_id: str | None = None
|
|
35
|
+
inline_data_b64url: str | None = None
|
|
36
|
+
filename: str | None = None
|
|
37
|
+
mime_type: str | None = None
|
|
38
|
+
size: NonNegativeInt | None = None
|
|
39
|
+
content_disposition: str | None = None
|
|
40
|
+
content_id: str | None = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class _GmailMessageIngressBase(IngressModel):
|
|
44
|
+
id: MessageId
|
|
45
|
+
label_ids: list[LabelId] = Field(default_factory=list, alias="labelIds")
|
|
46
|
+
snippet: str | None = None
|
|
47
|
+
history_id: CoercedNonNegativeInt | None = Field(default=None, alias="historyId")
|
|
48
|
+
internal_date: CoercedNonNegativeInt | None = Field(
|
|
49
|
+
default=None, alias="internalDate"
|
|
50
|
+
)
|
|
51
|
+
payload: GmailMessagePartIngress | None = None
|
|
52
|
+
size_estimate: NonNegativeInt | None = Field(default=None, alias="sizeEstimate")
|
|
53
|
+
classification_label_values: list[dict[str, Any]] = Field(
|
|
54
|
+
default_factory=list,
|
|
55
|
+
alias="classificationLabelValues",
|
|
56
|
+
)
|
|
57
|
+
raw: str | None = None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class GmailMessageIngress(_GmailMessageIngressBase):
|
|
61
|
+
history_id: CoercedNonNegativeInt = Field(alias="historyId")
|
|
62
|
+
thread_id: ThreadId = Field(alias="threadId")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class GmailMessageAttachmentIngress(IngressModel):
|
|
66
|
+
attachment_id: str = Field(min_length=1, alias="attachmentId")
|
|
67
|
+
size: NonNegativeInt | None = None
|
|
68
|
+
data: str | None = None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class GmailHistoryMessageIngress(_GmailMessageIngressBase):
|
|
72
|
+
thread_id: ThreadId | None = Field(default=None, alias="threadId")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class GmailThreadIngress(IngressModel):
|
|
76
|
+
id: ThreadId
|
|
77
|
+
history_id: CoercedNonNegativeInt = Field(alias="historyId")
|
|
78
|
+
snippet: str | None = None
|
|
79
|
+
messages: list[GmailMessageIngress] = Field(default_factory=list)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class GmailThreadListItemIngress(IngressModel):
|
|
83
|
+
id: ThreadId
|
|
84
|
+
history_id: CoercedNonNegativeInt = Field(alias="historyId")
|
|
85
|
+
snippet: str | None = None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class GmailLabelColorIngress(IngressModel):
|
|
89
|
+
text_color: str | None = Field(default=None, alias="textColor")
|
|
90
|
+
background_color: str | None = Field(default=None, alias="backgroundColor")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class GmailLabelIngress(IngressModel):
|
|
94
|
+
id: LabelId
|
|
95
|
+
name: str
|
|
96
|
+
type: str | None = None
|
|
97
|
+
message_list_visibility: str | None = Field(
|
|
98
|
+
default=None,
|
|
99
|
+
alias="messageListVisibility",
|
|
100
|
+
)
|
|
101
|
+
label_list_visibility: str | None = Field(default=None, alias="labelListVisibility")
|
|
102
|
+
messages_total: NonNegativeInt | None = Field(default=None, alias="messagesTotal")
|
|
103
|
+
messages_unread: NonNegativeInt | None = Field(default=None, alias="messagesUnread")
|
|
104
|
+
threads_total: NonNegativeInt | None = Field(default=None, alias="threadsTotal")
|
|
105
|
+
threads_unread: NonNegativeInt | None = Field(default=None, alias="threadsUnread")
|
|
106
|
+
color: GmailLabelColorIngress | None = None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class GmailProfileIngress(IngressModel):
|
|
110
|
+
email_address: str = Field(alias="emailAddress")
|
|
111
|
+
messages_total: NonNegativeInt | None = Field(default=None, alias="messagesTotal")
|
|
112
|
+
threads_total: NonNegativeInt | None = Field(default=None, alias="threadsTotal")
|
|
113
|
+
history_id: CoercedNonNegativeInt = Field(alias="historyId")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class GmailHistoryMessageAddedIngress(IngressModel):
|
|
117
|
+
message: GmailHistoryMessageIngress
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class GmailHistoryMessageDeletedIngress(IngressModel):
|
|
121
|
+
message: GmailHistoryMessageIngress
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class GmailHistoryLabelAddedIngress(IngressModel):
|
|
125
|
+
message: GmailHistoryMessageIngress
|
|
126
|
+
label_ids: list[LabelId] = Field(default_factory=list, alias="labelIds")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class GmailHistoryLabelRemovedIngress(IngressModel):
|
|
130
|
+
message: GmailHistoryMessageIngress
|
|
131
|
+
label_ids: list[LabelId] = Field(default_factory=list, alias="labelIds")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class GmailHistoryRecordIngress(IngressModel):
|
|
135
|
+
id: CoercedNonNegativeInt
|
|
136
|
+
messages: list[GmailHistoryMessageIngress] = Field(default_factory=list)
|
|
137
|
+
messages_added: list[GmailHistoryMessageAddedIngress] = Field(
|
|
138
|
+
default_factory=list,
|
|
139
|
+
alias="messagesAdded",
|
|
140
|
+
)
|
|
141
|
+
messages_deleted: list[GmailHistoryMessageDeletedIngress] = Field(
|
|
142
|
+
default_factory=list,
|
|
143
|
+
alias="messagesDeleted",
|
|
144
|
+
)
|
|
145
|
+
labels_added: list[GmailHistoryLabelAddedIngress] = Field(
|
|
146
|
+
default_factory=list,
|
|
147
|
+
alias="labelsAdded",
|
|
148
|
+
)
|
|
149
|
+
labels_removed: list[GmailHistoryLabelRemovedIngress] = Field(
|
|
150
|
+
default_factory=list,
|
|
151
|
+
alias="labelsRemoved",
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class GmailMessagesListItemIngress(IngressModel):
|
|
156
|
+
id: MessageId
|
|
157
|
+
thread_id: ThreadId | None = Field(default=None, alias="threadId")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class GmailMessagesListResponseIngress(IngressModel):
|
|
161
|
+
messages: list[GmailMessagesListItemIngress] = Field(default_factory=list)
|
|
162
|
+
next_page_token: str | None = Field(default=None, alias="nextPageToken")
|
|
163
|
+
result_size_estimate: NonNegativeInt | None = Field(
|
|
164
|
+
default=None,
|
|
165
|
+
alias="resultSizeEstimate",
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class GmailThreadsListResponseIngress(IngressModel):
|
|
170
|
+
threads: list[GmailThreadListItemIngress] = Field(default_factory=list)
|
|
171
|
+
next_page_token: str | None = Field(default=None, alias="nextPageToken")
|
|
172
|
+
result_size_estimate: NonNegativeInt | None = Field(
|
|
173
|
+
default=None,
|
|
174
|
+
alias="resultSizeEstimate",
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class GmailLabelsListResponseIngress(IngressModel):
|
|
179
|
+
labels: list[GmailLabelIngress] = Field(default_factory=list)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class GmailHistoryListResponseIngress(IngressModel):
|
|
183
|
+
history: list[GmailHistoryRecordIngress] = Field(default_factory=list)
|
|
184
|
+
next_page_token: str | None = Field(default=None, alias="nextPageToken")
|
|
185
|
+
history_id: CoercedNonNegativeInt = Field(alias="historyId")
|