contextbase-shared-plugins 0.0.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contextbase_shared_plugins-0.0.0a1.dist-info/METADATA +27 -0
- contextbase_shared_plugins-0.0.0a1.dist-info/RECORD +41 -0
- contextbase_shared_plugins-0.0.0a1.dist-info/WHEEL +4 -0
- shared_plugins/__init__.py +8 -0
- shared_plugins/automation.py +11 -0
- shared_plugins/base_platform.py +105 -0
- shared_plugins/bindings.py +260 -0
- shared_plugins/dlt.py +89 -0
- shared_plugins/env.py +104 -0
- shared_plugins/exceptions.py +10 -0
- shared_plugins/google_client/__init__.py +1 -0
- shared_plugins/google_client/auth.py +108 -0
- shared_plugins/google_client/batch_retry.py +308 -0
- shared_plugins/google_client/http_errors.py +27 -0
- shared_plugins/machine_token.py +419 -0
- shared_plugins/microsoft_dataverse/__init__.py +27 -0
- shared_plugins/microsoft_dataverse/annotations.py +61 -0
- shared_plugins/microsoft_dataverse/auth.py +26 -0
- shared_plugins/microsoft_dataverse/binding_config.py +35 -0
- shared_plugins/microsoft_dataverse/client.py +468 -0
- shared_plugins/microsoft_dataverse/ctx.py +21 -0
- shared_plugins/microsoft_dataverse/identifiers.py +62 -0
- shared_plugins/microsoft_dataverse/ingress.py +53 -0
- shared_plugins/microsoft_dataverse/metadata.py +106 -0
- shared_plugins/microsoft_dataverse/runtime_schema.py +332 -0
- shared_plugins/microsoft_dataverse/source.py +299 -0
- shared_plugins/microsoft_dataverse/tables.py +34 -0
- shared_plugins/microsoft_dataverse/translators.py +133 -0
- shared_plugins/microsoft_dataverse/types.py +355 -0
- shared_plugins/microsoft_graph.py +250 -0
- shared_plugins/models.py +91 -0
- shared_plugins/naming.py +83 -0
- shared_plugins/pg_column_comments.py +59 -0
- shared_plugins/provider_token.py +238 -0
- shared_plugins/pyairbyte.py +485 -0
- shared_plugins/resources.py +179 -0
- shared_plugins/scratch.py +127 -0
- shared_plugins/sentry.py +117 -0
- shared_plugins/sqlalchemy_types.py +225 -0
- shared_plugins/sqlite.py +123 -0
- shared_plugins/values.py +117 -0
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
"""Dataverse AttributeType → dlt TColumnSchema mapping.
|
|
2
|
+
|
|
3
|
+
Single source of truth for every type-mapping decision the plan §9 locks in.
|
|
4
|
+
Unknown types raise loudly — new Microsoft types must be added explicitly.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from shared_plugins.exceptions import PluginConfigurationError
|
|
13
|
+
|
|
14
|
+
from .annotations import ODATA_ANNOTATION_COLUMN_SUFFIXES
|
|
15
|
+
from .identifiers import annotation_column_name
|
|
16
|
+
from .metadata import AttributeMetadataIngress, extract_label
|
|
17
|
+
|
|
18
|
+
# Existing skips preserved from plugin's runtime_ingress.py.
|
|
19
|
+
SKIPPED_ATTRIBUTE_TYPES: frozenset[str] = frozenset(
|
|
20
|
+
{
|
|
21
|
+
"Virtual",
|
|
22
|
+
"File",
|
|
23
|
+
"Image",
|
|
24
|
+
"CalendarRules",
|
|
25
|
+
"PartyList",
|
|
26
|
+
"ManagedProperty",
|
|
27
|
+
}
|
|
28
|
+
)
|
|
29
|
+
# Attributes that appear in EntityDefinitions metadata but are not exposed as
|
|
30
|
+
# OData-selectable properties on the entity type, so a $select naming them
|
|
31
|
+
# fails with HTTP 400 ("Could not find a property named ..."). No single
|
|
32
|
+
# metadata flag distinguishes them (IsValidODataAttribute is false for many
|
|
33
|
+
# perfectly selectable columns too), so they are enumerated explicitly:
|
|
34
|
+
# stageid — BPF stage pointer, present on most entities
|
|
35
|
+
# attachmentcount — activity attachment rollup (email/appointment)
|
|
36
|
+
SKIPPED_ATTRIBUTE_NAMES: frozenset[str] = frozenset({"stageid", "attachmentcount"})
|
|
37
|
+
|
|
38
|
+
LOOKUP_ATTRIBUTE_TYPES: frozenset[str] = frozenset({"Customer", "Lookup", "Owner"})
|
|
39
|
+
PICKLIST_ATTRIBUTE_TYPES: frozenset[str] = frozenset({"Picklist", "State", "Status"})
|
|
40
|
+
|
|
41
|
+
# Scalar AttributeTypes that emit @OData.Community.Display.V1.FormattedValue
|
|
42
|
+
# alongside their raw value, so the builder reserves a _formatted_value
|
|
43
|
+
# companion column. Empirically observed: all numeric scalars + Boolean emit a
|
|
44
|
+
# locale-rendered string (customer tenant, May 2026); EntityName emits the
|
|
45
|
+
# referenced entity's display label (e.g. activitytypecode -> "Email" /
|
|
46
|
+
# "Meeting" on the activity entities, observed 2026-06-20). Memo / String /
|
|
47
|
+
# Uniqueidentifier do not. If a new type starts emitting one, the per-table
|
|
48
|
+
# model's extra="forbid" fails loud at validation — add it here then.
|
|
49
|
+
_SCALAR_TYPES_WITH_FORMATTED_VALUE: frozenset[str] = frozenset(
|
|
50
|
+
{"Integer", "BigInt", "Double", "Boolean", "EntityName"}
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
_ANNOTATION_DESCRIPTIONS: dict[str, str] = {
|
|
54
|
+
"_formatted_value": "Formatted value of {base}",
|
|
55
|
+
"_lookup_logical_name": "Logical-name of the entity referenced by {base}",
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class RuntimeAnnotationColumn:
|
|
61
|
+
column_name: str
|
|
62
|
+
pg_type: str
|
|
63
|
+
description: str | None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class RuntimeFieldSpec:
|
|
68
|
+
"""Per-attribute description ready for column emission.
|
|
69
|
+
|
|
70
|
+
Captures both Description and DisplayName for the
|
|
71
|
+
`description ?? display_name` fallback at the dlt edge. Other
|
|
72
|
+
consumers (LLM tool definitions, sibling metadata exporters) can
|
|
73
|
+
read the rich object directly without re-deriving from raw metadata.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
logical_name: str
|
|
77
|
+
pg_type: str
|
|
78
|
+
pg_type_args: dict[str, Any]
|
|
79
|
+
description: str | None
|
|
80
|
+
display_name: str | None
|
|
81
|
+
is_custom_attribute: bool
|
|
82
|
+
attribute_type: str
|
|
83
|
+
annotation_columns: tuple[RuntimeAnnotationColumn, ...]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def field_spec_for_attribute(
|
|
87
|
+
attribute: AttributeMetadataIngress,
|
|
88
|
+
) -> RuntimeFieldSpec | None:
|
|
89
|
+
"""Build a RuntimeFieldSpec from Dataverse attribute metadata.
|
|
90
|
+
|
|
91
|
+
Returns None if the attribute should be skipped per the existing filter
|
|
92
|
+
rules (IsValidForRead=False, AttributeOf set, type/name in skip sets).
|
|
93
|
+
|
|
94
|
+
Raises PluginConfigurationError if AttributeType is unknown — new
|
|
95
|
+
Microsoft types require explicit triage in this module.
|
|
96
|
+
"""
|
|
97
|
+
if attribute.is_valid_for_read is False:
|
|
98
|
+
return None
|
|
99
|
+
if attribute.attribute_of:
|
|
100
|
+
return None
|
|
101
|
+
if attribute.logical_name in SKIPPED_ATTRIBUTE_NAMES:
|
|
102
|
+
return None
|
|
103
|
+
|
|
104
|
+
attr_type = attribute.attribute_type or ""
|
|
105
|
+
|
|
106
|
+
# MultiSelectPicklist arrives as Virtual + AttributeTypeName.Value=MultiSelectPicklistType.
|
|
107
|
+
if attribute.type_name_value == "MultiSelectPicklistType":
|
|
108
|
+
return _build_multiselect_field(attribute)
|
|
109
|
+
|
|
110
|
+
if attr_type in SKIPPED_ATTRIBUTE_TYPES:
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
if attr_type in LOOKUP_ATTRIBUTE_TYPES:
|
|
114
|
+
return _build_lookup_field(attribute)
|
|
115
|
+
|
|
116
|
+
if attr_type in PICKLIST_ATTRIBUTE_TYPES:
|
|
117
|
+
return _build_picklist_field(attribute)
|
|
118
|
+
|
|
119
|
+
if attr_type == "DateTime":
|
|
120
|
+
return _build_datetime_field(attribute)
|
|
121
|
+
|
|
122
|
+
if attr_type in ("Decimal", "Money"):
|
|
123
|
+
return _build_numeric_field(attribute)
|
|
124
|
+
|
|
125
|
+
pg_type = _SCALAR_TYPE_MAP.get(attr_type)
|
|
126
|
+
if pg_type is None:
|
|
127
|
+
raise PluginConfigurationError(
|
|
128
|
+
f"Unknown Dataverse AttributeType {attr_type!r} for attribute "
|
|
129
|
+
f"{attribute.logical_name!r}. New Microsoft types require explicit "
|
|
130
|
+
"triage in shared_plugins.microsoft_dataverse.types."
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
if attr_type in _SCALAR_TYPES_WITH_FORMATTED_VALUE:
|
|
134
|
+
annotations = _build_annotation_columns(
|
|
135
|
+
attribute.logical_name,
|
|
136
|
+
kept_suffixes=("_formatted_value",),
|
|
137
|
+
context_table=attribute.logical_name,
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
annotations = ()
|
|
141
|
+
|
|
142
|
+
return RuntimeFieldSpec(
|
|
143
|
+
logical_name=attribute.logical_name,
|
|
144
|
+
pg_type=pg_type,
|
|
145
|
+
pg_type_args={},
|
|
146
|
+
description=extract_label(attribute.description),
|
|
147
|
+
display_name=extract_label(attribute.display_name),
|
|
148
|
+
is_custom_attribute=attribute.is_custom_attribute,
|
|
149
|
+
attribute_type=attr_type,
|
|
150
|
+
annotation_columns=annotations,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
_SCALAR_TYPE_MAP: dict[str, str] = {
|
|
155
|
+
"String": "text",
|
|
156
|
+
"Memo": "text",
|
|
157
|
+
"Integer": "bigint",
|
|
158
|
+
"BigInt": "bigint",
|
|
159
|
+
"Double": "double",
|
|
160
|
+
"Boolean": "bool",
|
|
161
|
+
"Uniqueidentifier": "uuid",
|
|
162
|
+
"EntityName": "text",
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _build_lookup_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
|
|
167
|
+
base = f"_{attr.logical_name}_value"
|
|
168
|
+
annotations = _build_annotation_columns(
|
|
169
|
+
base,
|
|
170
|
+
kept_suffixes=("_formatted_value", "_lookup_logical_name"),
|
|
171
|
+
context_table=attr.logical_name,
|
|
172
|
+
)
|
|
173
|
+
return RuntimeFieldSpec(
|
|
174
|
+
logical_name=base,
|
|
175
|
+
pg_type="uuid",
|
|
176
|
+
pg_type_args={},
|
|
177
|
+
description=extract_label(attr.description),
|
|
178
|
+
display_name=extract_label(attr.display_name),
|
|
179
|
+
is_custom_attribute=attr.is_custom_attribute,
|
|
180
|
+
attribute_type=attr.attribute_type or "",
|
|
181
|
+
annotation_columns=annotations,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _build_picklist_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
|
|
186
|
+
annotations = _build_annotation_columns(
|
|
187
|
+
attr.logical_name,
|
|
188
|
+
kept_suffixes=("_formatted_value",),
|
|
189
|
+
context_table=attr.logical_name,
|
|
190
|
+
)
|
|
191
|
+
return RuntimeFieldSpec(
|
|
192
|
+
logical_name=attr.logical_name,
|
|
193
|
+
pg_type="bigint",
|
|
194
|
+
pg_type_args={},
|
|
195
|
+
description=extract_label(attr.description),
|
|
196
|
+
display_name=extract_label(attr.display_name),
|
|
197
|
+
is_custom_attribute=attr.is_custom_attribute,
|
|
198
|
+
attribute_type=attr.attribute_type or "",
|
|
199
|
+
annotation_columns=annotations,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _build_multiselect_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
|
|
204
|
+
annotations = _build_annotation_columns(
|
|
205
|
+
attr.logical_name,
|
|
206
|
+
kept_suffixes=("_formatted_value",),
|
|
207
|
+
context_table=attr.logical_name,
|
|
208
|
+
)
|
|
209
|
+
return RuntimeFieldSpec(
|
|
210
|
+
logical_name=attr.logical_name,
|
|
211
|
+
pg_type="bigint[]",
|
|
212
|
+
pg_type_args={},
|
|
213
|
+
description=extract_label(attr.description),
|
|
214
|
+
display_name=extract_label(attr.display_name),
|
|
215
|
+
is_custom_attribute=attr.is_custom_attribute,
|
|
216
|
+
attribute_type=attr.attribute_type or "MultiSelectPicklist",
|
|
217
|
+
annotation_columns=annotations,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _build_datetime_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
|
|
222
|
+
fmt = attr.format
|
|
223
|
+
behavior = (attr.date_time_behavior or {}).get("Value")
|
|
224
|
+
if fmt == "DateOnly":
|
|
225
|
+
pg_type = "date"
|
|
226
|
+
pg_args: dict[str, Any] = {}
|
|
227
|
+
else:
|
|
228
|
+
pg_type = "timestamp"
|
|
229
|
+
pg_args = {"timezone": behavior != "TimeZoneIndependent"}
|
|
230
|
+
|
|
231
|
+
annotations = _build_annotation_columns(
|
|
232
|
+
attr.logical_name,
|
|
233
|
+
kept_suffixes=("_formatted_value",),
|
|
234
|
+
context_table=attr.logical_name,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
return RuntimeFieldSpec(
|
|
238
|
+
logical_name=attr.logical_name,
|
|
239
|
+
pg_type=pg_type,
|
|
240
|
+
pg_type_args=pg_args,
|
|
241
|
+
description=extract_label(attr.description),
|
|
242
|
+
display_name=extract_label(attr.display_name),
|
|
243
|
+
is_custom_attribute=attr.is_custom_attribute,
|
|
244
|
+
attribute_type=attr.attribute_type or "DateTime",
|
|
245
|
+
annotation_columns=annotations,
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _build_numeric_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
|
|
250
|
+
if attr.attribute_type == "Money":
|
|
251
|
+
precision = 28
|
|
252
|
+
scale = 4
|
|
253
|
+
else:
|
|
254
|
+
precision = 28
|
|
255
|
+
scale = attr.precision if attr.precision is not None else 4
|
|
256
|
+
|
|
257
|
+
annotations = _build_annotation_columns(
|
|
258
|
+
attr.logical_name,
|
|
259
|
+
kept_suffixes=("_formatted_value",),
|
|
260
|
+
context_table=attr.logical_name,
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
return RuntimeFieldSpec(
|
|
264
|
+
logical_name=attr.logical_name,
|
|
265
|
+
pg_type="numeric",
|
|
266
|
+
pg_type_args={"precision": precision, "scale": scale},
|
|
267
|
+
description=extract_label(attr.description),
|
|
268
|
+
display_name=extract_label(attr.display_name),
|
|
269
|
+
is_custom_attribute=attr.is_custom_attribute,
|
|
270
|
+
attribute_type=attr.attribute_type or "",
|
|
271
|
+
annotation_columns=annotations,
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _build_annotation_columns(
|
|
276
|
+
base: str,
|
|
277
|
+
*,
|
|
278
|
+
kept_suffixes: tuple[str, ...],
|
|
279
|
+
context_table: str,
|
|
280
|
+
) -> tuple[RuntimeAnnotationColumn, ...]:
|
|
281
|
+
columns: list[RuntimeAnnotationColumn] = []
|
|
282
|
+
for suffix in kept_suffixes:
|
|
283
|
+
# Verify suffix is in the registry; this catches typos.
|
|
284
|
+
if suffix not in ODATA_ANNOTATION_COLUMN_SUFFIXES.values():
|
|
285
|
+
raise PluginConfigurationError(
|
|
286
|
+
f"Annotation suffix {suffix!r} is not registered in "
|
|
287
|
+
"ODATA_ANNOTATION_COLUMN_SUFFIXES."
|
|
288
|
+
)
|
|
289
|
+
column_name = annotation_column_name(
|
|
290
|
+
base,
|
|
291
|
+
suffix,
|
|
292
|
+
context=f"{context_table}.{base}",
|
|
293
|
+
)
|
|
294
|
+
description_template = _ANNOTATION_DESCRIPTIONS[suffix]
|
|
295
|
+
columns.append(
|
|
296
|
+
RuntimeAnnotationColumn(
|
|
297
|
+
column_name=column_name,
|
|
298
|
+
pg_type="text",
|
|
299
|
+
description=description_template.format(base=base),
|
|
300
|
+
)
|
|
301
|
+
)
|
|
302
|
+
return tuple(columns)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
# dlt's TDataType literal (text, double, bool, timestamp, bigint, binary, json,
|
|
306
|
+
# decimal, wei, date, time) does not include "uuid", "numeric", or "bigint[]"
|
|
307
|
+
# today. We keep the richer pg_type on RuntimeFieldSpec so non-dlt consumers
|
|
308
|
+
# (LLM tool descriptions, sibling exporters, postgres-native upgrades) retain
|
|
309
|
+
# the fidelity. When dlt adds native uuid / numeric / array support, delete
|
|
310
|
+
# the corresponding entries from this mapping and pass pg_type through
|
|
311
|
+
# unchanged.
|
|
312
|
+
# Tracked: dlt's TDataType definition lives at
|
|
313
|
+
# dlt/common/data_types/typing.py (Literal); follow upstream support there.
|
|
314
|
+
_DLT_DATA_TYPE_OVERRIDES: dict[str, str] = {
|
|
315
|
+
"uuid": "text",
|
|
316
|
+
"numeric": "decimal",
|
|
317
|
+
"bigint[]": "json",
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _to_dlt_data_type(pg_type: str) -> str:
|
|
322
|
+
return _DLT_DATA_TYPE_OVERRIDES.get(pg_type, pg_type)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def build_dlt_column_hints(
|
|
326
|
+
fields: tuple[RuntimeFieldSpec, ...],
|
|
327
|
+
*,
|
|
328
|
+
sync_mode_is_delta: bool,
|
|
329
|
+
) -> dict[str, dict[str, Any]]:
|
|
330
|
+
"""Build the dlt `columns={col: {...}}` hint for `@ctx_dlt_resource`.
|
|
331
|
+
|
|
332
|
+
Description policy: `description ?? display_name`. If both are absent,
|
|
333
|
+
`description` is omitted from the hint entirely.
|
|
334
|
+
"""
|
|
335
|
+
hints: dict[str, dict[str, Any]] = {}
|
|
336
|
+
for field in fields:
|
|
337
|
+
entry: dict[str, Any] = {
|
|
338
|
+
"data_type": _to_dlt_data_type(field.pg_type),
|
|
339
|
+
**field.pg_type_args,
|
|
340
|
+
}
|
|
341
|
+
column_doc = field.description or field.display_name
|
|
342
|
+
if column_doc:
|
|
343
|
+
entry["description"] = column_doc
|
|
344
|
+
hints[field.logical_name] = entry
|
|
345
|
+
|
|
346
|
+
for ann in field.annotation_columns:
|
|
347
|
+
ann_entry: dict[str, Any] = {"data_type": _to_dlt_data_type(ann.pg_type)}
|
|
348
|
+
if ann.description:
|
|
349
|
+
ann_entry["description"] = ann.description
|
|
350
|
+
hints[ann.column_name] = ann_entry
|
|
351
|
+
|
|
352
|
+
if sync_mode_is_delta:
|
|
353
|
+
hints["is_deleted"] = {"hard_delete": True}
|
|
354
|
+
|
|
355
|
+
return hints
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""Shared Microsoft Graph client mechanics for Graph-backed plugins.
|
|
2
|
+
|
|
3
|
+
SDK-agnostic on purpose: the version-specific pieces — the SDK's
|
|
4
|
+
`GraphRequestAdapter` class, the default middleware `options` from its
|
|
5
|
+
`graph_request_adapter` module, and the matching `APIVersion` — are passed in
|
|
6
|
+
by each plugin, so a plugin pulls in only the Graph SDK it actually uses
|
|
7
|
+
(`msgraph` for v1.0, `msgraph_beta` for beta) and shared_plugins depends only
|
|
8
|
+
on the common `msgraph-core`/kiota runtime.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
14
|
+
from dataclasses import dataclass, make_dataclass
|
|
15
|
+
from typing import Any, TypeVar
|
|
16
|
+
from urllib.parse import parse_qs, quote, urlparse
|
|
17
|
+
|
|
18
|
+
import httpx
|
|
19
|
+
from azure.core.credentials import AccessToken
|
|
20
|
+
from azure.core.credentials_async import AsyncTokenCredential
|
|
21
|
+
from azure.identity.aio import ClientSecretCredential
|
|
22
|
+
from kiota_abstractions.base_request_configuration import RequestConfiguration
|
|
23
|
+
from kiota_abstractions.headers_collection import HeadersCollection
|
|
24
|
+
from kiota_abstractions.serialization import Parsable
|
|
25
|
+
from kiota_authentication_azure.azure_identity_authentication_provider import (
|
|
26
|
+
AzureIdentityAuthenticationProvider,
|
|
27
|
+
)
|
|
28
|
+
from kiota_serialization_json.json_serialization_writer import JsonSerializationWriter
|
|
29
|
+
from msgraph_core import APIVersion, BaseGraphRequestAdapter, GraphClientFactory
|
|
30
|
+
|
|
31
|
+
from base_client import (
|
|
32
|
+
BindingAuth_AuthenticatedAccount,
|
|
33
|
+
BindingAuth_ClientCredentials,
|
|
34
|
+
DagsterAllPlanBinding,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
from .provider_token import DelegatedTokenProvider, ProviderTokenClient
|
|
38
|
+
|
|
39
|
+
_GRAPH_SCOPES = ["https://graph.microsoft.com/.default"]
|
|
40
|
+
|
|
41
|
+
_RequestAdapterT = TypeVar("_RequestAdapterT", bound=BaseGraphRequestAdapter)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build_h11_request_adapter(
|
|
45
|
+
credential: AsyncTokenCredential,
|
|
46
|
+
*,
|
|
47
|
+
api_version: APIVersion,
|
|
48
|
+
adapter_cls: type[_RequestAdapterT],
|
|
49
|
+
adapter_options: dict[str, Any],
|
|
50
|
+
) -> _RequestAdapterT:
|
|
51
|
+
"""Build a Graph request adapter whose underlying httpx transport is
|
|
52
|
+
HTTP/1.1 only.
|
|
53
|
+
|
|
54
|
+
`api_version`, `adapter_cls`, and `adapter_options` come as a set from
|
|
55
|
+
the calling plugin's msgraph SDK: the `GraphRequestAdapter` class and the
|
|
56
|
+
default middleware `options` dict from its `graph_request_adapter`
|
|
57
|
+
module, plus the `APIVersion` it serves.
|
|
58
|
+
|
|
59
|
+
Force HTTP/1.1 (kiota's default is `http2=True`). Microsoft Graph's
|
|
60
|
+
frontdoor terminates HTTP/2 connections at ~30s with an advisory
|
|
61
|
+
GOAWAY (`error_code=NO_ERROR`, `last_stream_id=2^31-1`; RFC 7540
|
|
62
|
+
§6.8 graceful shutdown). httpcore turns that into a fatal
|
|
63
|
+
`RemoteProtocolError` for the in-flight stream
|
|
64
|
+
(`httpcore/_async/http2.py:350-355` — `stream_id > last_stream_id`
|
|
65
|
+
is unreachable when `last_stream_id` is `HIGHEST_ALLOWED_STREAM_ID`),
|
|
66
|
+
and kiota_http's `RetryHandler` doesn't catch transport exceptions
|
|
67
|
+
(it retries on status codes 429/503/504 only). HTTP/1.1 keep-alive
|
|
68
|
+
has no equivalent server-driven termination — empirically, a single
|
|
69
|
+
TCP connection comfortably serves 600+ requests over ~40s with the
|
|
70
|
+
same per-request latency as HTTP/2. Full empirical breakdown at
|
|
71
|
+
`internal/in-progress/2026-05-14-codecreators-komal-first-sync.md`
|
|
72
|
+
(B4 entry).
|
|
73
|
+
"""
|
|
74
|
+
raw = httpx.AsyncClient(
|
|
75
|
+
http2=False,
|
|
76
|
+
base_url=f"https://graph.microsoft.com/{api_version.value}",
|
|
77
|
+
)
|
|
78
|
+
wrapped = GraphClientFactory.create_with_default_middleware(
|
|
79
|
+
api_version=api_version,
|
|
80
|
+
client=raw,
|
|
81
|
+
options=adapter_options,
|
|
82
|
+
)
|
|
83
|
+
auth_provider = AzureIdentityAuthenticationProvider(
|
|
84
|
+
credential, scopes=_GRAPH_SCOPES
|
|
85
|
+
)
|
|
86
|
+
return adapter_cls(auth_provider, client=wrapped)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class DelegatedTokenCredential:
|
|
90
|
+
"""Adapts a DelegatedTokenProvider to azure's AsyncTokenCredential so a Graph
|
|
91
|
+
SDK's AzureIdentityAuthenticationProvider can stamp a delegated
|
|
92
|
+
(authenticated_account) bearer. The delegated token's scopes are fixed by the
|
|
93
|
+
grant, so the requested Graph scopes are ignored.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
def __init__(self, provider: DelegatedTokenProvider) -> None:
|
|
97
|
+
self._provider = provider
|
|
98
|
+
|
|
99
|
+
async def get_token(self, *scopes: str, **kwargs: object) -> AccessToken:
|
|
100
|
+
# current_lease() redeems synchronously on first use and may do so again
|
|
101
|
+
# during refresh; loop isolation is the calling client's concern.
|
|
102
|
+
lease = self._provider.current_lease()
|
|
103
|
+
if lease.access_token_expires_at is None:
|
|
104
|
+
raise ValueError(
|
|
105
|
+
"delegated microsoft lease declares no expiry; cannot build an "
|
|
106
|
+
"AccessToken.expires_on."
|
|
107
|
+
)
|
|
108
|
+
return AccessToken(
|
|
109
|
+
lease.access_token, int(lease.access_token_expires_at.timestamp())
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
async def close(self) -> None:
|
|
113
|
+
# The ProviderTokenClient is a Dagster resource; its lifecycle isn't ours.
|
|
114
|
+
return None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def build_credential_factory(
|
|
118
|
+
binding: DagsterAllPlanBinding,
|
|
119
|
+
*,
|
|
120
|
+
tenant_id: str,
|
|
121
|
+
provider_token: ProviderTokenClient,
|
|
122
|
+
) -> Callable[[], AsyncTokenCredential]:
|
|
123
|
+
"""One credential factory per Graph binding's auth: `client_credentials`
|
|
124
|
+
builds an app credential from the binding's secrets; `authenticated_account`
|
|
125
|
+
wraps a DelegatedTokenProvider in a DelegatedTokenCredential. Graph reads
|
|
126
|
+
address `/users/{id}` identically whichever credential produced the bearer,
|
|
127
|
+
so `binding.auth` branches ONLY the credential the client opens with. A
|
|
128
|
+
factory (not an instance) because Graph sync clients build it inside their own
|
|
129
|
+
event loop — aio credentials are loop-bound.
|
|
130
|
+
"""
|
|
131
|
+
auth = binding.auth
|
|
132
|
+
if isinstance(auth, BindingAuth_ClientCredentials):
|
|
133
|
+
return lambda: ClientSecretCredential(
|
|
134
|
+
tenant_id=tenant_id,
|
|
135
|
+
client_id=auth.client_id,
|
|
136
|
+
client_secret=auth.client_secret,
|
|
137
|
+
)
|
|
138
|
+
if isinstance(auth, BindingAuth_AuthenticatedAccount):
|
|
139
|
+
provider = DelegatedTokenProvider(
|
|
140
|
+
auth=auth, provider_token_client=provider_token
|
|
141
|
+
)
|
|
142
|
+
return lambda: DelegatedTokenCredential(provider)
|
|
143
|
+
raise RuntimeError(
|
|
144
|
+
f"{binding.plugin_id} binding requires auth.type 'client_credentials' or "
|
|
145
|
+
f"'authenticated_account', got '{auth.type}'."
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _query_parameters_from_mapping(
|
|
150
|
+
query_params: Mapping[str, object | None] | None,
|
|
151
|
+
) -> object | None:
|
|
152
|
+
values = [
|
|
153
|
+
(f"param_{index}", name, value)
|
|
154
|
+
for index, (name, value) in enumerate((query_params or {}).items())
|
|
155
|
+
if value is not None
|
|
156
|
+
]
|
|
157
|
+
if not values:
|
|
158
|
+
return None
|
|
159
|
+
|
|
160
|
+
encoded_name_by_field = {
|
|
161
|
+
field_name: quote(name, safe="") for field_name, name, _value in values
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
def get_query_parameter(_self: object, original_name: str) -> str:
|
|
165
|
+
return encoded_name_by_field.get(original_name, original_name)
|
|
166
|
+
|
|
167
|
+
query_type = make_dataclass(
|
|
168
|
+
"_GraphQueryParameters",
|
|
169
|
+
[(field_name, object) for field_name, _name, _value in values],
|
|
170
|
+
namespace={"get_query_parameter": get_query_parameter},
|
|
171
|
+
frozen=True,
|
|
172
|
+
)
|
|
173
|
+
return query_type(
|
|
174
|
+
**{field_name: value for field_name, _name, value in values},
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def request_configuration(
|
|
179
|
+
*,
|
|
180
|
+
query_params: Mapping[str, object | None] | None = None,
|
|
181
|
+
prefer_header: str | None = None,
|
|
182
|
+
) -> RequestConfiguration[Any]:
|
|
183
|
+
cfg: RequestConfiguration[Any] = RequestConfiguration()
|
|
184
|
+
cfg.headers = HeadersCollection()
|
|
185
|
+
cfg.query_parameters = _query_parameters_from_mapping(query_params)
|
|
186
|
+
if prefer_header:
|
|
187
|
+
cfg.headers.add("Prefer", prefer_header)
|
|
188
|
+
return cfg
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@dataclass(frozen=True)
|
|
192
|
+
class DeltaPage:
|
|
193
|
+
rows: list[Any]
|
|
194
|
+
cursor_url: str
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def is_delta_cursor_url(cursor_url: str) -> bool:
|
|
198
|
+
query = parse_qs(urlparse(cursor_url).query)
|
|
199
|
+
return "$deltatoken" in query or "deltatoken" in query
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def drain_delta_pages(
|
|
203
|
+
*,
|
|
204
|
+
initial_cursor_url: str | None,
|
|
205
|
+
fetch_page: Callable[[str | None], Any],
|
|
206
|
+
rows_from_page: Callable[[Any], list[Any]],
|
|
207
|
+
) -> Iterator[DeltaPage]:
|
|
208
|
+
cursor_url = initial_cursor_url
|
|
209
|
+
|
|
210
|
+
while True:
|
|
211
|
+
response = fetch_page(cursor_url)
|
|
212
|
+
rows = rows_from_page(response)
|
|
213
|
+
|
|
214
|
+
delta_link = getattr(response, "odata_delta_link", None)
|
|
215
|
+
if delta_link:
|
|
216
|
+
yield DeltaPage(rows=rows, cursor_url=delta_link)
|
|
217
|
+
return
|
|
218
|
+
|
|
219
|
+
next_link = getattr(response, "odata_next_link", None)
|
|
220
|
+
if not next_link:
|
|
221
|
+
raise RuntimeError(
|
|
222
|
+
"Graph delta response did not include @odata.nextLink or "
|
|
223
|
+
"@odata.deltaLink"
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
yield DeltaPage(rows=rows, cursor_url=next_link)
|
|
227
|
+
cursor_url = next_link
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def graph_object_to_payload(obj: object) -> dict[str, Any]:
|
|
231
|
+
if isinstance(obj, Mapping):
|
|
232
|
+
return dict(obj)
|
|
233
|
+
if not isinstance(obj, Parsable):
|
|
234
|
+
raise TypeError(
|
|
235
|
+
"Expected a Microsoft Graph Kiota Parsable object or mapping; "
|
|
236
|
+
f"got {type(obj).__name__}."
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
writer = JsonSerializationWriter()
|
|
240
|
+
writer.write_object_value(None, obj)
|
|
241
|
+
payload = writer.value if writer.value is not None else writer.writer
|
|
242
|
+
if not isinstance(payload, dict):
|
|
243
|
+
raise RuntimeError(
|
|
244
|
+
"Microsoft Graph Kiota serializer returned a non-object payload."
|
|
245
|
+
)
|
|
246
|
+
raw = dict(payload)
|
|
247
|
+
additional_data = getattr(obj, "additional_data", None)
|
|
248
|
+
if isinstance(additional_data, dict):
|
|
249
|
+
raw["additional_data"] = dict(additional_data)
|
|
250
|
+
return raw
|
shared_plugins/models.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from functools import lru_cache
|
|
4
|
+
from typing import Annotated, Any, Optional, TypeVar, cast
|
|
5
|
+
|
|
6
|
+
from pydantic import (
|
|
7
|
+
AwareDatetime,
|
|
8
|
+
BaseModel,
|
|
9
|
+
ConfigDict,
|
|
10
|
+
Field,
|
|
11
|
+
StringConstraints,
|
|
12
|
+
ValidationError,
|
|
13
|
+
create_model,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
IdStr = Annotated[str, StringConstraints(min_length=1, strict=True)]
|
|
17
|
+
NonNegativeInt = Annotated[int, Field(ge=0, strict=True)]
|
|
18
|
+
|
|
19
|
+
ModelT = TypeVar("ModelT", bound=BaseModel)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def format_validation_error(error: ValidationError) -> str:
|
|
23
|
+
parts: list[str] = []
|
|
24
|
+
for issue in error.errors():
|
|
25
|
+
location = ".".join(str(part) for part in issue["loc"]) or "root"
|
|
26
|
+
parts.append(f"{location}: {issue['msg']}")
|
|
27
|
+
return "; ".join(parts)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class StrictModel(BaseModel):
|
|
31
|
+
"""Shared strict model base: unknown fields are always rejected."""
|
|
32
|
+
|
|
33
|
+
model_config = ConfigDict(extra="forbid", populate_by_name=True)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class IngressModel(StrictModel):
|
|
37
|
+
"""Shared plugin ingress contract base."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class CtxModel(StrictModel):
|
|
41
|
+
"""Shared plugin row metadata contract."""
|
|
42
|
+
|
|
43
|
+
ctx_binding_id: str = Field(min_length=1, alias="_ctx_binding_id")
|
|
44
|
+
ctx_source_updated_at: AwareDatetime | None = Field(
|
|
45
|
+
default=None,
|
|
46
|
+
alias="_ctx_source_updated_at",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def partialize(model_cls: type[ModelT], *, name: str | None = None) -> type[ModelT]:
|
|
51
|
+
"""Return a cached partial model class with all inherited fields optional."""
|
|
52
|
+
|
|
53
|
+
partial_name = name or f"{model_cls.__name__}Partial"
|
|
54
|
+
return cast(type[ModelT], _partialize_cached(model_cls, partial_name))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _to_optional_annotation(annotation: Any) -> Any:
|
|
58
|
+
try:
|
|
59
|
+
return annotation | None
|
|
60
|
+
except TypeError:
|
|
61
|
+
return Optional[annotation] # type: ignore[valid-type]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@lru_cache(maxsize=None)
|
|
65
|
+
def _partialize_cached(
|
|
66
|
+
model_cls: type[BaseModel], partial_name: str
|
|
67
|
+
) -> type[BaseModel]:
|
|
68
|
+
partial_fields: dict[str, tuple[Any, Any]] = {}
|
|
69
|
+
|
|
70
|
+
for field_name, field_info in model_cls.model_fields.items():
|
|
71
|
+
field_dict = field_info.asdict()
|
|
72
|
+
field_attributes = dict(field_dict["attributes"])
|
|
73
|
+
field_attributes.pop("default", None)
|
|
74
|
+
field_attributes.pop("default_factory", None)
|
|
75
|
+
|
|
76
|
+
optional_annotation = _to_optional_annotation(field_dict["annotation"])
|
|
77
|
+
annotated_field = Annotated[
|
|
78
|
+
(
|
|
79
|
+
optional_annotation,
|
|
80
|
+
*field_dict["metadata"],
|
|
81
|
+
Field(**field_attributes),
|
|
82
|
+
)
|
|
83
|
+
]
|
|
84
|
+
partial_fields[field_name] = (annotated_field, None)
|
|
85
|
+
|
|
86
|
+
return create_model(
|
|
87
|
+
partial_name,
|
|
88
|
+
__base__=model_cls,
|
|
89
|
+
__module__=model_cls.__module__,
|
|
90
|
+
**partial_fields,
|
|
91
|
+
)
|