contextbase-shared-plugins 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contextbase_shared_plugins-0.2.3.dist-info/METADATA +22 -0
- contextbase_shared_plugins-0.2.3.dist-info/RECORD +37 -0
- contextbase_shared_plugins-0.2.3.dist-info/WHEEL +4 -0
- shared_plugins/__init__.py +12 -0
- shared_plugins/automation.py +11 -0
- shared_plugins/bindings.py +253 -0
- shared_plugins/control_plane.py +208 -0
- shared_plugins/dlt.py +84 -0
- shared_plugins/env.py +102 -0
- shared_plugins/exceptions.py +10 -0
- shared_plugins/google_client/__init__.py +1 -0
- shared_plugins/google_client/auth.py +82 -0
- shared_plugins/google_client/batch_retry.py +308 -0
- shared_plugins/google_client/http_errors.py +27 -0
- shared_plugins/microsoft_dataverse/__init__.py +27 -0
- shared_plugins/microsoft_dataverse/annotations.py +38 -0
- shared_plugins/microsoft_dataverse/auth.py +26 -0
- shared_plugins/microsoft_dataverse/binding_config.py +35 -0
- shared_plugins/microsoft_dataverse/client.py +456 -0
- shared_plugins/microsoft_dataverse/ctx.py +21 -0
- shared_plugins/microsoft_dataverse/identifiers.py +62 -0
- shared_plugins/microsoft_dataverse/ingress.py +53 -0
- shared_plugins/microsoft_dataverse/metadata.py +106 -0
- shared_plugins/microsoft_dataverse/runtime_schema.py +332 -0
- shared_plugins/microsoft_dataverse/source.py +250 -0
- shared_plugins/microsoft_dataverse/tables.py +34 -0
- shared_plugins/microsoft_dataverse/translators.py +128 -0
- shared_plugins/microsoft_dataverse/types.py +346 -0
- shared_plugins/models.py +91 -0
- shared_plugins/naming.py +83 -0
- shared_plugins/pg_column_comments.py +59 -0
- shared_plugins/pyairbyte.py +399 -0
- shared_plugins/resources.py +179 -0
- shared_plugins/scratch.py +127 -0
- shared_plugins/sqlalchemy_types.py +225 -0
- shared_plugins/sqlite.py +123 -0
- shared_plugins/values.py +117 -0
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import time
|
|
5
|
+
from collections.abc import Callable, Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any, Iterator
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
import tenacity
|
|
11
|
+
from pydantic import ValidationError
|
|
12
|
+
from shared_plugins.exceptions import PluginConfigurationError
|
|
13
|
+
from shared_plugins.models import format_validation_error
|
|
14
|
+
|
|
15
|
+
from .auth import ClientSecretTokenProvider
|
|
16
|
+
from .ingress import (
|
|
17
|
+
DataverseListEnvelopeIngress,
|
|
18
|
+
DataverseListResponseIngress,
|
|
19
|
+
DataverseRecordIngress,
|
|
20
|
+
)
|
|
21
|
+
from .tables import DataverseTableSpec
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
|
|
25
|
+
API_VERSION = "v9.2"
|
|
26
|
+
REQUEST_TIMEOUT_SECONDS = 60.0
|
|
27
|
+
# Dataverse only emits OData annotations (display labels for option-sets and
|
|
28
|
+
# lookups, lookup target-entity names) when the request asks for them via
|
|
29
|
+
# `odata.include-annotations`. The translator already routes these keys to
|
|
30
|
+
# the `_formatted_value` / `_lookup_logical_name` companion columns the
|
|
31
|
+
# schema reserves — they just never arrive without this Prefer.
|
|
32
|
+
INCLUDE_ANNOTATIONS_PREFERENCE = (
|
|
33
|
+
'odata.include-annotations="'
|
|
34
|
+
"OData.Community.Display.V1.FormattedValue,"
|
|
35
|
+
'Microsoft.Dynamics.CRM.lookuplogicalname"'
|
|
36
|
+
)
|
|
37
|
+
TRACK_CHANGES_PREFERENCE = f"odata.track-changes,{INCLUDE_ANNOTATIONS_PREFERENCE}"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# Microsoft's documented Dataverse retry policy: 408 + 429 + every 5xx, plus
|
|
41
|
+
# transport-level errors. Mirrors Polly's `HandleTransientHttpError().OrResult(429)`
|
|
42
|
+
# from the official sample.
|
|
43
|
+
# https://learn.microsoft.com/en-us/power-apps/developer/data-platform/api-limits
|
|
44
|
+
def _is_retryable_status(status_code: int) -> bool:
|
|
45
|
+
return status_code == 408 or status_code == 429 or 500 <= status_code < 600
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
AccessTokenProvider = Callable[[], str]
|
|
49
|
+
SleepFn = Callable[[float], None]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class DataverseRetryPolicy:
|
|
54
|
+
max_attempts: int = 5
|
|
55
|
+
max_backoff_seconds: float = 30.0
|
|
56
|
+
total_budget_seconds: float = 120.0
|
|
57
|
+
|
|
58
|
+
def __post_init__(self) -> None:
|
|
59
|
+
if self.max_attempts < 1:
|
|
60
|
+
raise ValueError("max_attempts must be >= 1")
|
|
61
|
+
if self.max_backoff_seconds < 0:
|
|
62
|
+
raise ValueError("max_backoff_seconds must be >= 0")
|
|
63
|
+
if self.total_budget_seconds < 0:
|
|
64
|
+
raise ValueError("total_budget_seconds must be >= 0")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_DEFAULT_RETRY_POLICY = DataverseRetryPolicy()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class _TransientResponseError(Exception):
|
|
71
|
+
"""Internal sentinel for retryable HTTP status codes.
|
|
72
|
+
|
|
73
|
+
Carries the failing response so the wait callable can read Retry-After
|
|
74
|
+
and so the boundary can convert exhausted retries into the user-facing
|
|
75
|
+
`httpx.HTTPStatusError`. Not exported from this module.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def __init__(
|
|
79
|
+
self,
|
|
80
|
+
response: httpx.Response,
|
|
81
|
+
retry_after_seconds: float | None,
|
|
82
|
+
status_error: httpx.HTTPStatusError,
|
|
83
|
+
) -> None:
|
|
84
|
+
super().__init__(f"transient Dataverse status {response.status_code}")
|
|
85
|
+
self.response = response
|
|
86
|
+
self.retry_after_seconds = retry_after_seconds
|
|
87
|
+
self.status_error = status_error
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class DataverseClient:
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
*,
|
|
94
|
+
org_url: str,
|
|
95
|
+
tenant_id: str | None = None,
|
|
96
|
+
client_id: str | None = None,
|
|
97
|
+
client_secret: str | None = None,
|
|
98
|
+
access_token_provider: AccessTokenProvider | None = None,
|
|
99
|
+
http_client: httpx.Client | None = None,
|
|
100
|
+
retry_policy: DataverseRetryPolicy | None = None,
|
|
101
|
+
sleep_fn: SleepFn = time.sleep,
|
|
102
|
+
) -> None:
|
|
103
|
+
self.org_url = org_url.rstrip("/")
|
|
104
|
+
if not self.org_url.startswith("https://"):
|
|
105
|
+
raise PluginConfigurationError(
|
|
106
|
+
"Dataverse org_url must start with https://."
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
if access_token_provider is None:
|
|
110
|
+
if not tenant_id or not client_id or not client_secret:
|
|
111
|
+
raise PluginConfigurationError(
|
|
112
|
+
"DataverseClient requires tenant_id, client_id, and "
|
|
113
|
+
"client_secret when access_token_provider is not supplied."
|
|
114
|
+
)
|
|
115
|
+
access_token_provider = ClientSecretTokenProvider(
|
|
116
|
+
tenant_id=tenant_id,
|
|
117
|
+
client_id=client_id,
|
|
118
|
+
client_secret=client_secret,
|
|
119
|
+
scope=f"{self.org_url}/.default",
|
|
120
|
+
)
|
|
121
|
+
self._owns_access_token_provider = True
|
|
122
|
+
else:
|
|
123
|
+
self._owns_access_token_provider = False
|
|
124
|
+
|
|
125
|
+
self._access_token_provider = access_token_provider
|
|
126
|
+
self._http_client = http_client or httpx.Client(
|
|
127
|
+
timeout=REQUEST_TIMEOUT_SECONDS,
|
|
128
|
+
)
|
|
129
|
+
self._owns_http_client = http_client is None
|
|
130
|
+
self._retry_policy = retry_policy or _DEFAULT_RETRY_POLICY
|
|
131
|
+
self._sleep_fn = sleep_fn
|
|
132
|
+
self._retrying = _build_retrying(self._retry_policy, self._sleep_fn)
|
|
133
|
+
|
|
134
|
+
def close(self) -> None:
|
|
135
|
+
if self._owns_http_client:
|
|
136
|
+
self._http_client.close()
|
|
137
|
+
if self._owns_access_token_provider:
|
|
138
|
+
close = getattr(self._access_token_provider, "close", None)
|
|
139
|
+
if close is not None:
|
|
140
|
+
close()
|
|
141
|
+
|
|
142
|
+
def __enter__(self) -> DataverseClient:
|
|
143
|
+
return self
|
|
144
|
+
|
|
145
|
+
def __exit__(self, *args: object) -> None:
|
|
146
|
+
self.close()
|
|
147
|
+
|
|
148
|
+
def iter_change_tracking_pages(
|
|
149
|
+
self,
|
|
150
|
+
spec: DataverseTableSpec,
|
|
151
|
+
*,
|
|
152
|
+
delta_link: str | None = None,
|
|
153
|
+
record_model: type[DataverseRecordIngress] | None = None,
|
|
154
|
+
select: tuple[str, ...] | None = None,
|
|
155
|
+
) -> Iterator[DataverseListResponseIngress]:
|
|
156
|
+
yield from self._iter_table_pages(
|
|
157
|
+
spec,
|
|
158
|
+
delta_link=delta_link,
|
|
159
|
+
prefer=(
|
|
160
|
+
INCLUDE_ANNOTATIONS_PREFERENCE
|
|
161
|
+
if delta_link
|
|
162
|
+
else TRACK_CHANGES_PREFERENCE
|
|
163
|
+
),
|
|
164
|
+
record_model=record_model,
|
|
165
|
+
select=select,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
def iter_snapshot_pages(
|
|
169
|
+
self,
|
|
170
|
+
spec: DataverseTableSpec,
|
|
171
|
+
*,
|
|
172
|
+
record_model: type[DataverseRecordIngress] | None = None,
|
|
173
|
+
select: tuple[str, ...] | None = None,
|
|
174
|
+
) -> Iterator[DataverseListResponseIngress]:
|
|
175
|
+
yield from self._iter_table_pages(
|
|
176
|
+
spec,
|
|
177
|
+
delta_link=None,
|
|
178
|
+
prefer=INCLUDE_ANNOTATIONS_PREFERENCE,
|
|
179
|
+
record_model=record_model,
|
|
180
|
+
select=select,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
def get_json(
|
|
184
|
+
self,
|
|
185
|
+
path: str,
|
|
186
|
+
*,
|
|
187
|
+
params: Mapping[str, str] | None = None,
|
|
188
|
+
) -> Mapping[str, Any]:
|
|
189
|
+
payload = self._get_json(
|
|
190
|
+
self._api_url(path),
|
|
191
|
+
params=params,
|
|
192
|
+
prefer=None,
|
|
193
|
+
)
|
|
194
|
+
if not isinstance(payload, Mapping):
|
|
195
|
+
raise RuntimeError("Dataverse API returned a non-object response payload.")
|
|
196
|
+
return payload
|
|
197
|
+
|
|
198
|
+
def _iter_table_pages(
|
|
199
|
+
self,
|
|
200
|
+
spec: DataverseTableSpec,
|
|
201
|
+
*,
|
|
202
|
+
delta_link: str | None,
|
|
203
|
+
prefer: str | None,
|
|
204
|
+
record_model: type[DataverseRecordIngress] | None,
|
|
205
|
+
select: tuple[str, ...] | None,
|
|
206
|
+
) -> Iterator[DataverseListResponseIngress]:
|
|
207
|
+
url = delta_link or self._api_url(spec.entity_set)
|
|
208
|
+
params: dict[str, str] | None = None
|
|
209
|
+
if delta_link is None:
|
|
210
|
+
if select is None:
|
|
211
|
+
raise RuntimeError(
|
|
212
|
+
f"DataverseClient._iter_table_pages requires `select` for "
|
|
213
|
+
f"entity_set={spec.entity_set!r} initial fetch (delta_link=None)."
|
|
214
|
+
)
|
|
215
|
+
params = {"$select": ",".join(select)}
|
|
216
|
+
|
|
217
|
+
while True:
|
|
218
|
+
page = self._get_list(
|
|
219
|
+
url,
|
|
220
|
+
params=params,
|
|
221
|
+
prefer=prefer,
|
|
222
|
+
record_model=record_model,
|
|
223
|
+
)
|
|
224
|
+
yield page
|
|
225
|
+
|
|
226
|
+
if not page.next_link:
|
|
227
|
+
break
|
|
228
|
+
|
|
229
|
+
url = page.next_link
|
|
230
|
+
params = None
|
|
231
|
+
|
|
232
|
+
def _get_list(
|
|
233
|
+
self,
|
|
234
|
+
url: str,
|
|
235
|
+
*,
|
|
236
|
+
params: Mapping[str, str] | None,
|
|
237
|
+
prefer: str | None,
|
|
238
|
+
record_model: type[DataverseRecordIngress] | None,
|
|
239
|
+
) -> DataverseListResponseIngress:
|
|
240
|
+
payload = self._get_json(url, params=params, prefer=prefer)
|
|
241
|
+
|
|
242
|
+
try:
|
|
243
|
+
envelope = DataverseListEnvelopeIngress.model_validate(payload)
|
|
244
|
+
resolved_record_model = record_model or DataverseRecordIngress
|
|
245
|
+
records = [
|
|
246
|
+
resolved_record_model.model_validate(item) for item in envelope.value
|
|
247
|
+
]
|
|
248
|
+
return DataverseListResponseIngress.model_construct(
|
|
249
|
+
odata_context=envelope.odata_context,
|
|
250
|
+
value=records,
|
|
251
|
+
next_link=envelope.next_link,
|
|
252
|
+
delta_link=envelope.delta_link,
|
|
253
|
+
)
|
|
254
|
+
except ValidationError as exc:
|
|
255
|
+
raise RuntimeError(
|
|
256
|
+
"Dataverse API response did not match the expected list shape: "
|
|
257
|
+
f"{format_validation_error(exc)}"
|
|
258
|
+
) from exc
|
|
259
|
+
|
|
260
|
+
def _get_json(
|
|
261
|
+
self,
|
|
262
|
+
url: str,
|
|
263
|
+
*,
|
|
264
|
+
params: Mapping[str, str] | None,
|
|
265
|
+
prefer: str | None,
|
|
266
|
+
) -> Any:
|
|
267
|
+
# `reraise=True` lets the last attempt's exception propagate. For
|
|
268
|
+
# transport errors that exception is already user-facing; for
|
|
269
|
+
# `_TransientResponseError` we convert at this boundary so callers
|
|
270
|
+
# see the same `httpx.HTTPStatusError` they did before tenacity.
|
|
271
|
+
try:
|
|
272
|
+
for attempt in self._retrying:
|
|
273
|
+
with attempt:
|
|
274
|
+
return self._get_json_once(url, params=params, prefer=prefer)
|
|
275
|
+
except _TransientResponseError as exc:
|
|
276
|
+
raise exc.status_error from exc
|
|
277
|
+
|
|
278
|
+
# `Retrying.__iter__` either yields successful results via the
|
|
279
|
+
# generator or raises through `reraise=True`. It does not exit
|
|
280
|
+
# cleanly without a result.
|
|
281
|
+
raise RuntimeError("Dataverse retry loop exited without a result.")
|
|
282
|
+
|
|
283
|
+
def _get_json_once(
|
|
284
|
+
self,
|
|
285
|
+
url: str,
|
|
286
|
+
*,
|
|
287
|
+
params: Mapping[str, str] | None,
|
|
288
|
+
prefer: str | None,
|
|
289
|
+
) -> Any:
|
|
290
|
+
response = self._http_client.get(
|
|
291
|
+
url,
|
|
292
|
+
params=params,
|
|
293
|
+
headers=self._headers(prefer=prefer),
|
|
294
|
+
)
|
|
295
|
+
payload = _response_json(response)
|
|
296
|
+
|
|
297
|
+
if response.is_error:
|
|
298
|
+
status_error = httpx.HTTPStatusError(
|
|
299
|
+
_format_error_message(response, payload),
|
|
300
|
+
request=response.request,
|
|
301
|
+
response=response,
|
|
302
|
+
)
|
|
303
|
+
if _is_retryable_status(response.status_code):
|
|
304
|
+
raise _TransientResponseError(
|
|
305
|
+
response=response,
|
|
306
|
+
retry_after_seconds=_retry_after_seconds(response),
|
|
307
|
+
status_error=status_error,
|
|
308
|
+
)
|
|
309
|
+
raise status_error
|
|
310
|
+
|
|
311
|
+
return payload
|
|
312
|
+
|
|
313
|
+
def _headers(self, *, prefer: str | None) -> dict[str, str]:
|
|
314
|
+
headers = {
|
|
315
|
+
"Authorization": f"Bearer {self._access_token_provider()}",
|
|
316
|
+
"OData-Version": "4.0",
|
|
317
|
+
"OData-MaxVersion": "4.0",
|
|
318
|
+
"Accept": "application/json",
|
|
319
|
+
}
|
|
320
|
+
if prefer:
|
|
321
|
+
headers["Prefer"] = prefer
|
|
322
|
+
return headers
|
|
323
|
+
|
|
324
|
+
def _api_url(self, path: str) -> str:
|
|
325
|
+
return f"{self.org_url}/api/data/{API_VERSION}/{path.lstrip('/')}"
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _build_retrying(
|
|
329
|
+
policy: DataverseRetryPolicy,
|
|
330
|
+
sleep_fn: SleepFn,
|
|
331
|
+
) -> tenacity.Retrying:
|
|
332
|
+
"""Build a per-instance tenacity controller from the retry policy.
|
|
333
|
+
|
|
334
|
+
Tenacity's bundled stops and waits don't cover what we need here:
|
|
335
|
+
|
|
336
|
+
* `stop_after_delay` measures real wall-clock time. Tests inject a fake
|
|
337
|
+
`sleep_fn` that records intended delays without burning real time, so
|
|
338
|
+
the wall clock never advances. We need to budget against the *injected
|
|
339
|
+
sleeps* (`retry_state.idle_for`), not real elapsed time.
|
|
340
|
+
* `wait_exponential` doesn't read `Retry-After` from the failure.
|
|
341
|
+
* Both need to clamp the next sleep so a single huge `Retry-After` can't
|
|
342
|
+
blow past the total budget.
|
|
343
|
+
|
|
344
|
+
So we hand-write small `wait` and `stop` callables that share the policy
|
|
345
|
+
and read the sentinel exception. Everything else (attempt iteration,
|
|
346
|
+
sleep dispatch, reraise) is delegated to tenacity.
|
|
347
|
+
"""
|
|
348
|
+
|
|
349
|
+
def is_retryable(exc: BaseException) -> bool:
|
|
350
|
+
return isinstance(exc, (_TransientResponseError, httpx.RequestError))
|
|
351
|
+
|
|
352
|
+
def compute_wait(retry_state: tenacity.RetryCallState) -> float:
|
|
353
|
+
outcome = retry_state.outcome
|
|
354
|
+
exc = outcome.exception() if outcome is not None else None
|
|
355
|
+
|
|
356
|
+
retry_after: float | None = None
|
|
357
|
+
if isinstance(exc, _TransientResponseError):
|
|
358
|
+
retry_after = exc.retry_after_seconds
|
|
359
|
+
|
|
360
|
+
# Match Microsoft's sample: `Math.Pow(2, count)` where count is the
|
|
361
|
+
# 1-indexed retry number. In tenacity, the just-failed attempt number
|
|
362
|
+
# is the same value when `compute_wait` runs.
|
|
363
|
+
backoff = min(
|
|
364
|
+
policy.max_backoff_seconds,
|
|
365
|
+
float(2**retry_state.attempt_number),
|
|
366
|
+
)
|
|
367
|
+
wait_seconds = max(backoff, retry_after) if retry_after is not None else backoff
|
|
368
|
+
|
|
369
|
+
# Clamp the *next* sleep to whatever budget remains. Mirrors the old
|
|
370
|
+
# `min(wait_seconds, remaining_budget)` line.
|
|
371
|
+
remaining_budget = policy.total_budget_seconds - retry_state.idle_for
|
|
372
|
+
if remaining_budget <= 0:
|
|
373
|
+
return 0.0
|
|
374
|
+
return min(wait_seconds, remaining_budget)
|
|
375
|
+
|
|
376
|
+
def should_stop(retry_state: tenacity.RetryCallState) -> bool:
|
|
377
|
+
# Budget is measured against accumulated injected sleeps so tests
|
|
378
|
+
# that fake `sleep_fn` get the same "abort before next sleep"
|
|
379
|
+
# semantics production gets with real `time.sleep`.
|
|
380
|
+
if retry_state.idle_for >= policy.total_budget_seconds:
|
|
381
|
+
return True
|
|
382
|
+
return retry_state.attempt_number >= policy.max_attempts
|
|
383
|
+
|
|
384
|
+
def log_before_sleep(retry_state: tenacity.RetryCallState) -> None:
|
|
385
|
+
outcome = retry_state.outcome
|
|
386
|
+
exc = outcome.exception() if outcome is not None else None
|
|
387
|
+
status_code: int | None = None
|
|
388
|
+
transport_name: str | None = None
|
|
389
|
+
if isinstance(exc, _TransientResponseError):
|
|
390
|
+
status_code = exc.response.status_code
|
|
391
|
+
elif exc is not None:
|
|
392
|
+
transport_name = type(exc).__name__
|
|
393
|
+
logger.warning(
|
|
394
|
+
"Dataverse request transient failure (attempt %d/%d), "
|
|
395
|
+
"retrying after %.2fs. status=%s transport=%s",
|
|
396
|
+
retry_state.attempt_number,
|
|
397
|
+
policy.max_attempts,
|
|
398
|
+
retry_state.upcoming_sleep,
|
|
399
|
+
status_code,
|
|
400
|
+
transport_name,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
return tenacity.Retrying(
|
|
404
|
+
retry=tenacity.retry_if_exception(is_retryable),
|
|
405
|
+
wait=compute_wait,
|
|
406
|
+
stop=should_stop,
|
|
407
|
+
before_sleep=log_before_sleep,
|
|
408
|
+
sleep=sleep_fn,
|
|
409
|
+
reraise=True,
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _response_json(response: httpx.Response) -> Any:
|
|
414
|
+
if not response.content:
|
|
415
|
+
return None
|
|
416
|
+
try:
|
|
417
|
+
return response.json()
|
|
418
|
+
except ValueError as exc:
|
|
419
|
+
raise RuntimeError("Dataverse API returned a non-JSON response.") from exc
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _format_error_message(response: httpx.Response, payload: Any) -> str:
|
|
423
|
+
message = _dataverse_error_message(payload)
|
|
424
|
+
if message:
|
|
425
|
+
return f"Dataverse request failed with status {response.status_code}: {message}"
|
|
426
|
+
return f"Dataverse request failed with status {response.status_code}."
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _dataverse_error_message(payload: Any) -> str:
|
|
430
|
+
if not isinstance(payload, Mapping):
|
|
431
|
+
return ""
|
|
432
|
+
|
|
433
|
+
error = payload.get("error")
|
|
434
|
+
if not isinstance(error, Mapping):
|
|
435
|
+
return ""
|
|
436
|
+
|
|
437
|
+
code = error.get("code")
|
|
438
|
+
message = error.get("message")
|
|
439
|
+
parts = [part for part in (code, message) if isinstance(part, str) and part]
|
|
440
|
+
return ": ".join(parts)
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _retry_after_seconds(response: httpx.Response) -> float | None:
|
|
444
|
+
"""Parse the Dataverse Retry-After header.
|
|
445
|
+
|
|
446
|
+
Microsoft documents this as an integer number of seconds and the
|
|
447
|
+
official sample uses `int.Parse` on the value:
|
|
448
|
+
https://learn.microsoft.com/en-us/power-apps/developer/data-platform/api-limits
|
|
449
|
+
|
|
450
|
+
Returns None when the header is absent. Raises if the header is present
|
|
451
|
+
but not an integer (matches the Microsoft sample's behavior).
|
|
452
|
+
"""
|
|
453
|
+
header_value = response.headers.get("Retry-After")
|
|
454
|
+
if header_value is None:
|
|
455
|
+
return None
|
|
456
|
+
return float(int(header_value))
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from shared_plugins.models import CtxModel
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DataverseRowBase(CtxModel):
|
|
7
|
+
"""Common ctx fields on every Dataverse row beyond _ctx_*.
|
|
8
|
+
|
|
9
|
+
Every Dataverse row, regardless of which table it comes from, carries:
|
|
10
|
+
- etag: the OData @odata.etag value, used for downstream change detection.
|
|
11
|
+
- is_deleted: set by tombstone detection on delta drains.
|
|
12
|
+
- delete_reason: the OData reason field on tombstones.
|
|
13
|
+
|
|
14
|
+
Per-table runtime models extend this with one field per Dataverse attribute
|
|
15
|
+
that survived filtering, plus the per-table primary-key field. The
|
|
16
|
+
_ctx_binding_id and _ctx_source_updated_at fields come from CtxModel.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
etag: str | None = None
|
|
20
|
+
is_deleted: bool = False
|
|
21
|
+
delete_reason: str | None = None
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Postgres identifier construction and validation for Dataverse columns.
|
|
2
|
+
|
|
3
|
+
Postgres caps identifiers at NAMEDATALEN-1 = 63 bytes. With the verbose
|
|
4
|
+
annotation suffixes (_formatted_value, _lookup_logical_name), Dataverse
|
|
5
|
+
attribute logical names can produce columns that overflow. Silent
|
|
6
|
+
truncation would risk collisions and lost data, so we validate at
|
|
7
|
+
warmup and raise loudly on overflow. Resolution is human-driven (skip
|
|
8
|
+
the underlying attribute via plugin override or shorten the suffix).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import keyword
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
from shared_plugins.exceptions import PluginConfigurationError
|
|
17
|
+
|
|
18
|
+
PG_IDENTIFIER_BYTE_LIMIT = 63
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def validate_identifier(name: str, *, context: str) -> None:
|
|
22
|
+
"""Raise PluginConfigurationError if name exceeds postgres's 63-byte limit.
|
|
23
|
+
|
|
24
|
+
`context` is included in the error message so the operator can locate the
|
|
25
|
+
offending attribute (e.g. "msdyn_projects.foo_lookup_logical_name").
|
|
26
|
+
"""
|
|
27
|
+
encoded = name.encode("utf-8")
|
|
28
|
+
if len(encoded) > PG_IDENTIFIER_BYTE_LIMIT:
|
|
29
|
+
raise PluginConfigurationError(
|
|
30
|
+
f"Postgres identifier {name!r} ({len(encoded)} bytes) exceeds postgres "
|
|
31
|
+
f"NAMEDATALEN-1 limit of {PG_IDENTIFIER_BYTE_LIMIT} bytes. "
|
|
32
|
+
f"Context: {context}. "
|
|
33
|
+
"Resolution: skip the underlying attribute via the plugin's "
|
|
34
|
+
"skipped_logical_names override, or shorten the annotation suffix "
|
|
35
|
+
"in shared_plugins.microsoft_dataverse.annotations."
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def annotation_column_name(base_column: str, suffix: str, *, context: str) -> str:
|
|
40
|
+
"""Compose `f"{base_column}{suffix}"` and validate the result fits in 63 bytes."""
|
|
41
|
+
name = f"{base_column}{suffix}"
|
|
42
|
+
validate_identifier(name, context=context)
|
|
43
|
+
return name
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def safe_identifier(value: str) -> str:
|
|
47
|
+
candidate = re.sub(r"\W+", "_", value).strip("_").lower()
|
|
48
|
+
if not candidate:
|
|
49
|
+
candidate = "value"
|
|
50
|
+
if candidate[0].isdigit():
|
|
51
|
+
candidate = f"field_{candidate}"
|
|
52
|
+
if keyword.iskeyword(candidate):
|
|
53
|
+
candidate = f"{candidate}_field"
|
|
54
|
+
return candidate
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def pascal_case(value: str) -> str:
|
|
58
|
+
return "".join(part.capitalize() for part in re.split(r"[^a-zA-Z0-9]+", value))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def escape_odata_string(value: str) -> str:
|
|
62
|
+
return value.replace("'", "''")
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import ConfigDict, Field
|
|
6
|
+
from shared_plugins.models import IdStr, IngressModel
|
|
7
|
+
|
|
8
|
+
from .tables import DataverseTableSpec
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DataverseRecordIngress(IngressModel):
|
|
12
|
+
# Real OData rows carry per-table typed columns (msdyn_*, statecode, etc.)
|
|
13
|
+
# that this envelope-only model does not declare. `extra="allow"` keeps
|
|
14
|
+
# those fields on the model and round-trips them through model_dump(...)
|
|
15
|
+
# so the translator's raw_payload dict still contains every column. Strict
|
|
16
|
+
# forbid would reject every real row; ignore would silently drop typed
|
|
17
|
+
# columns before model_dump emits them. Per-table validation of typed
|
|
18
|
+
# columns happens at the row-layer CtxModel built in runtime_schema.py.
|
|
19
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
20
|
+
|
|
21
|
+
odata_etag: str | None = Field(default=None, alias="@odata.etag")
|
|
22
|
+
odata_context: str | None = Field(default=None, alias="@odata.context")
|
|
23
|
+
odata_id: str | None = Field(default=None, alias="@odata.id")
|
|
24
|
+
odata_type: str | None = Field(default=None, alias="@odata.type")
|
|
25
|
+
id: IdStr | None = None
|
|
26
|
+
reason: str | None = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DataverseListEnvelopeIngress(IngressModel):
|
|
30
|
+
odata_context: str | None = Field(default=None, alias="@odata.context")
|
|
31
|
+
value: list[dict[str, Any]] = Field(default_factory=list)
|
|
32
|
+
next_link: str | None = Field(default=None, alias="@odata.nextLink")
|
|
33
|
+
delta_link: str | None = Field(default=None, alias="@odata.deltaLink")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class DataverseListResponseIngress(IngressModel):
|
|
37
|
+
odata_context: str | None = Field(default=None, alias="@odata.context")
|
|
38
|
+
value: list[DataverseRecordIngress] = Field(default_factory=list)
|
|
39
|
+
next_link: str | None = Field(default=None, alias="@odata.nextLink")
|
|
40
|
+
delta_link: str | None = Field(default=None, alias="@odata.deltaLink")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def is_deleted_record(
|
|
44
|
+
record: DataverseRecordIngress,
|
|
45
|
+
spec: DataverseTableSpec,
|
|
46
|
+
*,
|
|
47
|
+
payload: dict[str, Any],
|
|
48
|
+
) -> bool:
|
|
49
|
+
# Dataverse delta tombstones can arrive as a partial record carrying only
|
|
50
|
+
# @odata.id, with the selected primary-key column absent from the payload.
|
|
51
|
+
return bool(record.reason) or (
|
|
52
|
+
isinstance(record.id, str) and spec.primary_key not in payload
|
|
53
|
+
)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
6
|
+
from shared_plugins.models import IngressModel
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class LocalizedLabelIngress(IngressModel):
|
|
10
|
+
# Verified 2026-05-03 against customer tenant with `Prefer: odata.include-annotations="*"`:
|
|
11
|
+
# no @odata.* decoration on this complex type, so extra="forbid" via IngressModel is safe.
|
|
12
|
+
label: str | None = Field(default=None, alias="Label")
|
|
13
|
+
language_code: int | None = Field(default=None, alias="LanguageCode")
|
|
14
|
+
is_managed: bool | None = Field(default=None, alias="IsManaged")
|
|
15
|
+
metadata_id: str | None = Field(default=None, alias="MetadataId")
|
|
16
|
+
has_changed: bool | None = Field(default=None, alias="HasChanged")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class LabelIngress(IngressModel):
|
|
20
|
+
# Verified 2026-05-03 against customer tenant with `Prefer: odata.include-annotations="*"`:
|
|
21
|
+
# no @odata.* decoration on this complex type, so extra="forbid" via IngressModel is safe.
|
|
22
|
+
localized_labels: list[LocalizedLabelIngress] | None = Field(
|
|
23
|
+
default=None,
|
|
24
|
+
alias="LocalizedLabels",
|
|
25
|
+
)
|
|
26
|
+
user_localized_label: LocalizedLabelIngress | None = Field(
|
|
27
|
+
default=None,
|
|
28
|
+
alias="UserLocalizedLabel",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class AttributeTypeNameIngress(IngressModel):
|
|
33
|
+
# Verified 2026-05-03 against customer tenant with `Prefer: odata.include-annotations="*"`:
|
|
34
|
+
# no @odata.* decoration on this complex type, so extra="forbid" via IngressModel is safe.
|
|
35
|
+
value: str | None = Field(default=None, alias="Value")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class AttributeMetadataIngress(BaseModel):
|
|
39
|
+
"""Per-attribute metadata returned by EntityDefinitions/<n>/Attributes.
|
|
40
|
+
|
|
41
|
+
Uses extra="ignore" (rather than the codebase default extra="forbid")
|
|
42
|
+
because we $select only a subset of fields and Microsoft includes
|
|
43
|
+
@odata.* meta + a wide range of type-specific fields on every attribute.
|
|
44
|
+
The strict-rejection model would fail on every record. The trade-off is
|
|
45
|
+
documented and accepted at this single boundary.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
model_config = ConfigDict(extra="ignore", populate_by_name=True)
|
|
49
|
+
|
|
50
|
+
logical_name: str = Field(alias="LogicalName")
|
|
51
|
+
attribute_type: str | None = Field(default=None, alias="AttributeType")
|
|
52
|
+
attribute_type_name: AttributeTypeNameIngress | None = Field(
|
|
53
|
+
default=None,
|
|
54
|
+
alias="AttributeTypeName",
|
|
55
|
+
)
|
|
56
|
+
attribute_of: str | None = Field(default=None, alias="AttributeOf")
|
|
57
|
+
is_valid_for_read: bool | None = Field(default=None, alias="IsValidForRead")
|
|
58
|
+
is_custom_attribute: bool = Field(default=False, alias="IsCustomAttribute")
|
|
59
|
+
is_primary_id: bool = Field(default=False, alias="IsPrimaryId")
|
|
60
|
+
description: LabelIngress | None = Field(default=None, alias="Description")
|
|
61
|
+
display_name: LabelIngress | None = Field(default=None, alias="DisplayName")
|
|
62
|
+
precision: int | None = Field(default=None, alias="Precision")
|
|
63
|
+
targets: list[str] | None = Field(default=None, alias="Targets")
|
|
64
|
+
format: str | None = Field(default=None, alias="Format")
|
|
65
|
+
date_time_behavior: dict[str, Any] | None = Field(
|
|
66
|
+
default=None,
|
|
67
|
+
alias="DateTimeBehavior",
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def type_name_value(self) -> str | None:
|
|
72
|
+
return (
|
|
73
|
+
None if self.attribute_type_name is None else self.attribute_type_name.value
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class EntityMetadataIngress(IngressModel):
|
|
78
|
+
metadata_id: str | None = Field(default=None, alias="MetadataId")
|
|
79
|
+
logical_name: str = Field(alias="LogicalName")
|
|
80
|
+
entity_set_name: str = Field(alias="EntitySetName")
|
|
81
|
+
primary_id_attribute: str = Field(alias="PrimaryIdAttribute")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class MetadataListResponseIngress(IngressModel):
|
|
85
|
+
odata_context: str | None = Field(default=None, alias="@odata.context")
|
|
86
|
+
value: list[dict[str, Any]] = Field(default_factory=list)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def extract_label(label: LabelIngress | None) -> str | None:
|
|
90
|
+
"""Pull the most useful localized label, or None.
|
|
91
|
+
|
|
92
|
+
Tries UserLocalizedLabel first (the calling principal's preferred locale).
|
|
93
|
+
Falls back to the first non-empty LocalizedLabels entry for tenants where
|
|
94
|
+
service-principal access doesn't populate UserLocalizedLabel. Treats empty
|
|
95
|
+
and whitespace-only strings as absent.
|
|
96
|
+
"""
|
|
97
|
+
if label is None:
|
|
98
|
+
return None
|
|
99
|
+
if label.user_localized_label is not None:
|
|
100
|
+
text = label.user_localized_label.label
|
|
101
|
+
if text and text.strip():
|
|
102
|
+
return text
|
|
103
|
+
for localized in label.localized_labels or ():
|
|
104
|
+
if localized.label and localized.label.strip():
|
|
105
|
+
return localized.label
|
|
106
|
+
return None
|