contextbase-shared-plugins 0.0.0a1__py3-none-any.whl

shared_plugins/microsoft_dataverse/client.py

@@ -0,0 +1,468 @@
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from typing import Any, Iterator
+
+import httpx
+import tenacity
+from pydantic import ValidationError
+from shared_plugins.exceptions import PluginConfigurationError
+from shared_plugins.models import format_validation_error
+
+from .auth import ClientSecretTokenProvider
+from .ingress import (
+    DataverseListEnvelopeIngress,
+    DataverseListResponseIngress,
+    DataverseRecordIngress,
+)
+from .tables import DataverseTableSpec
+
+logger = logging.getLogger(__name__)
+
+API_VERSION = "v9.2"
+REQUEST_TIMEOUT_SECONDS = 60.0
+# Dataverse only emits OData annotations (display labels for option-sets and
+# lookups, lookup target-entity names) when the request asks for them via
+# `odata.include-annotations`. The translator already routes these keys to
+# the `_formatted_value` / `_lookup_logical_name` companion columns the
+# schema reserves — they just never arrive without this Prefer.
+INCLUDE_ANNOTATIONS_PREFERENCE = (
+    'odata.include-annotations="'
+    "OData.Community.Display.V1.FormattedValue,"
+    'Microsoft.Dynamics.CRM.lookuplogicalname"'
+)
+# Dataverse returns up to 5000 rows per page by default. Wide entities with
+# large text columns (e.g. email HTML bodies) can exceed REQUEST_TIMEOUT_SECONDS
+# on a full 5000-row page. Bounding the page size keeps each request small; dlt
+# follows the @odata.nextLink chain, so this only changes the number of
+# round-trips, not the rows drained.
+PAGE_SIZE = 500
+_PAGE_SIZE_PREFERENCE = f"odata.maxpagesize={PAGE_SIZE}"
+# Every list request bounds the page size and asks for display-label annotations.
+LIST_PREFERENCE = f"{_PAGE_SIZE_PREFERENCE},{INCLUDE_ANNOTATIONS_PREFERENCE}"
+# The initial change-tracking enumeration also opts into track-changes. Order
+# matters: Dataverse rejects the Prefer header ("Invalid prefer header",
+# 0x80060888) when odata.maxpagesize sits directly between odata.track-changes
+# and the quoted odata.include-annotations value (verified empirically against
+# the tenant), so maxpagesize must lead.
+TRACK_CHANGES_PREFERENCE = (
+    f"{_PAGE_SIZE_PREFERENCE},odata.track-changes,{INCLUDE_ANNOTATIONS_PREFERENCE}"
+)
+
+
+# Microsoft's documented Dataverse retry policy: 408 + 429 + every 5xx, plus
+# transport-level errors. Mirrors Polly's `HandleTransientHttpError().OrResult(429)`
+# from the official sample.
+# https://learn.microsoft.com/en-us/power-apps/developer/data-platform/api-limits
+def _is_retryable_status(status_code: int) -> bool:
+    return status_code == 408 or status_code == 429 or 500 <= status_code < 600
+
+
+AccessTokenProvider = Callable[[], str]
+SleepFn = Callable[[float], None]
+
+
+@dataclass(frozen=True)
+class DataverseRetryPolicy:
+    max_attempts: int = 5
+    max_backoff_seconds: float = 30.0
+    total_budget_seconds: float = 120.0
+
+    def __post_init__(self) -> None:
+        if self.max_attempts < 1:
+            raise ValueError("max_attempts must be >= 1")
+        if self.max_backoff_seconds < 0:
+            raise ValueError("max_backoff_seconds must be >= 0")
+        if self.total_budget_seconds < 0:
+            raise ValueError("total_budget_seconds must be >= 0")
+
+
+_DEFAULT_RETRY_POLICY = DataverseRetryPolicy()
+
+
+class _TransientResponseError(Exception):
+    """Internal sentinel for retryable HTTP status codes.
+
+    Carries the failing response so the wait callable can read Retry-After
+    and so the boundary can convert exhausted retries into the user-facing
+    `httpx.HTTPStatusError`. Not exported from this module.
+    """
+
+    def __init__(
+        self,
+        response: httpx.Response,
+        retry_after_seconds: float | None,
+        status_error: httpx.HTTPStatusError,
+    ) -> None:
+        super().__init__(f"transient Dataverse status {response.status_code}")
+        self.response = response
+        self.retry_after_seconds = retry_after_seconds
+        self.status_error = status_error
+
+
+class DataverseClient:
+    def __init__(
+        self,
+        *,
+        org_url: str,
+        tenant_id: str | None = None,
+        client_id: str | None = None,
+        client_secret: str | None = None,
+        access_token_provider: AccessTokenProvider | None = None,
+        http_client: httpx.Client | None = None,
+        retry_policy: DataverseRetryPolicy | None = None,
+        sleep_fn: SleepFn = time.sleep,
+    ) -> None:
+        self.org_url = org_url.rstrip("/")
+        if not self.org_url.startswith("https://"):
+            raise PluginConfigurationError(
+                "Dataverse org_url must start with https://."
+            )
+
+        if access_token_provider is None:
+            if not tenant_id or not client_id or not client_secret:
+                raise PluginConfigurationError(
+                    "DataverseClient requires tenant_id, client_id, and "
+                    "client_secret when access_token_provider is not supplied."
+                )
+            access_token_provider = ClientSecretTokenProvider(
+                tenant_id=tenant_id,
+                client_id=client_id,
+                client_secret=client_secret,
+                scope=f"{self.org_url}/.default",
+            )
+            self._owns_access_token_provider = True
+        else:
+            self._owns_access_token_provider = False
+
+        self._access_token_provider = access_token_provider
+        self._http_client = http_client or httpx.Client(
+            timeout=REQUEST_TIMEOUT_SECONDS,
+        )
+        self._owns_http_client = http_client is None
+        self._retry_policy = retry_policy or _DEFAULT_RETRY_POLICY
+        self._sleep_fn = sleep_fn
+        self._retrying = _build_retrying(self._retry_policy, self._sleep_fn)
+
+    def close(self) -> None:
+        if self._owns_http_client:
+            self._http_client.close()
+        if self._owns_access_token_provider:
+            close = getattr(self._access_token_provider, "close", None)
+            if close is not None:
+                close()
+
+    def __enter__(self) -> DataverseClient:
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()
+
+    def iter_change_tracking_pages(
+        self,
+        spec: DataverseTableSpec,
+        *,
+        delta_link: str | None = None,
+        record_model: type[DataverseRecordIngress] | None = None,
+        select: tuple[str, ...] | None = None,
+    ) -> Iterator[DataverseListResponseIngress]:
+        yield from self._iter_table_pages(
+            spec,
+            delta_link=delta_link,
+            prefer=(LIST_PREFERENCE if delta_link else TRACK_CHANGES_PREFERENCE),
+            record_model=record_model,
+            select=select,
+        )
+
+    def iter_snapshot_pages(
+        self,
+        spec: DataverseTableSpec,
+        *,
+        record_model: type[DataverseRecordIngress] | None = None,
+        select: tuple[str, ...] | None = None,
+    ) -> Iterator[DataverseListResponseIngress]:
+        yield from self._iter_table_pages(
+            spec,
+            delta_link=None,
+            prefer=LIST_PREFERENCE,
+            record_model=record_model,
+            select=select,
+        )
+
+    def get_json(
+        self,
+        path: str,
+        *,
+        params: Mapping[str, str] | None = None,
+    ) -> Mapping[str, Any]:
+        payload = self._get_json(
+            self._api_url(path),
+            params=params,
+            prefer=None,
+        )
+        if not isinstance(payload, Mapping):
+            raise RuntimeError("Dataverse API returned a non-object response payload.")
+        return payload
+
+    def _iter_table_pages(
+        self,
+        spec: DataverseTableSpec,
+        *,
+        delta_link: str | None,
+        prefer: str | None,
+        record_model: type[DataverseRecordIngress] | None,
+        select: tuple[str, ...] | None,
+    ) -> Iterator[DataverseListResponseIngress]:
+        url = delta_link or self._api_url(spec.entity_set)
+        params: dict[str, str] | None = None
+        if delta_link is None:
+            if select is None:
+                raise RuntimeError(
+                    f"DataverseClient._iter_table_pages requires `select` for "
+                    f"entity_set={spec.entity_set!r} initial fetch (delta_link=None)."
+                )
+            params = {"$select": ",".join(select)}
+
+        while True:
+            page = self._get_list(
+                url,
+                params=params,
+                prefer=prefer,
+                record_model=record_model,
+            )
+            yield page
+
+            if not page.next_link:
+                break
+
+            url = page.next_link
+            params = None
+
+    def _get_list(
+        self,
+        url: str,
+        *,
+        params: Mapping[str, str] | None,
+        prefer: str | None,
+        record_model: type[DataverseRecordIngress] | None,
+    ) -> DataverseListResponseIngress:
+        payload = self._get_json(url, params=params, prefer=prefer)
+
+        try:
+            envelope = DataverseListEnvelopeIngress.model_validate(payload)
+            resolved_record_model = record_model or DataverseRecordIngress
+            records = [
+                resolved_record_model.model_validate(item) for item in envelope.value
+            ]
+            return DataverseListResponseIngress.model_construct(
+                odata_context=envelope.odata_context,
+                value=records,
+                next_link=envelope.next_link,
+                delta_link=envelope.delta_link,
+            )
+        except ValidationError as exc:
+            raise RuntimeError(
+                "Dataverse API response did not match the expected list shape: "
+                f"{format_validation_error(exc)}"
+            ) from exc
+
+    def _get_json(
+        self,
+        url: str,
+        *,
+        params: Mapping[str, str] | None,
+        prefer: str | None,
+    ) -> Any:
+        # `reraise=True` lets the last attempt's exception propagate. For
+        # transport errors that exception is already user-facing; for
+        # `_TransientResponseError` we convert at this boundary so callers
+        # see the same `httpx.HTTPStatusError` they did before tenacity.
+        try:
+            for attempt in self._retrying:
+                with attempt:
+                    return self._get_json_once(url, params=params, prefer=prefer)
+        except _TransientResponseError as exc:
+            raise exc.status_error from exc
+
+        # `Retrying.__iter__` either yields successful results via the
+        # generator or raises through `reraise=True`. It does not exit
+        # cleanly without a result.
+        raise RuntimeError("Dataverse retry loop exited without a result.")
+
+    def _get_json_once(
+        self,
+        url: str,
+        *,
+        params: Mapping[str, str] | None,
+        prefer: str | None,
+    ) -> Any:
+        response = self._http_client.get(
+            url,
+            params=params,
+            headers=self._headers(prefer=prefer),
+        )
+        payload = _response_json(response)
+
+        if response.is_error:
+            status_error = httpx.HTTPStatusError(
+                _format_error_message(response, payload),
+                request=response.request,
+                response=response,
+            )
+            if _is_retryable_status(response.status_code):
+                raise _TransientResponseError(
+                    response=response,
+                    retry_after_seconds=_retry_after_seconds(response),
+                    status_error=status_error,
+                )
+            raise status_error
+
+        return payload
+
+    def _headers(self, *, prefer: str | None) -> dict[str, str]:
+        headers = {
+            "Authorization": f"Bearer {self._access_token_provider()}",
+            "OData-Version": "4.0",
+            "OData-MaxVersion": "4.0",
+            "Accept": "application/json",
+        }
+        if prefer:
+            headers["Prefer"] = prefer
+        return headers
+
+    def _api_url(self, path: str) -> str:
+        return f"{self.org_url}/api/data/{API_VERSION}/{path.lstrip('/')}"
+
+
+def _build_retrying(
+    policy: DataverseRetryPolicy,
+    sleep_fn: SleepFn,
+) -> tenacity.Retrying:
+    """Build a per-instance tenacity controller from the retry policy.
+
+    Tenacity's bundled stops and waits don't cover what we need here:
+
+    * `stop_after_delay` measures real wall-clock time. Tests inject a fake
+      `sleep_fn` that records intended delays without burning real time, so
+      the wall clock never advances. We need to budget against the *injected
+      sleeps* (`retry_state.idle_for`), not real elapsed time.
+    * `wait_exponential` doesn't read `Retry-After` from the failure.
+    * Both need to clamp the next sleep so a single huge `Retry-After` can't
+      blow past the total budget.
+
+    So we hand-write small `wait` and `stop` callables that share the policy
+    and read the sentinel exception. Everything else (attempt iteration,
+    sleep dispatch, reraise) is delegated to tenacity.
+    """
+
+    def is_retryable(exc: BaseException) -> bool:
+        return isinstance(exc, (_TransientResponseError, httpx.RequestError))
+
+    def compute_wait(retry_state: tenacity.RetryCallState) -> float:
+        outcome = retry_state.outcome
+        exc = outcome.exception() if outcome is not None else None
+
+        retry_after: float | None = None
+        if isinstance(exc, _TransientResponseError):
+            retry_after = exc.retry_after_seconds
+
+        # Match Microsoft's sample: `Math.Pow(2, count)` where count is the
+        # 1-indexed retry number. In tenacity, the just-failed attempt number
+        # is the same value when `compute_wait` runs.
+        backoff = min(
+            policy.max_backoff_seconds,
+            float(2**retry_state.attempt_number),
+        )
+        wait_seconds = max(backoff, retry_after) if retry_after is not None else backoff
+
+        # Clamp the *next* sleep to whatever budget remains. Mirrors the old
+        # `min(wait_seconds, remaining_budget)` line.
+        remaining_budget = policy.total_budget_seconds - retry_state.idle_for
+        if remaining_budget <= 0:
+            return 0.0
+        return min(wait_seconds, remaining_budget)
+
+    def should_stop(retry_state: tenacity.RetryCallState) -> bool:
+        # Budget is measured against accumulated injected sleeps so tests
+        # that fake `sleep_fn` get the same "abort before next sleep"
+        # semantics production gets with real `time.sleep`.
+        if retry_state.idle_for >= policy.total_budget_seconds:
+            return True
+        return retry_state.attempt_number >= policy.max_attempts
+
+    def log_before_sleep(retry_state: tenacity.RetryCallState) -> None:
+        outcome = retry_state.outcome
+        exc = outcome.exception() if outcome is not None else None
+        status_code: int | None = None
+        transport_name: str | None = None
+        if isinstance(exc, _TransientResponseError):
+            status_code = exc.response.status_code
+        elif exc is not None:
+            transport_name = type(exc).__name__
+        logger.warning(
+            "Dataverse request transient failure (attempt %d/%d), "
+            "retrying after %.2fs. status=%s transport=%s",
+            retry_state.attempt_number,
+            policy.max_attempts,
+            retry_state.upcoming_sleep,
+            status_code,
+            transport_name,
+        )
+
+    return tenacity.Retrying(
+        retry=tenacity.retry_if_exception(is_retryable),
+        wait=compute_wait,
+        stop=should_stop,
+        before_sleep=log_before_sleep,
+        sleep=sleep_fn,
+        reraise=True,
+    )
+
+
+def _response_json(response: httpx.Response) -> Any:
+    if not response.content:
+        return None
+    try:
+        return response.json()
+    except ValueError as exc:
+        raise RuntimeError("Dataverse API returned a non-JSON response.") from exc
+
+
+def _format_error_message(response: httpx.Response, payload: Any) -> str:
+    message = _dataverse_error_message(payload)
+    if message:
+        return f"Dataverse request failed with status {response.status_code}: {message}"
+    return f"Dataverse request failed with status {response.status_code}."
+
+
+def _dataverse_error_message(payload: Any) -> str:
+    if not isinstance(payload, Mapping):
+        return ""
+
+    error = payload.get("error")
+    if not isinstance(error, Mapping):
+        return ""
+
+    code = error.get("code")
+    message = error.get("message")
+    parts = [part for part in (code, message) if isinstance(part, str) and part]
+    return ": ".join(parts)
+
+
+def _retry_after_seconds(response: httpx.Response) -> float | None:
+    """Parse the Dataverse Retry-After header.
+
+    Microsoft documents this as an integer number of seconds and the
+    official sample uses `int.Parse` on the value:
+    https://learn.microsoft.com/en-us/power-apps/developer/data-platform/api-limits
+
+    Returns None when the header is absent. Raises if the header is present
+    but not an integer (matches the Microsoft sample's behavior).
+    """
+    header_value = response.headers.get("Retry-After")
+    if header_value is None:
+        return None
+    return float(int(header_value))

shared_plugins/microsoft_dataverse/ctx.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from shared_plugins.models import CtxModel
+
+
+class DataverseRowBase(CtxModel):
+    """Common ctx fields on every Dataverse row beyond _ctx_*.
+
+    Every Dataverse row, regardless of which table it comes from, carries:
+    - etag: the OData @odata.etag value, used for downstream change detection.
+    - is_deleted: set by tombstone detection on delta drains.
+    - delete_reason: the OData reason field on tombstones.
+
+    Per-table runtime models extend this with one field per Dataverse attribute
+    that survived filtering, plus the per-table primary-key field. The
+    _ctx_binding_id and _ctx_source_updated_at fields come from CtxModel.
+    """
+
+    etag: str | None = None
+    is_deleted: bool = False
+    delete_reason: str | None = None

shared_plugins/microsoft_dataverse/identifiers.py

@@ -0,0 +1,62 @@
+"""Postgres identifier construction and validation for Dataverse columns.
+
+Postgres caps identifiers at NAMEDATALEN-1 = 63 bytes. With the verbose
+annotation suffixes (_formatted_value, _lookup_logical_name), Dataverse
+attribute logical names can produce columns that overflow. Silent
+truncation would risk collisions and lost data, so we validate at
+warmup and raise loudly on overflow. Resolution is human-driven (skip
+the underlying attribute via plugin override or shorten the suffix).
+"""
+
+from __future__ import annotations
+
+import keyword
+import re
+
+from shared_plugins.exceptions import PluginConfigurationError
+
+PG_IDENTIFIER_BYTE_LIMIT = 63
+
+
+def validate_identifier(name: str, *, context: str) -> None:
+    """Raise PluginConfigurationError if name exceeds postgres's 63-byte limit.
+
+    `context` is included in the error message so the operator can locate the
+    offending attribute (e.g. "msdyn_projects.foo_lookup_logical_name").
+    """
+    encoded = name.encode("utf-8")
+    if len(encoded) > PG_IDENTIFIER_BYTE_LIMIT:
+        raise PluginConfigurationError(
+            f"Postgres identifier {name!r} ({len(encoded)} bytes) exceeds postgres "
+            f"NAMEDATALEN-1 limit of {PG_IDENTIFIER_BYTE_LIMIT} bytes. "
+            f"Context: {context}. "
+            "Resolution: skip the underlying attribute via the plugin's "
+            "skipped_logical_names override, or shorten the annotation suffix "
+            "in shared_plugins.microsoft_dataverse.annotations."
+        )
+
+
+def annotation_column_name(base_column: str, suffix: str, *, context: str) -> str:
+    """Compose `f"{base_column}{suffix}"` and validate the result fits in 63 bytes."""
+    name = f"{base_column}{suffix}"
+    validate_identifier(name, context=context)
+    return name
+
+
+def safe_identifier(value: str) -> str:
+    candidate = re.sub(r"\W+", "_", value).strip("_").lower()
+    if not candidate:
+        candidate = "value"
+    if candidate[0].isdigit():
+        candidate = f"field_{candidate}"
+    if keyword.iskeyword(candidate):
+        candidate = f"{candidate}_field"
+    return candidate
+
+
+def pascal_case(value: str) -> str:
+    return "".join(part.capitalize() for part in re.split(r"[^a-zA-Z0-9]+", value))
+
+
+def escape_odata_string(value: str) -> str:
+    return value.replace("'", "''")

shared_plugins/microsoft_dataverse/ingress.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import ConfigDict, Field
+from shared_plugins.models import IdStr, IngressModel
+
+from .tables import DataverseTableSpec
+
+
+class DataverseRecordIngress(IngressModel):
+    # Real OData rows carry per-table typed columns (msdyn_*, statecode, etc.)
+    # that this envelope-only model does not declare. `extra="allow"` keeps
+    # those fields on the model and round-trips them through model_dump(...)
+    # so the translator's raw_payload dict still contains every column. Strict
+    # forbid would reject every real row; ignore would silently drop typed
+    # columns before model_dump emits them. Per-table validation of typed
+    # columns happens at the row-layer CtxModel built in runtime_schema.py.
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+    odata_etag: str | None = Field(default=None, alias="@odata.etag")
+    odata_context: str | None = Field(default=None, alias="@odata.context")
+    odata_id: str | None = Field(default=None, alias="@odata.id")
+    odata_type: str | None = Field(default=None, alias="@odata.type")
+    id: IdStr | None = None
+    reason: str | None = None
+
+
+class DataverseListEnvelopeIngress(IngressModel):
+    odata_context: str | None = Field(default=None, alias="@odata.context")
+    value: list[dict[str, Any]] = Field(default_factory=list)
+    next_link: str | None = Field(default=None, alias="@odata.nextLink")
+    delta_link: str | None = Field(default=None, alias="@odata.deltaLink")
+
+
+class DataverseListResponseIngress(IngressModel):
+    odata_context: str | None = Field(default=None, alias="@odata.context")
+    value: list[DataverseRecordIngress] = Field(default_factory=list)
+    next_link: str | None = Field(default=None, alias="@odata.nextLink")
+    delta_link: str | None = Field(default=None, alias="@odata.deltaLink")
+
+
+def is_deleted_record(
+    record: DataverseRecordIngress,
+    spec: DataverseTableSpec,
+    *,
+    payload: dict[str, Any],
+) -> bool:
+    # Dataverse delta tombstones can arrive as a partial record carrying only
+    # @odata.id, with the selected primary-key column absent from the payload.
+    return bool(record.reason) or (
+        isinstance(record.id, str) and spec.primary_key not in payload
+    )