contextbase-shared-plugins 0.2.3__py3-none-any.whl

shared_plugins/microsoft_dataverse/runtime_schema.py

@@ -0,0 +1,332 @@
+"""Per-table schema construction at warmup.
+
+Fetches Dataverse entity + attribute metadata, builds RuntimeFieldSpecs,
+validates 63-byte postgres identifier limits, and assembles per-table
+runtime CtxModel subclasses + dlt column hints. The plugin's source factory
+calls `build_runtime_table_schemas(...)` once per asset run.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from decimal import Decimal
+from typing import Annotated, Any
+
+from pydantic import BeforeValidator, Field, create_model
+from shared_plugins.exceptions import PluginConfigurationError
+from shared_plugins.models import IdStr
+
+from .ctx import DataverseRowBase
+from .identifiers import (
+    escape_odata_string,
+    pascal_case,
+    safe_identifier,
+    validate_identifier,
+)
+from .metadata import (
+    AttributeMetadataIngress,
+    EntityMetadataIngress,
+    MetadataListResponseIngress,
+)
+from .tables import DataverseSyncMode, DataverseTableSpec
+from .types import (
+    RuntimeFieldSpec,
+    build_dlt_column_hints,
+    field_spec_for_attribute,
+)
+
+# Field names already declared on DataverseRowBase (and CtxModel via inheritance).
+# A runtime field whose Python name lands in this set would silently shadow the
+# parent contract via pydantic.create_model, breaking tombstone semantics or the
+# _ctx_binding_id invariant. The collision check in _create_runtime_record_model
+# rejects such attributes loudly at warmup.
+_RESERVED_ROW_BASE_FIELDS: frozenset[str] = frozenset(
+    DataverseRowBase.model_fields.keys()
+)
+
+ENTITY_SELECT = "LogicalName,EntitySetName,PrimaryIdAttribute"
+# Only base-type fields. Subtype fields (Precision, Targets, Format,
+# DateTimeBehavior) live on Microsoft.Dynamics.CRM.{Decimal,Lookup,DateTime,...}
+# AttributeMetadata, NOT on the base type. Microsoft's OData rejects
+# $select=Precision,... here with HTTP 400 because the cast is implicit.
+# Decimal columns therefore default to numeric(28, 4); DateTime columns default
+# to timestamptz (UserLocal); Lookup loses Targets metadata. See
+# internal/planned/2026-05-03-dataverse-subtype-metadata-fetch.md for the
+# upgrade path (per-subtype cast queries, ~5x metadata round-trips at warmup,
+# tenant-specific precision + Format + DateTimeBehavior fidelity).
+ATTRIBUTE_SELECT = (
+    "LogicalName,"
+    "AttributeType,"
+    "AttributeTypeName,"
+    "AttributeOf,"
+    "IsValidForRead,"
+    "IsCustomAttribute,"
+    "IsPrimaryId,"
+    "Description,"
+    "DisplayName"
+)
+
+
+@dataclass(frozen=True)
+class RuntimeDataverseTableSchema:
+    spec: DataverseTableSpec
+    fields: tuple[RuntimeFieldSpec, ...]
+    record_model: type[DataverseRowBase]
+    select_columns: tuple[str, ...]
+    dlt_columns: dict[str, dict[str, Any]]
+
+
+def build_runtime_table_schemas(
+    *,
+    client: Any,
+    specs: Iterable[DataverseTableSpec],
+) -> dict[str, RuntimeDataverseTableSchema]:
+    return {
+        schema.spec.entity_set: schema
+        for schema in (
+            _build_runtime_table_schema(client=client, spec=spec) for spec in specs
+        )
+    }
+
+
+def _build_runtime_table_schema(
+    *,
+    client: Any,
+    spec: DataverseTableSpec,
+) -> RuntimeDataverseTableSchema:
+    entity = _fetch_entity_metadata(client=client, entity_set=spec.entity_set)
+    attributes = _fetch_attribute_metadata(
+        client=client,
+        logical_name=entity.logical_name,
+    )
+
+    fields: list[RuntimeFieldSpec] = []
+    for attribute in attributes:
+        spec_field = field_spec_for_attribute(attribute)
+        if spec_field is None:
+            continue
+
+        validate_identifier(
+            spec_field.logical_name,
+            context=f"{spec.entity_set}.{spec_field.logical_name}",
+        )
+        for ann in spec_field.annotation_columns:
+            validate_identifier(
+                ann.column_name,
+                context=f"{spec.entity_set}.{ann.column_name}",
+            )
+
+        fields.append(spec_field)
+
+    select_columns = _compute_select_columns(
+        fields=fields,
+        primary_key=entity.primary_id_attribute,
+    )
+    record_model = _create_runtime_record_model(
+        class_name=f"{pascal_case(entity.entity_set_name)}Row",
+        fields=fields,
+        primary_key=entity.primary_id_attribute,
+        entity_set=spec.entity_set,
+    )
+    dlt_columns = build_dlt_column_hints(
+        tuple(fields),
+        sync_mode_is_delta=spec.sync_mode is DataverseSyncMode.DELTA,
+    )
+
+    return RuntimeDataverseTableSchema(
+        spec=spec,
+        fields=tuple(fields),
+        record_model=record_model,
+        select_columns=select_columns,
+        dlt_columns=dlt_columns,
+    )
+
+
+def _compute_select_columns(
+    *,
+    fields: list[RuntimeFieldSpec],
+    primary_key: str,
+) -> tuple[str, ...]:
+    columns = [field.logical_name for field in fields]
+    if primary_key not in columns:
+        # PK injection here mirrors the model-side fallback in
+        # _create_runtime_record_model; keep the two consistent.
+        columns.insert(0, primary_key)
+    # Preserve order while deduping.
+    seen: set[str] = set()
+    deduped: list[str] = []
+    for col in columns:
+        if col not in seen:
+            seen.add(col)
+            deduped.append(col)
+    return tuple(deduped)
+
+
+def _create_runtime_record_model(
+    *,
+    class_name: str,
+    fields: list[RuntimeFieldSpec],
+    primary_key: str,
+    entity_set: str,
+) -> type[DataverseRowBase]:
+    """Construct a per-table CtxModel subclass via pydantic.create_model.
+
+    Each Dataverse attribute becomes a Pydantic field with a lenient ingress
+    type (string for date/timestamp/uuid, Decimal for numeric, native Python
+    scalars for others — dlt's normalizer handles the destination cast).
+    Field aliases match the Dataverse logical name so record validation can
+    take the raw OData payload directly.
+
+    Raises PluginConfigurationError if any attribute's Python field name
+    collides with a field on DataverseRowBase. pydantic.create_model would
+    silently override the parent field, breaking the row metadata contract.
+    """
+    field_definitions: dict[str, tuple[Any, Any]] = {}
+    field_logical_names = {f.logical_name for f in fields}
+    pk_present_in_fields = primary_key in field_logical_names
+
+    if not pk_present_in_fields:
+        # Add the PK as a string field so the schema knows about it for
+        # OData $select ordering. Most tenants always include the PK; this
+        # mirrors the select-side fallback in _compute_select_columns.
+        pk_py_name = safe_identifier(primary_key)
+        _check_no_row_base_collision(
+            python_name=pk_py_name,
+            logical_name=primary_key,
+            entity_set=entity_set,
+        )
+        field_definitions[pk_py_name] = (
+            IdStr | None,
+            Field(default=None, alias=primary_key),
+        )
+
+    for field in fields:
+        py_name = safe_identifier(field.logical_name)
+        _check_no_row_base_collision(
+            python_name=py_name,
+            logical_name=field.logical_name,
+            entity_set=entity_set,
+        )
+        annotation = _ingress_type_for(field.pg_type)
+        alias = field.logical_name if py_name != field.logical_name else None
+        default_kwargs: dict[str, Any] = {"default": None}
+        if alias is not None:
+            default_kwargs["alias"] = alias
+        field_definitions[py_name] = (
+            annotation | None,
+            Field(**default_kwargs),
+        )
+
+        for ann in field.annotation_columns:
+            ann_py_name = safe_identifier(ann.column_name)
+            _check_no_row_base_collision(
+                python_name=ann_py_name,
+                logical_name=ann.column_name,
+                entity_set=entity_set,
+            )
+            ann_alias = ann.column_name if ann_py_name != ann.column_name else None
+            ann_kwargs: dict[str, Any] = {"default": None}
+            if ann_alias is not None:
+                ann_kwargs["alias"] = ann_alias
+            field_definitions[ann_py_name] = (
+                str | None,
+                Field(**ann_kwargs),
+            )
+
+    return create_model(  # type: ignore[no-any-return]
+        class_name,
+        __base__=DataverseRowBase,
+        __module__=__name__,
+        **field_definitions,
+    )
+
+
+def _check_no_row_base_collision(
+    *,
+    python_name: str,
+    logical_name: str,
+    entity_set: str,
+) -> None:
+    if python_name in _RESERVED_ROW_BASE_FIELDS:
+        raise PluginConfigurationError(
+            f"Dataverse attribute {logical_name!r} on entity_set "
+            f"{entity_set!r} produces field name {python_name!r}, which "
+            f"would silently shadow reserved {python_name!r} on "
+            "DataverseRowBase. This collision is unhandled."
+        )
+
+
+def _ingress_type_for(pg_type: str) -> Any:
+    """Map dlt pg_type back to the lenient Python ingress type.
+
+    Most scalars arrive from OData as JSON-typed values; pydantic + dlt's
+    normalizer coerce on the way to postgres.
+    """
+    if pg_type == "bigint":
+        return int
+    if pg_type == "double":
+        return float
+    if pg_type == "bool":
+        return bool
+    if pg_type == "bigint[]":
+        # Dataverse v9.2 Web API serializes MultiSelectPicklist as a
+        # comma-separated string of option-set integers (e.g.
+        # "975620006,975620013"), not a JSON array. The attribute metadata
+        # describes the conceptual type (list of options) but is silent on
+        # wire format; verified empirically against a real tenant on
+        # 2026-05-12. A pre-validator splits the CSV before pydantic
+        # validates against list[int]; if Microsoft ever ships an actual
+        # JSON array, the passthrough branch keeps it working.
+        return Annotated[list[int], BeforeValidator(_coerce_multiselect_picklist_csv)]
+    if pg_type == "numeric":
+        # Decimal/Money values arrive as JSON numbers (e.g. 0.5, 8.0), not
+        # strings. Pydantic auto-coerces JSON numbers to Decimal in lenient
+        # mode (CtxModel inherits extra="forbid" but not strict=True).
+        return Decimal
+    # text, uuid, date, timestamp arrive as strings from OData.
+    return str
+
+
+def _coerce_multiselect_picklist_csv(value: Any) -> Any:
+    if isinstance(value, str):
+        return [int(part) for part in value.split(",")]
+    return value
+
+
+def _fetch_entity_metadata(
+    *,
+    client: Any,
+    entity_set: str,
+) -> EntityMetadataIngress:
+    payload = client.get_json(
+        "EntityDefinitions",
+        params={
+            "$select": ENTITY_SELECT,
+            "$filter": f"EntitySetName eq '{escape_odata_string(entity_set)}'",
+        },
+    )
+    response = MetadataListResponseIngress.model_validate(payload)
+    if len(response.value) != 1:
+        raise PluginConfigurationError(
+            f"Expected exactly one EntityDefinitions match for {entity_set}, "
+            f"got {len(response.value)}. Confirm the entity_set name in TABLE_SPECS "
+            "exists in the tenant's Dataverse schema."
+        )
+    return EntityMetadataIngress.model_validate(response.value[0])
+
+
+def _fetch_attribute_metadata(
+    *,
+    client: Any,
+    logical_name: str,
+) -> tuple[AttributeMetadataIngress, ...]:
+    payload = client.get_json(
+        f"EntityDefinitions(LogicalName='{escape_odata_string(logical_name)}')"
+        "/Attributes",
+        params={"$select": ATTRIBUTE_SELECT},
+    )
+    response = MetadataListResponseIngress.model_validate(payload)
+    return tuple(
+        AttributeMetadataIngress.model_validate(item) for item in response.value
+    )

shared_plugins/microsoft_dataverse/source.py

@@ -0,0 +1,250 @@
+"""Dlt source factory for any Dataverse-backed plugin.
+
+Plugin asset bodies call `build_dataverse_dlt_source(plugin_id, job, ...)`
+to produce a `@dlt.source`-wrapped tuple of resources. One resource per
+active table (delta or snapshot mode). The plugin's component.py is
+agnostic to OData/Pydantic/runtime-schema concerns -- those all live here.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Iterator, Mapping
+from typing import Any
+
+import dlt
+from shared_plugins.bindings import ResolvedBindingModels, iter_active_model_rows
+from shared_plugins.naming import dlt_resource_name, dlt_source_name
+from shared_plugins.resources import ctx_dlt_resource
+
+from .client import DataverseClient
+from .ctx import DataverseRowBase
+from .runtime_schema import RuntimeDataverseTableSchema, build_runtime_table_schemas
+from .tables import DataverseSyncMode, DataverseTableSpec
+from .translators import dataverse_record_to_row
+
+LOGGER = logging.getLogger(__name__)
+DELTA_LINK_KEY = "delta_link"
+SNAPSHOT_WRITE_DISPOSITION = {
+    "disposition": "merge",
+    "strategy": "delete-insert",
+}
+SNAPSHOT_MERGE_KEY = ("_ctx_binding_id",)
+
+
+def build_dataverse_dlt_source(
+    *,
+    plugin_id: str,
+    job: str,
+    binding_id: str,
+    client: DataverseClient,
+    binding_models: ResolvedBindingModels,
+    specs: tuple[DataverseTableSpec, ...],
+) -> Any:
+    active_specs = tuple(
+        spec for spec in specs if spec.resource_name in binding_models.active
+    )
+    runtime_schemas: Mapping[str, RuntimeDataverseTableSchema] = (
+        build_runtime_table_schemas(client=client, specs=active_specs)
+    )
+
+    @dlt.source(name=dlt_source_name(plugin_id, job))
+    def _source() -> tuple[Any, ...]:
+        resources: list[Any] = []
+        for spec in active_specs:
+            schema = runtime_schemas[spec.entity_set]
+            if spec.sync_mode is DataverseSyncMode.DELTA:
+                resources.append(
+                    _build_delta_resource(
+                        binding_id=binding_id,
+                        client=client,
+                        schema=schema,
+                        binding_models=binding_models,
+                    )
+                )
+            else:
+                resources.append(
+                    _build_snapshot_resource(
+                        binding_id=binding_id,
+                        client=client,
+                        schema=schema,
+                        binding_models=binding_models,
+                    )
+                )
+
+        if not resources:
+            LOGGER.info(
+                "%s.%s.empty_source binding_id=%s active_models=%s",
+                plugin_id,
+                job,
+                binding_id,
+                tuple(binding_models.active),
+            )
+
+        return tuple(resources)
+
+    return _source()
+
+
+def _build_delta_resource(
+    *,
+    binding_id: str,
+    client: DataverseClient,
+    schema: RuntimeDataverseTableSchema,
+    binding_models: ResolvedBindingModels,
+) -> Any:
+    spec = schema.spec
+
+    @ctx_dlt_resource(
+        name=dlt_resource_name(spec.resource_name),
+        write_disposition="merge",
+        primary_key=("_ctx_binding_id", spec.primary_key),
+        columns=schema.dlt_columns,
+    )
+    def delta_resource() -> Iterator[DataverseRowBase]:
+        state = dlt.current.resource_state()
+        initial_delta_link = state.get(DELTA_LINK_KEY)
+        if initial_delta_link is not None and not isinstance(initial_delta_link, str):
+            raise RuntimeError(
+                f"Stored Dataverse deltaLink for {spec.entity_set} is not a string."
+            )
+        next_delta_link: dict[str, str] = {}
+        rows = _iter_delta_rows(
+            binding_id=binding_id,
+            client=client,
+            schema=schema,
+            initial_delta_link=initial_delta_link,
+            next_delta_link=next_delta_link,
+        )
+        yield from iter_active_model_rows(
+            model_name=spec.resource_name,
+            rows=rows,
+            binding_models=binding_models,
+        )
+        state[DELTA_LINK_KEY] = next_delta_link[DELTA_LINK_KEY]
+
+    return delta_resource
+
+
+def _build_snapshot_resource(
+    *,
+    binding_id: str,
+    client: DataverseClient,
+    schema: RuntimeDataverseTableSchema,
+    binding_models: ResolvedBindingModels,
+) -> Any:
+    spec = schema.spec
+
+    @ctx_dlt_resource(
+        name=dlt_resource_name(spec.resource_name),
+        write_disposition=SNAPSHOT_WRITE_DISPOSITION,
+        merge_key=SNAPSHOT_MERGE_KEY,
+        primary_key=("_ctx_binding_id", spec.primary_key),
+        columns=schema.dlt_columns,
+    )
+    def snapshot_resource() -> Iterator[DataverseRowBase]:
+        rows = _iter_snapshot_rows(
+            binding_id=binding_id,
+            client=client,
+            schema=schema,
+        )
+        yield from iter_active_model_rows(
+            model_name=spec.resource_name,
+            rows=rows,
+            binding_models=binding_models,
+        )
+
+    return snapshot_resource
+
+
+def _iter_delta_rows(
+    *,
+    binding_id: str,
+    client: DataverseClient,
+    schema: RuntimeDataverseTableSchema,
+    initial_delta_link: str | None,
+    next_delta_link: dict[str, str],
+) -> Iterator[DataverseRowBase]:
+    spec = schema.spec
+    latest_delta_link = initial_delta_link
+    row_count = 0
+    page_count = 0
+    t0 = time.monotonic()
+
+    LOGGER.info(
+        "dataverse.delta.start entity_set=%s mode=%s",
+        spec.entity_set,
+        "incremental" if initial_delta_link else "full",
+    )
+
+    for page in client.iter_change_tracking_pages(
+        spec,
+        delta_link=initial_delta_link,
+        select=schema.select_columns,
+    ):
+        page_count += 1
+        if page.delta_link:
+            latest_delta_link = page.delta_link
+
+        for record in page.value:
+            raw_payload = record.model_dump(by_alias=True, exclude_none=True)
+            row_count += 1
+            yield dataverse_record_to_row(
+                binding_id=binding_id,
+                schema=schema,
+                record=record,
+                raw_payload=raw_payload,
+            )
+
+    if latest_delta_link is None or "$deltatoken" not in latest_delta_link:
+        raise RuntimeError(
+            f"Dataverse delta drain for {spec.entity_set} did not finish with "
+            "a deltaLink."
+        )
+    next_delta_link[DELTA_LINK_KEY] = latest_delta_link
+
+    LOGGER.info(
+        "dataverse.delta.done entity_set=%s pages=%d rows=%d elapsed=%.1fs",
+        spec.entity_set,
+        page_count,
+        row_count,
+        time.monotonic() - t0,
+    )
+
+
+def _iter_snapshot_rows(
+    *,
+    binding_id: str,
+    client: DataverseClient,
+    schema: RuntimeDataverseTableSchema,
+) -> Iterator[DataverseRowBase]:
+    spec = schema.spec
+    row_count = 0
+    page_count = 0
+    t0 = time.monotonic()
+
+    LOGGER.info("dataverse.snapshot.start entity_set=%s", spec.entity_set)
+
+    for page in client.iter_snapshot_pages(
+        spec,
+        select=schema.select_columns,
+    ):
+        page_count += 1
+        for record in page.value:
+            raw_payload = record.model_dump(by_alias=True, exclude_none=True)
+            row_count += 1
+            yield dataverse_record_to_row(
+                binding_id=binding_id,
+                schema=schema,
+                record=record,
+                raw_payload=raw_payload,
+            )
+
+    LOGGER.info(
+        "dataverse.snapshot.done entity_set=%s pages=%d rows=%d elapsed=%.1fs",
+        spec.entity_set,
+        page_count,
+        row_count,
+        time.monotonic() - t0,
+    )

shared_plugins/microsoft_dataverse/tables.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class DataverseSyncMode(str, Enum):
+    DELTA = "delta"
+    SNAPSHOT = "snapshot"
+
+
+@dataclass(frozen=True)
+class DataverseTableSpec:
+    """Identifies one Dataverse entity set and how it should be synced.
+
+    Fields:
+    - entity_set: the OData entity-set name (e.g. "msdyn_projects").
+    - primary_key: the Dataverse logical name of the table's PK column
+      (e.g. "msdyn_projectid"). Used as the dlt primary_key trailing
+      element after _ctx_binding_id. Plugin authors look up this value
+      from Dataverse EntityDefinitions.PrimaryIdAttribute and hard-code
+      it in the spec — keeps the spec list self-documenting.
+    - sync_mode: DELTA (Prefer: odata.track-changes) or SNAPSHOT (full pull).
+    - resource_name: the dlt resource name. Conventionally equal to
+      entity_set, but kept distinct so plugins can rename if needed.
+
+    Note: $select is NOT on the spec. Selected columns are computed at
+    warmup from per-tenant attribute metadata (see runtime_schema.py).
+    """
+
+    entity_set: str
+    primary_key: str
+    sync_mode: DataverseSyncMode
+    resource_name: str