contextbase-shared-plugins 0.0.0a1__py3-none-any.whl

shared_plugins/microsoft_dataverse/metadata.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+from shared_plugins.models import IngressModel
+
+
+class LocalizedLabelIngress(IngressModel):
+    # Verified 2026-05-03 against customer tenant with `Prefer: odata.include-annotations="*"`:
+    # no @odata.* decoration on this complex type, so extra="forbid" via IngressModel is safe.
+    label: str | None = Field(default=None, alias="Label")
+    language_code: int | None = Field(default=None, alias="LanguageCode")
+    is_managed: bool | None = Field(default=None, alias="IsManaged")
+    metadata_id: str | None = Field(default=None, alias="MetadataId")
+    has_changed: bool | None = Field(default=None, alias="HasChanged")
+
+
+class LabelIngress(IngressModel):
+    # Verified 2026-05-03 against customer tenant with `Prefer: odata.include-annotations="*"`:
+    # no @odata.* decoration on this complex type, so extra="forbid" via IngressModel is safe.
+    localized_labels: list[LocalizedLabelIngress] | None = Field(
+        default=None,
+        alias="LocalizedLabels",
+    )
+    user_localized_label: LocalizedLabelIngress | None = Field(
+        default=None,
+        alias="UserLocalizedLabel",
+    )
+
+
+class AttributeTypeNameIngress(IngressModel):
+    # Verified 2026-05-03 against customer tenant with `Prefer: odata.include-annotations="*"`:
+    # no @odata.* decoration on this complex type, so extra="forbid" via IngressModel is safe.
+    value: str | None = Field(default=None, alias="Value")
+
+
+class AttributeMetadataIngress(BaseModel):
+    """Per-attribute metadata returned by EntityDefinitions/<n>/Attributes.
+
+    Uses extra="ignore" (rather than the codebase default extra="forbid")
+    because we $select only a subset of fields and Microsoft includes
+    @odata.* meta + a wide range of type-specific fields on every attribute.
+    The strict-rejection model would fail on every record. The trade-off is
+    documented and accepted at this single boundary.
+    """
+
+    model_config = ConfigDict(extra="ignore", populate_by_name=True)
+
+    logical_name: str = Field(alias="LogicalName")
+    attribute_type: str | None = Field(default=None, alias="AttributeType")
+    attribute_type_name: AttributeTypeNameIngress | None = Field(
+        default=None,
+        alias="AttributeTypeName",
+    )
+    attribute_of: str | None = Field(default=None, alias="AttributeOf")
+    is_valid_for_read: bool | None = Field(default=None, alias="IsValidForRead")
+    is_custom_attribute: bool = Field(default=False, alias="IsCustomAttribute")
+    is_primary_id: bool = Field(default=False, alias="IsPrimaryId")
+    description: LabelIngress | None = Field(default=None, alias="Description")
+    display_name: LabelIngress | None = Field(default=None, alias="DisplayName")
+    precision: int | None = Field(default=None, alias="Precision")
+    targets: list[str] | None = Field(default=None, alias="Targets")
+    format: str | None = Field(default=None, alias="Format")
+    date_time_behavior: dict[str, Any] | None = Field(
+        default=None,
+        alias="DateTimeBehavior",
+    )
+
+    @property
+    def type_name_value(self) -> str | None:
+        return (
+            None if self.attribute_type_name is None else self.attribute_type_name.value
+        )
+
+
+class EntityMetadataIngress(IngressModel):
+    metadata_id: str | None = Field(default=None, alias="MetadataId")
+    logical_name: str = Field(alias="LogicalName")
+    entity_set_name: str = Field(alias="EntitySetName")
+    primary_id_attribute: str = Field(alias="PrimaryIdAttribute")
+
+
+class MetadataListResponseIngress(IngressModel):
+    odata_context: str | None = Field(default=None, alias="@odata.context")
+    value: list[dict[str, Any]] = Field(default_factory=list)
+
+
+def extract_label(label: LabelIngress | None) -> str | None:
+    """Pull the most useful localized label, or None.
+
+    Tries UserLocalizedLabel first (the calling principal's preferred locale).
+    Falls back to the first non-empty LocalizedLabels entry for tenants where
+    service-principal access doesn't populate UserLocalizedLabel. Treats empty
+    and whitespace-only strings as absent.
+    """
+    if label is None:
+        return None
+    if label.user_localized_label is not None:
+        text = label.user_localized_label.label
+        if text and text.strip():
+            return text
+    for localized in label.localized_labels or ():
+        if localized.label and localized.label.strip():
+            return localized.label
+    return None

shared_plugins/microsoft_dataverse/runtime_schema.py

@@ -0,0 +1,332 @@
+"""Per-table schema construction at warmup.
+
+Fetches Dataverse entity + attribute metadata, builds RuntimeFieldSpecs,
+validates 63-byte postgres identifier limits, and assembles per-table
+runtime CtxModel subclasses + dlt column hints. The plugin's source factory
+calls `build_runtime_table_schemas(...)` once per asset run.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from decimal import Decimal
+from typing import Annotated, Any
+
+from pydantic import BeforeValidator, Field, create_model
+from shared_plugins.exceptions import PluginConfigurationError
+from shared_plugins.models import IdStr
+
+from .ctx import DataverseRowBase
+from .identifiers import (
+    escape_odata_string,
+    pascal_case,
+    safe_identifier,
+    validate_identifier,
+)
+from .metadata import (
+    AttributeMetadataIngress,
+    EntityMetadataIngress,
+    MetadataListResponseIngress,
+)
+from .tables import DataverseSyncMode, DataverseTableSpec
+from .types import (
+    RuntimeFieldSpec,
+    build_dlt_column_hints,
+    field_spec_for_attribute,
+)
+
+# Field names already declared on DataverseRowBase (and CtxModel via inheritance).
+# A runtime field whose Python name lands in this set would silently shadow the
+# parent contract via pydantic.create_model, breaking tombstone semantics or the
+# _ctx_binding_id invariant. The collision check in _create_runtime_record_model
+# rejects such attributes loudly at warmup.
+_RESERVED_ROW_BASE_FIELDS: frozenset[str] = frozenset(
+    DataverseRowBase.model_fields.keys()
+)
+
+ENTITY_SELECT = "LogicalName,EntitySetName,PrimaryIdAttribute"
+# Only base-type fields. Subtype fields (Precision, Targets, Format,
+# DateTimeBehavior) live on Microsoft.Dynamics.CRM.{Decimal,Lookup,DateTime,...}
+# AttributeMetadata, NOT on the base type. Microsoft's OData rejects
+# $select=Precision,... here with HTTP 400 because the cast is implicit.
+# Decimal columns therefore default to numeric(28, 4); DateTime columns default
+# to timestamptz (UserLocal); Lookup loses Targets metadata. See
+# internal/planned/2026-05-03-dataverse-subtype-metadata-fetch.md for the
+# upgrade path (per-subtype cast queries, ~5x metadata round-trips at warmup,
+# tenant-specific precision + Format + DateTimeBehavior fidelity).
+ATTRIBUTE_SELECT = (
+    "LogicalName,"
+    "AttributeType,"
+    "AttributeTypeName,"
+    "AttributeOf,"
+    "IsValidForRead,"
+    "IsCustomAttribute,"
+    "IsPrimaryId,"
+    "Description,"
+    "DisplayName"
+)
+
+
+@dataclass(frozen=True)
+class RuntimeDataverseTableSchema:
+    spec: DataverseTableSpec
+    fields: tuple[RuntimeFieldSpec, ...]
+    record_model: type[DataverseRowBase]
+    select_columns: tuple[str, ...]
+    dlt_columns: dict[str, dict[str, Any]]
+
+
+def build_runtime_table_schemas(
+    *,
+    client: Any,
+    specs: Iterable[DataverseTableSpec],
+) -> dict[str, RuntimeDataverseTableSchema]:
+    return {
+        schema.spec.entity_set: schema
+        for schema in (
+            _build_runtime_table_schema(client=client, spec=spec) for spec in specs
+        )
+    }
+
+
+def _build_runtime_table_schema(
+    *,
+    client: Any,
+    spec: DataverseTableSpec,
+) -> RuntimeDataverseTableSchema:
+    entity = _fetch_entity_metadata(client=client, entity_set=spec.entity_set)
+    attributes = _fetch_attribute_metadata(
+        client=client,
+        logical_name=entity.logical_name,
+    )
+
+    fields: list[RuntimeFieldSpec] = []
+    for attribute in attributes:
+        spec_field = field_spec_for_attribute(attribute)
+        if spec_field is None:
+            continue
+
+        validate_identifier(
+            spec_field.logical_name,
+            context=f"{spec.entity_set}.{spec_field.logical_name}",
+        )
+        for ann in spec_field.annotation_columns:
+            validate_identifier(
+                ann.column_name,
+                context=f"{spec.entity_set}.{ann.column_name}",
+            )
+
+        fields.append(spec_field)
+
+    select_columns = _compute_select_columns(
+        fields=fields,
+        primary_key=entity.primary_id_attribute,
+    )
+    record_model = _create_runtime_record_model(
+        class_name=f"{pascal_case(entity.entity_set_name)}Row",
+        fields=fields,
+        primary_key=entity.primary_id_attribute,
+        entity_set=spec.entity_set,
+    )
+    dlt_columns = build_dlt_column_hints(
+        tuple(fields),
+        sync_mode_is_delta=spec.sync_mode is DataverseSyncMode.DELTA,
+    )
+
+    return RuntimeDataverseTableSchema(
+        spec=spec,
+        fields=tuple(fields),
+        record_model=record_model,
+        select_columns=select_columns,
+        dlt_columns=dlt_columns,
+    )
+
+
+def _compute_select_columns(
+    *,
+    fields: list[RuntimeFieldSpec],
+    primary_key: str,
+) -> tuple[str, ...]:
+    columns = [field.logical_name for field in fields]
+    if primary_key not in columns:
+        # PK injection here mirrors the model-side fallback in
+        # _create_runtime_record_model; keep the two consistent.
+        columns.insert(0, primary_key)
+    # Preserve order while deduping.
+    seen: set[str] = set()
+    deduped: list[str] = []
+    for col in columns:
+        if col not in seen:
+            seen.add(col)
+            deduped.append(col)
+    return tuple(deduped)
+
+
+def _create_runtime_record_model(
+    *,
+    class_name: str,
+    fields: list[RuntimeFieldSpec],
+    primary_key: str,
+    entity_set: str,
+) -> type[DataverseRowBase]:
+    """Construct a per-table CtxModel subclass via pydantic.create_model.
+
+    Each Dataverse attribute becomes a Pydantic field with a lenient ingress
+    type (string for date/timestamp/uuid, Decimal for numeric, native Python
+    scalars for others — dlt's normalizer handles the destination cast).
+    Field aliases match the Dataverse logical name so record validation can
+    take the raw OData payload directly.
+
+    Raises PluginConfigurationError if any attribute's Python field name
+    collides with a field on DataverseRowBase. pydantic.create_model would
+    silently override the parent field, breaking the row metadata contract.
+    """
+    field_definitions: dict[str, tuple[Any, Any]] = {}
+    field_logical_names = {f.logical_name for f in fields}
+    pk_present_in_fields = primary_key in field_logical_names
+
+    if not pk_present_in_fields:
+        # Add the PK as a string field so the schema knows about it for
+        # OData $select ordering. Most tenants always include the PK; this
+        # mirrors the select-side fallback in _compute_select_columns.
+        pk_py_name = safe_identifier(primary_key)
+        _check_no_row_base_collision(
+            python_name=pk_py_name,
+            logical_name=primary_key,
+            entity_set=entity_set,
+        )
+        field_definitions[pk_py_name] = (
+            IdStr | None,
+            Field(default=None, alias=primary_key),
+        )
+
+    for field in fields:
+        py_name = safe_identifier(field.logical_name)
+        _check_no_row_base_collision(
+            python_name=py_name,
+            logical_name=field.logical_name,
+            entity_set=entity_set,
+        )
+        annotation = _ingress_type_for(field.pg_type)
+        alias = field.logical_name if py_name != field.logical_name else None
+        default_kwargs: dict[str, Any] = {"default": None}
+        if alias is not None:
+            default_kwargs["alias"] = alias
+        field_definitions[py_name] = (
+            annotation | None,
+            Field(**default_kwargs),
+        )
+
+        for ann in field.annotation_columns:
+            ann_py_name = safe_identifier(ann.column_name)
+            _check_no_row_base_collision(
+                python_name=ann_py_name,
+                logical_name=ann.column_name,
+                entity_set=entity_set,
+            )
+            ann_alias = ann.column_name if ann_py_name != ann.column_name else None
+            ann_kwargs: dict[str, Any] = {"default": None}
+            if ann_alias is not None:
+                ann_kwargs["alias"] = ann_alias
+            field_definitions[ann_py_name] = (
+                str | None,
+                Field(**ann_kwargs),
+            )
+
+    return create_model(  # type: ignore[no-any-return]
+        class_name,
+        __base__=DataverseRowBase,
+        __module__=__name__,
+        **field_definitions,
+    )
+
+
+def _check_no_row_base_collision(
+    *,
+    python_name: str,
+    logical_name: str,
+    entity_set: str,
+) -> None:
+    if python_name in _RESERVED_ROW_BASE_FIELDS:
+        raise PluginConfigurationError(
+            f"Dataverse attribute {logical_name!r} on entity_set "
+            f"{entity_set!r} produces field name {python_name!r}, which "
+            f"would silently shadow reserved {python_name!r} on "
+            "DataverseRowBase. This collision is unhandled."
+        )
+
+
+def _ingress_type_for(pg_type: str) -> Any:
+    """Map dlt pg_type back to the lenient Python ingress type.
+
+    Most scalars arrive from OData as JSON-typed values; pydantic + dlt's
+    normalizer coerce on the way to postgres.
+    """
+    if pg_type == "bigint":
+        return int
+    if pg_type == "double":
+        return float
+    if pg_type == "bool":
+        return bool
+    if pg_type == "bigint[]":
+        # Dataverse v9.2 Web API serializes MultiSelectPicklist as a
+        # comma-separated string of option-set integers (e.g.
+        # "975620006,975620013"), not a JSON array. The attribute metadata
+        # describes the conceptual type (list of options) but is silent on
+        # wire format; verified empirically against a real tenant on
+        # 2026-05-12. A pre-validator splits the CSV before pydantic
+        # validates against list[int]; if Microsoft ever ships an actual
+        # JSON array, the passthrough branch keeps it working.
+        return Annotated[list[int], BeforeValidator(_coerce_multiselect_picklist_csv)]
+    if pg_type == "numeric":
+        # Decimal/Money values arrive as JSON numbers (e.g. 0.5, 8.0), not
+        # strings. Pydantic auto-coerces JSON numbers to Decimal in lenient
+        # mode (CtxModel inherits extra="forbid" but not strict=True).
+        return Decimal
+    # text, uuid, date, timestamp arrive as strings from OData.
+    return str
+
+
+def _coerce_multiselect_picklist_csv(value: Any) -> Any:
+    if isinstance(value, str):
+        return [int(part) for part in value.split(",")]
+    return value
+
+
+def _fetch_entity_metadata(
+    *,
+    client: Any,
+    entity_set: str,
+) -> EntityMetadataIngress:
+    payload = client.get_json(
+        "EntityDefinitions",
+        params={
+            "$select": ENTITY_SELECT,
+            "$filter": f"EntitySetName eq '{escape_odata_string(entity_set)}'",
+        },
+    )
+    response = MetadataListResponseIngress.model_validate(payload)
+    if len(response.value) != 1:
+        raise PluginConfigurationError(
+            f"Expected exactly one EntityDefinitions match for {entity_set}, "
+            f"got {len(response.value)}. Confirm the entity_set name in TABLE_SPECS "
+            "exists in the tenant's Dataverse schema."
+        )
+    return EntityMetadataIngress.model_validate(response.value[0])
+
+
+def _fetch_attribute_metadata(
+    *,
+    client: Any,
+    logical_name: str,
+) -> tuple[AttributeMetadataIngress, ...]:
+    payload = client.get_json(
+        f"EntityDefinitions(LogicalName='{escape_odata_string(logical_name)}')"
+        "/Attributes",
+        params={"$select": ATTRIBUTE_SELECT},
+    )
+    response = MetadataListResponseIngress.model_validate(payload)
+    return tuple(
+        AttributeMetadataIngress.model_validate(item) for item in response.value
+    )