contextbase-shared-plugins 0.2.3__py3-none-any.whl

shared_plugins/microsoft_dataverse/translators.py

@@ -0,0 +1,128 @@
+"""Per-record translation: Dataverse OData payload → per-table CtxModel instance.
+
+The translator walks the raw OData payload, splits annotation keys
+(`<col>@<annotation>`) per the registry policy, and constructs an instance
+of the per-table runtime CtxModel subclass. Returns a CtxModel — NOT a dict
+— because @ctx_dlt_resource validates emitted rows are CtxModel instances.
+
+Unknown annotations raise PluginConfigurationError. Dropped annotations are
+silently omitted (they're triaged and intentional). Kept annotations land
+on the companion column suffix.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from shared_plugins.exceptions import PluginConfigurationError
+
+from .annotations import (
+    DROPPED_ODATA_ANNOTATIONS,
+    ODATA_ANNOTATION_COLUMN_SUFFIXES,
+)
+from .ctx import DataverseRowBase
+from .ingress import DataverseRecordIngress, is_deleted_record
+from .runtime_schema import RuntimeDataverseTableSchema
+
+# Keys that DataverseRecordIngress emits via model_dump(by_alias=True): the
+# @odata.* aliases plus the non-aliased tombstone fields (`id`, `reason`).
+# The column-flatten step skips these so only column data (extras under
+# DataverseRecordIngress's extra="allow") reaches the per-table row model.
+# Derived from the ingress model so new envelope fields are picked up
+# automatically without touching the flatten filter.
+_DATAVERSE_ENVELOPE_KEYS: frozenset[str] = frozenset(
+    (field.alias or name) for name, field in DataverseRecordIngress.model_fields.items()
+)
+
+
+def dataverse_record_to_row(
+    *,
+    binding_id: str,
+    schema: RuntimeDataverseTableSchema,
+    record: DataverseRecordIngress,
+    raw_payload: dict[str, Any],
+) -> DataverseRowBase:
+    """Convert one Dataverse record into a per-table CtxModel instance.
+
+    `raw_payload` is the same OData record dict that produced `record` — the
+    strict envelope view (`record`) carries only the validated envelope
+    fields, while the full raw view (`raw_payload`) keeps every key from the
+    source response, including annotation keys (`<col>@<ann>`) that the
+    strict envelope does not preserve.
+
+    Tombstones (delta-stream delete entries) arrive without column data and
+    without the entity's PK column; the deleted row's identifier is on
+    `record.id`. This translator injects it as the PK column so dlt's
+    `hard_delete` hint on `is_deleted` (see build_dlt_column_hints in
+    types.py) issues a DELETE for the matching destination row during merge.
+    """
+    deleted = is_deleted_record(record, schema.spec, payload=raw_payload)
+    ctx_source_updated_at = raw_payload.get("modifiedon")
+
+    base_values: dict[str, Any] = {
+        "_ctx_binding_id": binding_id,
+        "_ctx_source_updated_at": ctx_source_updated_at,
+        "etag": record.odata_etag,
+        "is_deleted": deleted,
+        "delete_reason": record.reason,
+    }
+
+    column_values = _flatten_columns_and_annotations(
+        raw_payload=raw_payload,
+        schema=schema,
+    )
+    base_values.update(column_values)
+
+    if deleted:
+        if not isinstance(record.id, str):
+            raise PluginConfigurationError(
+                f"Dataverse tombstone for entity_set "
+                f"{schema.spec.entity_set!r} has reason={record.reason!r} "
+                "but no 'id' field on the envelope; cannot identify the "
+                "deleted row."
+            )
+        base_values[schema.spec.primary_key] = record.id
+
+    return schema.record_model.model_validate(base_values)
+
+
+def _flatten_columns_and_annotations(
+    *,
+    raw_payload: dict[str, Any],
+    schema: RuntimeDataverseTableSchema,
+) -> dict[str, Any]:
+    """Walk raw payload keys, returning a {column_name: value} dict.
+
+    - Envelope keys (declared on DataverseRecordIngress: @odata.* aliases
+      plus `id` and `reason`) are skipped — those are read from `record`
+      directly by the caller.
+    - Plain keys land under their logical name.
+    - Annotation keys (`<col>@<annotation>`) land under
+      `<col><suffix>` per ODATA_ANNOTATION_COLUMN_SUFFIXES, or are dropped
+      per DROPPED_ODATA_ANNOTATIONS, or raise on unknown.
+    """
+    out: dict[str, Any] = {}
+    for key, value in raw_payload.items():
+        if key in _DATAVERSE_ENVELOPE_KEYS:
+            continue
+
+        if "@" not in key:
+            out[key] = value
+            continue
+
+        base, _, annotation_token = key.partition("@")
+        annotation = f"@{annotation_token}"
+        if annotation in DROPPED_ODATA_ANNOTATIONS:
+            continue
+        suffix = ODATA_ANNOTATION_COLUMN_SUFFIXES.get(annotation)
+        if suffix is None:
+            raise PluginConfigurationError(
+                f"Encountered unknown OData annotation {annotation!r} on "
+                f"column {base!r} in entity_set {schema.spec.entity_set!r}. "
+                "Triage by adding it to ODATA_ANNOTATION_COLUMN_SUFFIXES "
+                "(keep) or DROPPED_ODATA_ANNOTATIONS (drop) in "
+                "shared_plugins.microsoft_dataverse.annotations."
+            )
+        out[f"{base}{suffix}"] = value
+
+    return out

shared_plugins/microsoft_dataverse/types.py

@@ -0,0 +1,346 @@
+"""Dataverse AttributeType → dlt TColumnSchema mapping.
+
+Single source of truth for every type-mapping decision the plan §9 locks in.
+Unknown types raise loudly — new Microsoft types must be added explicitly.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from shared_plugins.exceptions import PluginConfigurationError
+
+from .annotations import ODATA_ANNOTATION_COLUMN_SUFFIXES
+from .identifiers import annotation_column_name
+from .metadata import AttributeMetadataIngress, extract_label
+
+# Existing skips preserved from plugin's runtime_ingress.py.
+SKIPPED_ATTRIBUTE_TYPES: frozenset[str] = frozenset(
+    {
+        "Virtual",
+        "File",
+        "Image",
+        "CalendarRules",
+        "PartyList",
+        "ManagedProperty",
+    }
+)
+SKIPPED_ATTRIBUTE_NAMES: frozenset[str] = frozenset({"stageid"})
+
+LOOKUP_ATTRIBUTE_TYPES: frozenset[str] = frozenset({"Customer", "Lookup", "Owner"})
+PICKLIST_ATTRIBUTE_TYPES: frozenset[str] = frozenset({"Picklist", "State", "Status"})
+
+# Scalar AttributeTypes that emit @OData.Community.Display.V1.FormattedValue
+# alongside their raw value. Empirically observed on the customer tenant
+# across ~5000 non-null values per type (May 2026): all numeric scalars and
+# Boolean emit a locale-rendered string; Memo / String / Uniqueidentifier /
+# EntityName never did. If a new type starts emitting one, the translator's
+# unknown-column path fails loud at pydantic validation — add it here then.
+_SCALAR_TYPES_WITH_FORMATTED_VALUE: frozenset[str] = frozenset(
+    {"Integer", "BigInt", "Double", "Boolean"}
+)
+
+_ANNOTATION_DESCRIPTIONS: dict[str, str] = {
+    "_formatted_value": "Formatted value of {base}",
+    "_lookup_logical_name": "Logical-name of the entity referenced by {base}",
+}
+
+
+@dataclass(frozen=True)
+class RuntimeAnnotationColumn:
+    column_name: str
+    pg_type: str
+    description: str | None
+
+
+@dataclass(frozen=True)
+class RuntimeFieldSpec:
+    """Per-attribute description ready for column emission.
+
+    Captures both Description and DisplayName for the
+    `description ?? display_name` fallback at the dlt edge. Other
+    consumers (LLM tool definitions, sibling metadata exporters) can
+    read the rich object directly without re-deriving from raw metadata.
+    """
+
+    logical_name: str
+    pg_type: str
+    pg_type_args: dict[str, Any]
+    description: str | None
+    display_name: str | None
+    is_custom_attribute: bool
+    attribute_type: str
+    annotation_columns: tuple[RuntimeAnnotationColumn, ...]
+
+
+def field_spec_for_attribute(
+    attribute: AttributeMetadataIngress,
+) -> RuntimeFieldSpec | None:
+    """Build a RuntimeFieldSpec from Dataverse attribute metadata.
+
+    Returns None if the attribute should be skipped per the existing filter
+    rules (IsValidForRead=False, AttributeOf set, type/name in skip sets).
+
+    Raises PluginConfigurationError if AttributeType is unknown — new
+    Microsoft types require explicit triage in this module.
+    """
+    if attribute.is_valid_for_read is False:
+        return None
+    if attribute.attribute_of:
+        return None
+    if attribute.logical_name in SKIPPED_ATTRIBUTE_NAMES:
+        return None
+
+    attr_type = attribute.attribute_type or ""
+
+    # MultiSelectPicklist arrives as Virtual + AttributeTypeName.Value=MultiSelectPicklistType.
+    if attribute.type_name_value == "MultiSelectPicklistType":
+        return _build_multiselect_field(attribute)
+
+    if attr_type in SKIPPED_ATTRIBUTE_TYPES:
+        return None
+
+    if attr_type in LOOKUP_ATTRIBUTE_TYPES:
+        return _build_lookup_field(attribute)
+
+    if attr_type in PICKLIST_ATTRIBUTE_TYPES:
+        return _build_picklist_field(attribute)
+
+    if attr_type == "DateTime":
+        return _build_datetime_field(attribute)
+
+    if attr_type in ("Decimal", "Money"):
+        return _build_numeric_field(attribute)
+
+    pg_type = _SCALAR_TYPE_MAP.get(attr_type)
+    if pg_type is None:
+        raise PluginConfigurationError(
+            f"Unknown Dataverse AttributeType {attr_type!r} for attribute "
+            f"{attribute.logical_name!r}. New Microsoft types require explicit "
+            "triage in shared_plugins.microsoft_dataverse.types."
+        )
+
+    if attr_type in _SCALAR_TYPES_WITH_FORMATTED_VALUE:
+        annotations = _build_annotation_columns(
+            attribute.logical_name,
+            kept_suffixes=("_formatted_value",),
+            context_table=attribute.logical_name,
+        )
+    else:
+        annotations = ()
+
+    return RuntimeFieldSpec(
+        logical_name=attribute.logical_name,
+        pg_type=pg_type,
+        pg_type_args={},
+        description=extract_label(attribute.description),
+        display_name=extract_label(attribute.display_name),
+        is_custom_attribute=attribute.is_custom_attribute,
+        attribute_type=attr_type,
+        annotation_columns=annotations,
+    )
+
+
+_SCALAR_TYPE_MAP: dict[str, str] = {
+    "String": "text",
+    "Memo": "text",
+    "Integer": "bigint",
+    "BigInt": "bigint",
+    "Double": "double",
+    "Boolean": "bool",
+    "Uniqueidentifier": "uuid",
+    "EntityName": "text",
+}
+
+
+def _build_lookup_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
+    base = f"_{attr.logical_name}_value"
+    annotations = _build_annotation_columns(
+        base,
+        kept_suffixes=("_formatted_value", "_lookup_logical_name"),
+        context_table=attr.logical_name,
+    )
+    return RuntimeFieldSpec(
+        logical_name=base,
+        pg_type="uuid",
+        pg_type_args={},
+        description=extract_label(attr.description),
+        display_name=extract_label(attr.display_name),
+        is_custom_attribute=attr.is_custom_attribute,
+        attribute_type=attr.attribute_type or "",
+        annotation_columns=annotations,
+    )
+
+
+def _build_picklist_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
+    annotations = _build_annotation_columns(
+        attr.logical_name,
+        kept_suffixes=("_formatted_value",),
+        context_table=attr.logical_name,
+    )
+    return RuntimeFieldSpec(
+        logical_name=attr.logical_name,
+        pg_type="bigint",
+        pg_type_args={},
+        description=extract_label(attr.description),
+        display_name=extract_label(attr.display_name),
+        is_custom_attribute=attr.is_custom_attribute,
+        attribute_type=attr.attribute_type or "",
+        annotation_columns=annotations,
+    )
+
+
+def _build_multiselect_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
+    annotations = _build_annotation_columns(
+        attr.logical_name,
+        kept_suffixes=("_formatted_value",),
+        context_table=attr.logical_name,
+    )
+    return RuntimeFieldSpec(
+        logical_name=attr.logical_name,
+        pg_type="bigint[]",
+        pg_type_args={},
+        description=extract_label(attr.description),
+        display_name=extract_label(attr.display_name),
+        is_custom_attribute=attr.is_custom_attribute,
+        attribute_type=attr.attribute_type or "MultiSelectPicklist",
+        annotation_columns=annotations,
+    )
+
+
+def _build_datetime_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
+    fmt = attr.format
+    behavior = (attr.date_time_behavior or {}).get("Value")
+    if fmt == "DateOnly":
+        pg_type = "date"
+        pg_args: dict[str, Any] = {}
+    else:
+        pg_type = "timestamp"
+        pg_args = {"timezone": behavior != "TimeZoneIndependent"}
+
+    annotations = _build_annotation_columns(
+        attr.logical_name,
+        kept_suffixes=("_formatted_value",),
+        context_table=attr.logical_name,
+    )
+
+    return RuntimeFieldSpec(
+        logical_name=attr.logical_name,
+        pg_type=pg_type,
+        pg_type_args=pg_args,
+        description=extract_label(attr.description),
+        display_name=extract_label(attr.display_name),
+        is_custom_attribute=attr.is_custom_attribute,
+        attribute_type=attr.attribute_type or "DateTime",
+        annotation_columns=annotations,
+    )
+
+
+def _build_numeric_field(attr: AttributeMetadataIngress) -> RuntimeFieldSpec:
+    if attr.attribute_type == "Money":
+        precision = 28
+        scale = 4
+    else:
+        precision = 28
+        scale = attr.precision if attr.precision is not None else 4
+
+    annotations = _build_annotation_columns(
+        attr.logical_name,
+        kept_suffixes=("_formatted_value",),
+        context_table=attr.logical_name,
+    )
+
+    return RuntimeFieldSpec(
+        logical_name=attr.logical_name,
+        pg_type="numeric",
+        pg_type_args={"precision": precision, "scale": scale},
+        description=extract_label(attr.description),
+        display_name=extract_label(attr.display_name),
+        is_custom_attribute=attr.is_custom_attribute,
+        attribute_type=attr.attribute_type or "",
+        annotation_columns=annotations,
+    )
+
+
+def _build_annotation_columns(
+    base: str,
+    *,
+    kept_suffixes: tuple[str, ...],
+    context_table: str,
+) -> tuple[RuntimeAnnotationColumn, ...]:
+    columns: list[RuntimeAnnotationColumn] = []
+    for suffix in kept_suffixes:
+        # Verify suffix is in the registry; this catches typos.
+        if suffix not in ODATA_ANNOTATION_COLUMN_SUFFIXES.values():
+            raise PluginConfigurationError(
+                f"Annotation suffix {suffix!r} is not registered in "
+                "ODATA_ANNOTATION_COLUMN_SUFFIXES."
+            )
+        column_name = annotation_column_name(
+            base,
+            suffix,
+            context=f"{context_table}.{base}",
+        )
+        description_template = _ANNOTATION_DESCRIPTIONS[suffix]
+        columns.append(
+            RuntimeAnnotationColumn(
+                column_name=column_name,
+                pg_type="text",
+                description=description_template.format(base=base),
+            )
+        )
+    return tuple(columns)
+
+
+# dlt's TDataType literal (text, double, bool, timestamp, bigint, binary, json,
+# decimal, wei, date, time) does not include "uuid", "numeric", or "bigint[]"
+# today. We keep the richer pg_type on RuntimeFieldSpec so non-dlt consumers
+# (LLM tool descriptions, sibling exporters, postgres-native upgrades) retain
+# the fidelity. When dlt adds native uuid / numeric / array support, delete
+# the corresponding entries from this mapping and pass pg_type through
+# unchanged.
+# Tracked: dlt's TDataType definition lives at
+# dlt/common/data_types/typing.py (Literal); follow upstream support there.
+_DLT_DATA_TYPE_OVERRIDES: dict[str, str] = {
+    "uuid": "text",
+    "numeric": "decimal",
+    "bigint[]": "json",
+}
+
+
+def _to_dlt_data_type(pg_type: str) -> str:
+    return _DLT_DATA_TYPE_OVERRIDES.get(pg_type, pg_type)
+
+
+def build_dlt_column_hints(
+    fields: tuple[RuntimeFieldSpec, ...],
+    *,
+    sync_mode_is_delta: bool,
+) -> dict[str, dict[str, Any]]:
+    """Build the dlt `columns={col: {...}}` hint for `@ctx_dlt_resource`.
+
+    Description policy: `description ?? display_name`. If both are absent,
+    `description` is omitted from the hint entirely.
+    """
+    hints: dict[str, dict[str, Any]] = {}
+    for field in fields:
+        entry: dict[str, Any] = {
+            "data_type": _to_dlt_data_type(field.pg_type),
+            **field.pg_type_args,
+        }
+        column_doc = field.description or field.display_name
+        if column_doc:
+            entry["description"] = column_doc
+        hints[field.logical_name] = entry
+
+        for ann in field.annotation_columns:
+            ann_entry: dict[str, Any] = {"data_type": _to_dlt_data_type(ann.pg_type)}
+            if ann.description:
+                ann_entry["description"] = ann.description
+            hints[ann.column_name] = ann_entry
+
+    if sync_mode_is_delta:
+        hints["is_deleted"] = {"hard_delete": True}
+
+    return hints

shared_plugins/models.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Annotated, Any, Optional, TypeVar, cast
+
+from pydantic import (
+    AwareDatetime,
+    BaseModel,
+    ConfigDict,
+    Field,
+    StringConstraints,
+    ValidationError,
+    create_model,
+)
+
+IdStr = Annotated[str, StringConstraints(min_length=1, strict=True)]
+NonNegativeInt = Annotated[int, Field(ge=0, strict=True)]
+
+ModelT = TypeVar("ModelT", bound=BaseModel)
+
+
+def format_validation_error(error: ValidationError) -> str:
+    parts: list[str] = []
+    for issue in error.errors():
+        location = ".".join(str(part) for part in issue["loc"]) or "root"
+        parts.append(f"{location}: {issue['msg']}")
+    return "; ".join(parts)
+
+
+class StrictModel(BaseModel):
+    """Shared strict model base: unknown fields are always rejected."""
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+
+class IngressModel(StrictModel):
+    """Shared plugin ingress contract base."""
+
+
+class CtxModel(StrictModel):
+    """Shared plugin row metadata contract."""
+
+    ctx_binding_id: str = Field(min_length=1, alias="_ctx_binding_id")
+    ctx_source_updated_at: AwareDatetime | None = Field(
+        default=None,
+        alias="_ctx_source_updated_at",
+    )
+
+
+def partialize(model_cls: type[ModelT], *, name: str | None = None) -> type[ModelT]:
+    """Return a cached partial model class with all inherited fields optional."""
+
+    partial_name = name or f"{model_cls.__name__}Partial"
+    return cast(type[ModelT], _partialize_cached(model_cls, partial_name))
+
+
+def _to_optional_annotation(annotation: Any) -> Any:
+    try:
+        return annotation | None
+    except TypeError:
+        return Optional[annotation]  # type: ignore[valid-type]
+
+
+@lru_cache(maxsize=None)
+def _partialize_cached(
+    model_cls: type[BaseModel], partial_name: str
+) -> type[BaseModel]:
+    partial_fields: dict[str, tuple[Any, Any]] = {}
+
+    for field_name, field_info in model_cls.model_fields.items():
+        field_dict = field_info.asdict()
+        field_attributes = dict(field_dict["attributes"])
+        field_attributes.pop("default", None)
+        field_attributes.pop("default_factory", None)
+
+        optional_annotation = _to_optional_annotation(field_dict["annotation"])
+        annotated_field = Annotated[
+            (
+                optional_annotation,
+                *field_dict["metadata"],
+                Field(**field_attributes),
+            )
+        ]
+        partial_fields[field_name] = (annotated_field, None)
+
+    return create_model(
+        partial_name,
+        __base__=model_cls,
+        __module__=model_cls.__module__,
+        **partial_fields,
+    )

shared_plugins/naming.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Mapping
+from functools import lru_cache
+from pathlib import Path
+
+
+def _slugify(value: str) -> str:
+    return re.sub(r"[^a-zA-Z0-9]+", "_", value).strip("_").lower() or "value"
+
+
+@lru_cache(maxsize=None)
+def _plugin_id_from_dir(plugin_dir: str) -> str:
+    manifest_path = Path(plugin_dir) / "plugin.json"
+    with manifest_path.open("r", encoding="utf-8") as manifest_file:
+        manifest = json.load(manifest_file)
+    return str(manifest["plugin_id"]).strip()
+
+
+def plugin_id_from_module(module_file: str) -> str:
+    module_path = Path(module_file).resolve()
+    current = module_path.parent
+
+    while True:
+        if (current / "plugin.json").exists():
+            return _plugin_id_from_dir(str(current))
+        if current == current.parent:
+            break
+        current = current.parent
+
+    raise FileNotFoundError(
+        f"Could not resolve plugin.json for module '{module_file}'."
+    )
+
+
+def dlt_dataset_name(plugin_id: str) -> str:
+    return plugin_id.strip()
+
+
+def dagster_asset_group_name(plugin_id: str) -> str:
+    return plugin_id.strip()
+
+
+def dagster_partition_def_name(plugin_id: str) -> str:
+    return f"{plugin_id.strip()}_bindings"
+
+
+def dagster_pool_name(plugin_id: str) -> str:
+    return f"{plugin_id.strip()}_pool"
+
+
+def dlt_source_name(plugin_id: str, job: str) -> str:
+    return f"{_slugify(plugin_id)}_{_slugify(job)}"
+
+
+def dlt_resource_name(entity: str) -> str:
+    return entity.strip()
+
+
+def dlt_pipeline_name(plugin_id: str, binding_id: str, job: str) -> str:
+    return f"{_slugify(plugin_id)}__{_slugify(binding_id)}__{_slugify(job)}"
+
+
+def dagster_dlt_asset_key(source_name: str, entity: str) -> str:
+    """Default asset key produced by DagsterDltTranslator for a dlt resource."""
+    return f"dlt_{source_name}_{entity}"
+
+
+def dagster_airbyte_sync_asset_key(plugin_id: str) -> str:
+    return f"ab_{_slugify(plugin_id)}_sync"
+
+
+def dagster_asset_tags(
+    plugin_id: str,
+    *,
+    extra_tags: Mapping[str, str] | None = None,
+) -> dict[str, str]:
+    tags = {"plugin_id": plugin_id.strip()}
+    if extra_tags:
+        tags.update(dict(extra_tags))
+    return tags