contextbase-shared-plugins 0.0.0a1__py3-none-any.whl

shared_plugins/microsoft_dataverse/source.py

@@ -0,0 +1,299 @@
+"""Dlt source factory for any Dataverse-backed plugin.
+
+Plugin asset bodies call `build_dataverse_dlt_source(plugin_id, job, ...)`
+to produce a `@dlt.source`-wrapped tuple of resources. One resource per
+active table (delta or snapshot mode). The plugin's component.py is
+agnostic to OData/Pydantic/runtime-schema concerns -- those all live here.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Iterator, Mapping
+from typing import Any
+
+import dlt
+from shared_plugins.bindings import ResolvedBindingModels, iter_active_model_rows
+from shared_plugins.naming import dlt_resource_name, dlt_source_name
+from shared_plugins.resources import ctx_dlt_resource
+
+from .client import DataverseClient
+from .ctx import DataverseRowBase
+from .runtime_schema import RuntimeDataverseTableSchema, build_runtime_table_schemas
+from .tables import DataverseSyncMode, DataverseTableSpec
+from .translators import dataverse_record_to_row
+
+LOGGER = logging.getLogger(__name__)
+# State key for the per-table change-tracking cursor. The stored value is a
+# Dataverse change-tracking link: a mid-enumeration @odata.nextLink ($skiptoken)
+# while an initial drain is still in flight, or a terminal @odata.deltaLink
+# ($deltatoken) once it completes. The literal key stays "delta_link" so
+# bindings already carrying a deltatoken under that key keep resuming instead of
+# re-draining from scratch.
+CHANGE_TRACKING_CURSOR_KEY = "delta_link"
+# Upper bound on rows drained from a single table per pipeline run. dlt buffers
+# each run's extract+normalize+load, and the memory peak is during normalize,
+# driven by *wide* rows (e.g. emails with full HTML bodies) more than by row
+# count. An unbounded initial drain OOM-killed an 8 GB host at ~6 GB; a 30k cap
+# did NOT help, because the byte-heavy tables hold only a few thousand rows (well
+# under 30k) so it never chunked them. cap=2000 ran comfortably under budget.
+# A later raise to 5000 (to backfill activityparties in fewer runs) let the
+# content tables emails (~5.5k) and crff1_emaillogses (~5.1k) drain in roughly
+# one run each and OOM-killed a fresh 8 GB deployment on every cycle: the step
+# peaked at ~5.8 GB in normalize, the kill landed before the load committed, so
+# the $skiptoken never persisted and each */15 run re-pulled the same rows and
+# re-died (codecreators V-6, 2026-07-09). 2000 is the empirically proven value
+# for 8 GB hosts — do NOT raise it to speed up backfills; the cap only sets how
+# many runs a drain takes, never what data ultimately lands. When the cap is
+# reached mid-enumeration the resource persists the change-tracking $skiptoken
+# and returns; the next run resumes from it and the `merge` disposition upserts
+# the chunks together, so a table backfills across several scheduled runs before
+# settling into incremental ($deltatoken) syncs.
+# Operator-level tuning knob, hence a module constant rather than a BindingConfig
+# field; size it against real-host memory, not row count.
+MAX_DRAIN_ROWS_PER_RUN = 2000
+SNAPSHOT_WRITE_DISPOSITION = {
+    "disposition": "merge",
+    "strategy": "delete-insert",
+}
+SNAPSHOT_MERGE_KEY = ("_ctx_binding_id",)
+
+
+def build_dataverse_dlt_source(
+    *,
+    plugin_id: str,
+    job: str,
+    binding_id: str,
+    client: DataverseClient,
+    binding_models: ResolvedBindingModels,
+    specs: tuple[DataverseTableSpec, ...],
+) -> Any:
+    active_specs = tuple(
+        spec for spec in specs if spec.resource_name in binding_models.active
+    )
+    runtime_schemas: Mapping[str, RuntimeDataverseTableSchema] = (
+        build_runtime_table_schemas(client=client, specs=active_specs)
+    )
+
+    @dlt.source(name=dlt_source_name(plugin_id, job))
+    def _source() -> tuple[Any, ...]:
+        resources: list[Any] = []
+        for spec in active_specs:
+            schema = runtime_schemas[spec.entity_set]
+            if spec.sync_mode is DataverseSyncMode.DELTA:
+                resources.append(
+                    _build_delta_resource(
+                        binding_id=binding_id,
+                        client=client,
+                        schema=schema,
+                        binding_models=binding_models,
+                    )
+                )
+            else:
+                resources.append(
+                    _build_snapshot_resource(
+                        binding_id=binding_id,
+                        client=client,
+                        schema=schema,
+                        binding_models=binding_models,
+                    )
+                )
+
+        if not resources:
+            LOGGER.info(
+                "%s.%s.empty_source binding_id=%s active_models=%s",
+                plugin_id,
+                job,
+                binding_id,
+                tuple(binding_models.active),
+            )
+
+        return tuple(resources)
+
+    return _source()
+
+
+def _build_delta_resource(
+    *,
+    binding_id: str,
+    client: DataverseClient,
+    schema: RuntimeDataverseTableSchema,
+    binding_models: ResolvedBindingModels,
+) -> Any:
+    spec = schema.spec
+
+    @ctx_dlt_resource(
+        name=dlt_resource_name(spec.resource_name),
+        write_disposition="merge",
+        primary_key=("_ctx_binding_id", spec.primary_key),
+        columns=schema.dlt_columns,
+    )
+    def delta_resource() -> Iterator[DataverseRowBase]:
+        state = dlt.current.resource_state()
+        initial_cursor = state.get(CHANGE_TRACKING_CURSOR_KEY)
+        if initial_cursor is not None and not isinstance(initial_cursor, str):
+            raise RuntimeError(
+                f"Stored Dataverse change-tracking cursor for {spec.entity_set} "
+                "is not a string."
+            )
+        next_cursor: dict[str, str] = {}
+        rows = _iter_delta_rows(
+            binding_id=binding_id,
+            client=client,
+            schema=schema,
+            initial_cursor=initial_cursor,
+            next_cursor=next_cursor,
+        )
+        yield from iter_active_model_rows(
+            model_name=spec.resource_name,
+            rows=rows,
+            binding_models=binding_models,
+        )
+        state[CHANGE_TRACKING_CURSOR_KEY] = next_cursor[CHANGE_TRACKING_CURSOR_KEY]
+
+    return delta_resource
+
+
+def _build_snapshot_resource(
+    *,
+    binding_id: str,
+    client: DataverseClient,
+    schema: RuntimeDataverseTableSchema,
+    binding_models: ResolvedBindingModels,
+) -> Any:
+    spec = schema.spec
+
+    @ctx_dlt_resource(
+        name=dlt_resource_name(spec.resource_name),
+        write_disposition=SNAPSHOT_WRITE_DISPOSITION,
+        merge_key=SNAPSHOT_MERGE_KEY,
+        primary_key=("_ctx_binding_id", spec.primary_key),
+        columns=schema.dlt_columns,
+    )
+    def snapshot_resource() -> Iterator[DataverseRowBase]:
+        rows = _iter_snapshot_rows(
+            binding_id=binding_id,
+            client=client,
+            schema=schema,
+        )
+        yield from iter_active_model_rows(
+            model_name=spec.resource_name,
+            rows=rows,
+            binding_models=binding_models,
+        )
+
+    return snapshot_resource
+
+
+def _iter_delta_rows(
+    *,
+    binding_id: str,
+    client: DataverseClient,
+    schema: RuntimeDataverseTableSchema,
+    initial_cursor: str | None,
+    next_cursor: dict[str, str],
+) -> Iterator[DataverseRowBase]:
+    spec = schema.spec
+    # An initial drain may already be partway through (resuming from a stored
+    # $skiptoken); only a stored $deltatoken means we are in steady-state
+    # incremental sync. `iter_change_tracking_pages` resumes from either link.
+    incremental = initial_cursor is not None and "$deltatoken" in initial_cursor
+    row_count = 0
+    page_count = 0
+    saved_cursor: str | None = None
+    t0 = time.monotonic()
+
+    LOGGER.info(
+        "dataverse.delta.start entity_set=%s mode=%s",
+        spec.entity_set,
+        "incremental" if incremental else "initial",
+    )
+
+    for page in client.iter_change_tracking_pages(
+        spec,
+        delta_link=initial_cursor,
+        select=schema.select_columns,
+    ):
+        page_count += 1
+        for record in page.value:
+            raw_payload = record.model_dump(by_alias=True, exclude_none=True)
+            row_count += 1
+            yield dataverse_record_to_row(
+                binding_id=binding_id,
+                schema=schema,
+                record=record,
+                raw_payload=raw_payload,
+            )
+
+        if page.delta_link is not None:
+            # Enumeration complete: the deltaLink is the incremental cursor for
+            # the next run.
+            saved_cursor = page.delta_link
+            break
+        if row_count >= MAX_DRAIN_ROWS_PER_RUN:
+            # Per-run cap reached while the initial drain is still in flight.
+            # Persist this page's nextLink ($skiptoken) so the next run resumes
+            # here; merge upserts accumulate the chunks across runs.
+            if page.next_link is None:
+                raise RuntimeError(
+                    f"Dataverse change-tracking page for {spec.entity_set} had "
+                    "neither a nextLink to resume from nor a deltaLink."
+                )
+            saved_cursor = page.next_link
+            break
+
+    if saved_cursor is None:
+        raise RuntimeError(
+            f"Dataverse change-tracking enumeration for {spec.entity_set} ended "
+            "without a deltaLink or a resumable nextLink."
+        )
+    next_cursor[CHANGE_TRACKING_CURSOR_KEY] = saved_cursor
+
+    LOGGER.info(
+        "dataverse.delta.done entity_set=%s mode=%s pages=%d rows=%d "
+        "more_pending=%s elapsed=%.1fs",
+        spec.entity_set,
+        "incremental" if incremental else "initial",
+        page_count,
+        row_count,
+        "$skiptoken" in saved_cursor,
+        time.monotonic() - t0,
+    )
+
+
+def _iter_snapshot_rows(
+    *,
+    binding_id: str,
+    client: DataverseClient,
+    schema: RuntimeDataverseTableSchema,
+) -> Iterator[DataverseRowBase]:
+    spec = schema.spec
+    row_count = 0
+    page_count = 0
+    t0 = time.monotonic()
+
+    LOGGER.info("dataverse.snapshot.start entity_set=%s", spec.entity_set)
+
+    for page in client.iter_snapshot_pages(
+        spec,
+        select=schema.select_columns,
+    ):
+        page_count += 1
+        for record in page.value:
+            raw_payload = record.model_dump(by_alias=True, exclude_none=True)
+            row_count += 1
+            yield dataverse_record_to_row(
+                binding_id=binding_id,
+                schema=schema,
+                record=record,
+                raw_payload=raw_payload,
+            )
+
+    LOGGER.info(
+        "dataverse.snapshot.done entity_set=%s pages=%d rows=%d elapsed=%.1fs",
+        spec.entity_set,
+        page_count,
+        row_count,
+        time.monotonic() - t0,
+    )

shared_plugins/microsoft_dataverse/tables.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class DataverseSyncMode(str, Enum):
+    DELTA = "delta"
+    SNAPSHOT = "snapshot"
+
+
+@dataclass(frozen=True)
+class DataverseTableSpec:
+    """Identifies one Dataverse entity set and how it should be synced.
+
+    Fields:
+    - entity_set: the OData entity-set name (e.g. "msdyn_projects").
+    - primary_key: the Dataverse logical name of the table's PK column
+      (e.g. "msdyn_projectid"). Used as the dlt primary_key trailing
+      element after _ctx_binding_id. Plugin authors look up this value
+      from Dataverse EntityDefinitions.PrimaryIdAttribute and hard-code
+      it in the spec — keeps the spec list self-documenting.
+    - sync_mode: DELTA (Prefer: odata.track-changes) or SNAPSHOT (full pull).
+    - resource_name: the dlt resource name. Conventionally equal to
+      entity_set, but kept distinct so plugins can rename if needed.
+
+    Note: $select is NOT on the spec. Selected columns are computed at
+    warmup from per-tenant attribute metadata (see runtime_schema.py).
+    """
+
+    entity_set: str
+    primary_key: str
+    sync_mode: DataverseSyncMode
+    resource_name: str

shared_plugins/microsoft_dataverse/translators.py

@@ -0,0 +1,133 @@
+"""Per-record translation: Dataverse OData payload → per-table CtxModel instance.
+
+The translator walks the raw OData payload, splits annotation keys
+(`<col>@<annotation>`) per the registry policy, and constructs an instance
+of the per-table runtime CtxModel subclass. Returns a CtxModel — NOT a dict
+— because @ctx_dlt_resource validates emitted rows are CtxModel instances.
+
+Unknown annotations raise PluginConfigurationError. Dropped annotations are
+silently omitted (they're triaged and intentional). Kept annotations land
+on the companion column suffix.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from shared_plugins.exceptions import PluginConfigurationError
+
+from .annotations import (
+    DROPPED_ODATA_ANNOTATIONS,
+    DROPPED_RESPONSE_FIELDS,
+    ODATA_ANNOTATION_COLUMN_SUFFIXES,
+)
+from .ctx import DataverseRowBase
+from .ingress import DataverseRecordIngress, is_deleted_record
+from .runtime_schema import RuntimeDataverseTableSchema
+
+# Keys that DataverseRecordIngress emits via model_dump(by_alias=True): the
+# @odata.* aliases plus the non-aliased tombstone fields (`id`, `reason`).
+# The column-flatten step skips these so only column data (extras under
+# DataverseRecordIngress's extra="allow") reaches the per-table row model.
+# Derived from the ingress model so new envelope fields are picked up
+# automatically without touching the flatten filter.
+_DATAVERSE_ENVELOPE_KEYS: frozenset[str] = frozenset(
+    (field.alias or name) for name, field in DataverseRecordIngress.model_fields.items()
+)
+
+
+def dataverse_record_to_row(
+    *,
+    binding_id: str,
+    schema: RuntimeDataverseTableSchema,
+    record: DataverseRecordIngress,
+    raw_payload: dict[str, Any],
+) -> DataverseRowBase:
+    """Convert one Dataverse record into a per-table CtxModel instance.
+
+    `raw_payload` is the same OData record dict that produced `record` — the
+    strict envelope view (`record`) carries only the validated envelope
+    fields, while the full raw view (`raw_payload`) keeps every key from the
+    source response, including annotation keys (`<col>@<ann>`) that the
+    strict envelope does not preserve.
+
+    Tombstones (delta-stream delete entries) arrive without column data and
+    without the entity's PK column; the deleted row's identifier is on
+    `record.id`. This translator injects it as the PK column so dlt's
+    `hard_delete` hint on `is_deleted` (see build_dlt_column_hints in
+    types.py) issues a DELETE for the matching destination row during merge.
+    """
+    deleted = is_deleted_record(record, schema.spec, payload=raw_payload)
+    ctx_source_updated_at = raw_payload.get("modifiedon")
+
+    base_values: dict[str, Any] = {
+        "_ctx_binding_id": binding_id,
+        "_ctx_source_updated_at": ctx_source_updated_at,
+        "etag": record.odata_etag,
+        "is_deleted": deleted,
+        "delete_reason": record.reason,
+    }
+
+    column_values = _flatten_columns_and_annotations(
+        raw_payload=raw_payload,
+        schema=schema,
+    )
+    base_values.update(column_values)
+
+    if deleted:
+        if not isinstance(record.id, str):
+            raise PluginConfigurationError(
+                f"Dataverse tombstone for entity_set "
+                f"{schema.spec.entity_set!r} has reason={record.reason!r} "
+                "but no 'id' field on the envelope; cannot identify the "
+                "deleted row."
+            )
+        base_values[schema.spec.primary_key] = record.id
+
+    return schema.record_model.model_validate(base_values)
+
+
+def _flatten_columns_and_annotations(
+    *,
+    raw_payload: dict[str, Any],
+    schema: RuntimeDataverseTableSchema,
+) -> dict[str, Any]:
+    """Walk raw payload keys, returning a {column_name: value} dict.
+
+    - Envelope keys (declared on DataverseRecordIngress: @odata.* aliases
+      plus `id` and `reason`) are skipped — those are read from `record`
+      directly by the caller.
+    - Plain keys in DROPPED_RESPONSE_FIELDS (unsolicited, unmodeled fields
+      Dataverse volunteers) are dropped per the triaged registry.
+    - Other plain keys land under their logical name.
+    - Annotation keys (`<col>@<annotation>`) land under
+      `<col><suffix>` per ODATA_ANNOTATION_COLUMN_SUFFIXES, or are dropped
+      per DROPPED_ODATA_ANNOTATIONS, or raise on unknown.
+    """
+    out: dict[str, Any] = {}
+    for key, value in raw_payload.items():
+        if key in _DATAVERSE_ENVELOPE_KEYS:
+            continue
+
+        if "@" not in key:
+            if key in DROPPED_RESPONSE_FIELDS:
+                continue
+            out[key] = value
+            continue
+
+        base, _, annotation_token = key.partition("@")
+        annotation = f"@{annotation_token}"
+        if annotation in DROPPED_ODATA_ANNOTATIONS:
+            continue
+        suffix = ODATA_ANNOTATION_COLUMN_SUFFIXES.get(annotation)
+        if suffix is None:
+            raise PluginConfigurationError(
+                f"Encountered unknown OData annotation {annotation!r} on "
+                f"column {base!r} in entity_set {schema.spec.entity_set!r}. "
+                "Triage by adding it to ODATA_ANNOTATION_COLUMN_SUFFIXES "
+                "(keep) or DROPPED_ODATA_ANNOTATIONS (drop) in "
+                "shared_plugins.microsoft_dataverse.annotations."
+            )
+        out[f"{base}{suffix}"] = value
+
+    return out