contextbase-shared-plugins 0.0.0a1__py3-none-any.whl

shared_plugins/machine_token.py

@@ -0,0 +1,419 @@
+from __future__ import annotations
+
+import base64
+import fcntl
+import json
+import os
+import threading
+import typing
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import httpx
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    ValidationError,
+    field_validator,
+    model_validator,
+)
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from shared_plugins.env import (
+    CTXB_SCRATCH_DIR_ENV_VAR,
+    normalize_optional_absolute_path,
+)
+
+# The platform machine identity (the Python parallel of the TS
+# `shared/lib/ctx-machine-token.ts`): a confidential OAuth client that mints
+# short-lived `aud=ctx` machine tokens against web's OAuth2 token endpoint with
+# its long-lived client credentials. There is no refresh token — "re-mint" is
+# simply re-authenticating with the client secret. This module is the ONE
+# audited copy of the mint + cache; both the provider-token redemption
+# (`provider_token.py`) and the base-platform SDK auth consume it.
+#
+# The cache is PER-DEPLOYMENT, not per-process: Dagster runs this code in a
+# swarm of short-lived processes (sensor evaluations, run workers, step
+# workers, recycled code servers), and a resource-scoped in-memory cache lives
+# for one tick or one step — so without shared state every tick and step minted
+# its own token, and a first-boot burst rate-limited web's token endpoint
+# (2026-07-08, codecreators v3: better-auth 429s starved every sync until the
+# processes went quiet). The shared JSON file under the ctxb scratch dir turns
+# the whole deployment into one token client (~1 mint/hour), with the flock
+# giving cross-process single-flight and the recorded backoff making every
+# process fail fast — without re-contacting the endpoint — after a 429.
+#
+# Host-side only: the machine secret and the minted token never enter the
+# synthesis guest, where a prompt-injected synthesizer could exfiltrate them.
+# The scratch dir is never mounted into guests and the state file is 0600.
+
+CTX_WEB_URL_ENV_VAR = "CTX_WEB_URL"
+CTX_MACHINE_CLIENT_ID_ENV_VAR = "CTX_MACHINE_CLIENT_ID"
+CTX_MACHINE_CLIENT_SECRET_ENV_VAR = "CTX_MACHINE_CLIENT_SECRET"
+
+# Re-mint this long before the stated expiry so a request never rides a token that
+# lapses in flight (clock skew + the round-trip to the RS). The machine token's TTL
+# is ~1h, so a minute of headroom costs nothing.
+_EXPIRY_SKEW = timedelta(seconds=60)
+
+# After a 429 with no parseable Retry-After, keep the whole deployment off the
+# endpoint this long. better-auth's limiter only resets after an idle gap, so
+# the backoff must comfortably exceed that gap for the window to clear.
+_BACKOFF_FALLBACK = timedelta(seconds=60)
+# The longest backoff any Retry-After is allowed to impose (see _retry_after).
+_BACKOFF_CAP = timedelta(minutes=15)
+
+_SHARED_STATE_FILENAME = "machine-token.json"
+_SHARED_LOCK_FILENAME = "machine-token.lock"
+
+
+def raise_on_error(response: httpx.Response, *, context: str) -> None:
+    """Fail loud on an HTTP error, surfacing the response body. The error
+    payload (e.g. ``{"error":"invalid_client"}`` when the machine creds are
+    wrong or the azp is not allowlisted) is the one field that says WHY — which
+    httpx's bare ``raise_for_status`` discards. Mirrors the TS mint's error path
+    (``shared/lib/ctx-machine-token.ts``)."""
+    if response.is_error:
+        detail = response.text.strip()
+        raise RuntimeError(
+            f"{context}: {response.status_code} {response.reason_phrase}"
+            + (f" — {detail}" if detail else "")
+        )
+
+
+class _MintResponse(BaseModel):
+    """The standard OAuth2 token response, parsed strictly so a malformed or opaque
+    (non-`expires_in`) response fails loud at the mint, not later as a stale token."""
+
+    access_token: str = Field(min_length=1)
+    expires_in: float = Field(gt=0)
+
+
+class _SharedTokenState(BaseModel):
+    """The on-disk shape of the deployment-shared token state: a token with its
+    absolute expiry, or a 429 backoff deadline — exactly one of the two, enforced
+    below so any other shape parses as corrupt and gets rebuilt by the next mint."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    access_token: str | None = Field(default=None, min_length=1)
+    expires_at: datetime | None = None
+    backoff_until: datetime | None = None
+
+    @field_validator("expires_at", "backoff_until")
+    @classmethod
+    def _require_tz_aware(cls, value: datetime | None) -> datetime | None:
+        # A naive datetime would raise TypeError at every comparison — and since
+        # the file persists, that would crash-loop every process of the
+        # deployment. Rejecting it here routes any such state through the
+        # corrupt-file path (discard and rebuild by minting).
+        if value is not None and (value.tzinfo is None or value.utcoffset() is None):
+            raise ValueError("shared token state timestamps must be timezone-aware")
+        return value
+
+    @model_validator(mode="after")
+    def _require_exactly_one_state(self) -> _SharedTokenState:
+        has_token = self.access_token is not None
+        if has_token != (self.expires_at is not None):
+            raise ValueError("a shared token must carry its expiry (and vice versa)")
+        if has_token == (self.backoff_until is not None):
+            raise ValueError(
+                "shared token state must be exactly one of token-with-expiry or backoff"
+            )
+        return self
+
+
+class MachineTokenMinter:
+    """Mints an `aud=ctx` machine token via `client_credentials` and caches it
+    until just before expiry. With a `shared_cache_dir`, the cache (and any 429
+    backoff) lives in a flock-guarded file shared by every process of the
+    deployment — cross-process single-flight, one mint per token lifetime.
+    Without one, the cache is instance-memory only (tests, ad-hoc callers).
+    `invalidate()` drops this instance's token and, if the shared state still
+    holds that same failed token, clears it too — so the next `token()` either
+    adopts a replacement another process already minted or re-mints. The hook
+    for callers that observe a downstream 401 (a lapsed/revoked token)."""
+
+    def __init__(
+        self,
+        *,
+        web_url: str,
+        client_id: str,
+        client_secret: str,
+        http_client: httpx.Client | None = None,
+        shared_cache_dir: Path | None = None,
+    ) -> None:
+        self._token_url = f"{web_url}/api/auth/oauth2/token"
+        basic = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode()
+        self._basic_auth = f"Basic {basic}"
+        self._http = http_client or httpx.Client(timeout=30.0)
+        self._lock = threading.Lock()
+        self._cached: tuple[str, datetime] | None = None
+        self._shared_cache_dir = shared_cache_dir
+
+    def token(self) -> str:
+        now = datetime.now(timezone.utc)
+        with self._lock:
+            cached = self._cached
+            if cached is not None and now < cached[1] - _EXPIRY_SKEW:
+                return cached[0]
+            cache_dir = self._shared_cache_dir
+            if cache_dir is None:
+                return self._mint(now, None)
+
+            state = self._read_shared_state(cache_dir)
+            token = self._usable_token(state, now)
+            if token is not None:
+                return token
+            self._raise_if_backing_off(state, now)
+
+            with self._shared_flock(cache_dir):
+                # Another process may have minted (or hit a 429) while this one
+                # waited on the lock — re-read before touching the endpoint.
+                state = self._read_shared_state(cache_dir)
+                token = self._usable_token(state, now)
+                if token is not None:
+                    return token
+                self._raise_if_backing_off(state, now)
+                return self._mint(now, cache_dir)
+
+    def invalidate(self) -> None:
+        with self._lock:
+            failed = self._cached
+            self._cached = None
+            cache_dir = self._shared_cache_dir
+            if cache_dir is None:
+                return
+            with self._shared_flock(cache_dir):
+                # Clear the shared state only if it still holds the token that
+                # just failed. After a revocation every process observes its own
+                # 401; unconditional deletion would let each of them destroy the
+                # fresh token the first one already re-minted (a thundering herd
+                # of mints against a rate-limited endpoint). If the file moved
+                # on, dropping this instance's memory is enough — the next
+                # token() serves the newer shared token.
+                state = self._read_shared_state(cache_dir)
+                if state is not None and (
+                    failed is None or state.access_token != failed[0]
+                ):
+                    return
+                state_path = cache_dir / _SHARED_STATE_FILENAME
+                if state_path.exists():
+                    state_path.unlink()
+
+    def _mint(self, now: datetime, cache_dir: Path | None) -> str:
+        response = self._http.post(
+            self._token_url,
+            data={"grant_type": "client_credentials", "resource": "ctx"},
+            headers={"authorization": self._basic_auth},
+        )
+        if (
+            response.status_code == httpx.codes.TOO_MANY_REQUESTS
+            and cache_dir is not None
+        ):
+            self._write_shared_state(
+                cache_dir,
+                _SharedTokenState(backoff_until=now + self._retry_after(response)),
+            )
+        raise_on_error(response, context="Failed to mint the ctx machine token")
+        parsed = _MintResponse.model_validate(response.json())
+        expires_at = now + timedelta(seconds=parsed.expires_in)
+        self._cached = (parsed.access_token, expires_at)
+        if cache_dir is not None:
+            self._write_shared_state(
+                cache_dir,
+                _SharedTokenState(
+                    access_token=parsed.access_token, expires_at=expires_at
+                ),
+            )
+        return parsed.access_token
+
+    def _usable_token(
+        self, state: _SharedTokenState | None, now: datetime
+    ) -> str | None:
+        if state is None or state.access_token is None or state.expires_at is None:
+            return None
+        if now >= state.expires_at - _EXPIRY_SKEW:
+            return None
+        self._cached = (state.access_token, state.expires_at)
+        return state.access_token
+
+    def _raise_if_backing_off(
+        self, state: _SharedTokenState | None, now: datetime
+    ) -> None:
+        if state is None or state.backoff_until is None or now >= state.backoff_until:
+            return
+        raise RuntimeError(
+            "The ctx machine-token mint is backing off after a 429 until "
+            f"{state.backoff_until.isoformat()} (shared across this deployment's "
+            "processes); not contacting the endpoint."
+        )
+
+    @staticmethod
+    def _retry_after(response: httpx.Response) -> timedelta:
+        header = response.headers.get("retry-after", "").strip()
+        if not header.isdigit():
+            return _BACKOFF_FALLBACK
+        # Capped: an absurd Retry-After (buggy or hostile server) must not
+        # write a backoff that keeps every process off the endpoint for days —
+        # the shared state would need hand-deletion to recover.
+        return min(timedelta(seconds=int(header)), _BACKOFF_CAP)
+
+    @staticmethod
+    def _read_shared_state(cache_dir: Path) -> _SharedTokenState | None:
+        state_path = cache_dir / _SHARED_STATE_FILENAME
+        if not state_path.exists():
+            return None
+        try:
+            return _SharedTokenState.model_validate_json(state_path.read_text())
+        except (ValidationError, json.JSONDecodeError, UnicodeDecodeError):
+            # The file is a CACHE of state whose source of truth is the mint
+            # endpoint — a corrupt file is rebuilt by the next mint (which
+            # overwrites it atomically), not a reason to crash-loop every
+            # process of the deployment. Not a swallowed error: the unreadable
+            # state is discarded and replaced, never acted on.
+            return None
+
+    @staticmethod
+    def _write_shared_state(cache_dir: Path, state: _SharedTokenState) -> None:
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        tmp_path = cache_dir / f"{_SHARED_STATE_FILENAME}.{os.getpid()}.tmp"
+        fd = os.open(tmp_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+        try:
+            os.write(fd, state.model_dump_json().encode())
+        finally:
+            os.close(fd)
+        os.replace(tmp_path, cache_dir / _SHARED_STATE_FILENAME)
+
+    @staticmethod
+    @contextmanager
+    def _shared_flock(cache_dir: Path) -> Iterator[None]:
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        fd = os.open(cache_dir / _SHARED_LOCK_FILENAME, os.O_CREAT | os.O_RDWR, 0o600)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            yield
+        finally:
+            fcntl.flock(fd, fcntl.LOCK_UN)
+            os.close(fd)
+
+
+class CtxMachineTokenAuth(httpx.Auth):
+    """Per-request Bearer injection for the platform APIs: stamps the minted
+    machine token on every request; on a 401 it invalidates the failed token and
+    retries once with a fresh one — re-minted, or adopted from the deployment's
+    shared cache when another process already replaced it. The minter must NOT
+    share an httpx client that carries this auth — the mint POST would recurse
+    through its own Bearer injection."""
+
+    def __init__(self, minter: MachineTokenMinter) -> None:
+        self._minter = minter
+
+    def auth_flow(
+        self, request: httpx.Request
+    ) -> typing.Generator[httpx.Request, httpx.Response, None]:
+        request.headers["authorization"] = f"Bearer {self._minter.token()}"
+        response = yield request
+        if response.status_code == httpx.codes.UNAUTHORIZED:
+            self._minter.invalidate()
+            request.headers["authorization"] = f"Bearer {self._minter.token()}"
+            yield request
+
+
+class _MachineAuthEnv(BaseSettings):
+    """The machine-cred presence probe: all fields optional, blank == absent (the
+    switch semantics live in `resolve_machine_token_auth`, not here). Contrast
+    `ProviderTokenSettings`, which requires every one of its vars — provider-token
+    redemption is always platform-managed, while the base-platform SDK also runs
+    tokenless in local dev."""
+
+    model_config = SettingsConfigDict(extra="ignore")
+
+    ctx_web_url: str | None = Field(default=None, alias=CTX_WEB_URL_ENV_VAR)
+    ctx_machine_client_id: str | None = Field(
+        default=None, alias=CTX_MACHINE_CLIENT_ID_ENV_VAR
+    )
+    ctx_machine_client_secret: str | None = Field(
+        default=None, alias=CTX_MACHINE_CLIENT_SECRET_ENV_VAR
+    )
+    ctx_scratch_dir: Path | None = Field(default=None, alias=CTXB_SCRATCH_DIR_ENV_VAR)
+
+    @field_validator(
+        "ctx_web_url",
+        "ctx_machine_client_id",
+        "ctx_machine_client_secret",
+        mode="before",
+    )
+    @classmethod
+    def _blank_is_absent(cls, value: object) -> object:
+        if isinstance(value, str) and not value.strip():
+            return None
+        return value.strip() if isinstance(value, str) else value
+
+    @field_validator("ctx_scratch_dir", mode="before")
+    @classmethod
+    def _validate_scratch_dir(cls, value: object) -> Path | None:
+        # Blank == absent, like the creds; a non-blank value goes through the
+        # one canonical CTXB_SCRATCH_DIR normalization (absolute, resolved) so
+        # this probe cannot accept a scratch path env.py would reject.
+        if isinstance(value, str) and not value.strip():
+            return None
+        return normalize_optional_absolute_path(
+            value, env_var_name=CTXB_SCRATCH_DIR_ENV_VAR
+        )
+
+    @field_validator("ctx_web_url")
+    @classmethod
+    def _validate_web_url(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        if not value.startswith(("http://", "https://")):
+            raise ValueError(f"{CTX_WEB_URL_ENV_VAR} must use http:// or https://.")
+        return value.rstrip("/")
+
+
+def resolve_machine_token_auth() -> CtxMachineTokenAuth | None:
+    """Select the platform-managed machine backing. Presence of the machine creds
+    is the switch (the twin of TS `resolveMachineTokenProvider`, and the same
+    pattern as the RS's `CTX_AUTH_SUB_ID`): creds present + a web origin + the
+    scratch dir for the deployment-shared token cache -> a `client_credentials`
+    Bearer auth; no creds -> None (no bearer; local-dev authorizes via the
+    `CTX_AUTH_SUB_ID` RS bypass, which needs no token). Partial config is a
+    deployment mistake, not a half-on mode -> fail loud."""
+    env = _MachineAuthEnv()
+
+    if env.ctx_machine_client_id is None and env.ctx_machine_client_secret is None:
+        return None
+
+    if (
+        env.ctx_machine_client_id is None
+        or env.ctx_machine_client_secret is None
+        or env.ctx_web_url is None
+    ):
+        raise RuntimeError(
+            f"Incomplete machine-auth config: set {CTX_MACHINE_CLIENT_ID_ENV_VAR}, "
+            f"{CTX_MACHINE_CLIENT_SECRET_ENV_VAR}, and {CTX_WEB_URL_ENV_VAR} "
+            "together (or none)."
+        )
+
+    if env.ctx_scratch_dir is None:
+        # Machine mode without the shared scratch dir would silently regress to
+        # the per-process cache that rate-limited web — same posture as the
+        # partial-creds guard above.
+        raise RuntimeError(
+            f"{CTXB_SCRATCH_DIR_ENV_VAR} is not set; machine-token minting needs "
+            "the shared scratch dir for its cross-process token cache (run via "
+            "ctxb so scratch paths are configured)."
+        )
+
+    return CtxMachineTokenAuth(
+        MachineTokenMinter(
+            web_url=env.ctx_web_url,
+            client_id=env.ctx_machine_client_id,
+            client_secret=env.ctx_machine_client_secret,
+            shared_cache_dir=env.ctx_scratch_dir,
+        )
+    )

shared_plugins/microsoft_dataverse/__init__.py

@@ -0,0 +1,27 @@
+"""Shared Microsoft Dataverse client + dlt source factory."""
+
+from shared_plugins.microsoft_dataverse.auth import ClientSecretTokenProvider
+from shared_plugins.microsoft_dataverse.binding_config import (
+    DataverseBindingConfigBase,
+)
+from shared_plugins.microsoft_dataverse.client import (
+    DataverseClient,
+    DataverseRetryPolicy,
+)
+from shared_plugins.microsoft_dataverse.ctx import DataverseRowBase
+from shared_plugins.microsoft_dataverse.source import build_dataverse_dlt_source
+from shared_plugins.microsoft_dataverse.tables import (
+    DataverseSyncMode,
+    DataverseTableSpec,
+)
+
+__all__ = (
+    "ClientSecretTokenProvider",
+    "DataverseBindingConfigBase",
+    "DataverseClient",
+    "DataverseRetryPolicy",
+    "DataverseRowBase",
+    "DataverseSyncMode",
+    "DataverseTableSpec",
+    "build_dataverse_dlt_source",
+)

shared_plugins/microsoft_dataverse/annotations.py

@@ -0,0 +1,61 @@
+"""Centralized policy for Dataverse OData response annotations.
+
+Three registries:
+
+- ODATA_ANNOTATION_COLUMN_SUFFIXES: annotations we keep, with the
+  postgres column-name suffix to use.
+- DROPPED_ODATA_ANNOTATIONS: annotations we have triaged and chosen
+  to drop, with the reason in an inline comment.
+- DROPPED_RESPONSE_FIELDS: plain (non-annotation) fields Dataverse
+  returns unsolicited that the runtime schema does not model, triaged
+  and dropped with the reason in an inline comment.
+
+Postgres identifier limit: column names cap at 63 bytes. With the
+verbose suffixes below, "<attribute>_lookup_logical_name" can exceed
+the cap for long attribute names (e.g.
+_msdyn_resourceassignmentcomputedrequirement_value would land at
+~70 chars). The runtime ingress flow MUST validate this at
+metadata-fetch time and raise loudly on overflow — silent truncation
+risks collisions and lost data, neither acceptable.
+
+Unknown annotations: any annotation present in a response that is in
+NEITHER registry raises a loud error from the translator. New
+Microsoft annotations must be triaged into one of these lists
+explicitly. We do not silently drop or include unknown annotations.
+"""
+
+from __future__ import annotations
+
+ODATA_ANNOTATION_COLUMN_SUFFIXES: dict[str, str] = {
+    "@OData.Community.Display.V1.FormattedValue": "_formatted_value",
+    "@Microsoft.Dynamics.CRM.lookuplogicalname": "_lookup_logical_name",
+}
+
+DROPPED_ODATA_ANNOTATIONS: frozenset[str] = frozenset(
+    {
+        # Pure OData $expand traversal name (e.g. "msdyn_Project" for
+        # _msdyn_project_value). No postgres-query use case — agents query
+        # by the GUID + label, not the navigation property.
+        "@Microsoft.Dynamics.CRM.associatednavigationproperty",
+    }
+)
+
+# Plain field names Dataverse returns in a response even though we did not
+# $select them and the runtime schema (built from EntityDefinitions metadata)
+# does not model them. Unlike unknown annotations — which raise — these are
+# triaged and dropped with a reason. Any plain field NOT listed here still
+# reaches the per-table model's extra="forbid" and fails loudly, so genuine
+# schema gaps continue to surface.
+DROPPED_RESPONSE_FIELDS: frozenset[str] = frozenset(
+    {
+        # Memo sub-attribute (AttributeOf="description") carrying a sanitized
+        # copy of `description`. Dataverse emits it alongside `description`;
+        # the schema builder excludes AttributeOf children, so it is unmodeled.
+        # We keep the real `description`; the sanitized dup is redundant.
+        "safedescription",
+        # systemuser returns a plain `ownerid` GUID that is absent from the
+        # entity's attribute metadata, so it cannot be typed or modeled. The
+        # owner is exposed elsewhere as the modeled `_ownerid_value` lookup.
+        "ownerid",
+    }
+)

shared_plugins/microsoft_dataverse/auth.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from azure.identity import ClientSecretCredential
+
+
+class ClientSecretTokenProvider:
+    def __init__(
+        self,
+        *,
+        tenant_id: str,
+        client_id: str,
+        client_secret: str,
+        scope: str,
+    ) -> None:
+        self._scope = scope
+        self._credential = ClientSecretCredential(
+            tenant_id=tenant_id,
+            client_id=client_id,
+            client_secret=client_secret,
+        )
+
+    def __call__(self) -> str:
+        return self._credential.get_token(self._scope).token
+
+    def close(self) -> None:
+        self._credential.close()

shared_plugins/microsoft_dataverse/binding_config.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from pydantic import Field, field_validator
+from shared_plugins.bindings import BaseBindingConfigModel, NonEmptyText
+
+
+class DataverseBindingConfigBase(BaseBindingConfigModel):
+    """Base for any plugin syncing from a Microsoft Dataverse instance.
+
+    Plugins extend this and add plugin-specific fields if any. tenant_id and
+    org_url are the only fields required for Dataverse Web API access.
+
+    Credentials (client_id / client_secret) come through
+    BindingAuth_ClientCredentials via shared_plugins.bindings.require_client_credentials,
+    NOT through BindingConfig — operator-level secrets stay out of binding.config
+    per repo policy.
+    """
+
+    tenant_id: NonEmptyText = Field(
+        description="Microsoft Entra tenant id used for Dataverse client credentials.",
+    )
+    org_url: NonEmptyText = Field(
+        description=(
+            "Dataverse organization URL, for example "
+            "https://org1c9f9fa0.crm3.dynamics.com."
+        ),
+    )
+
+    @field_validator("org_url")
+    @classmethod
+    def _normalize_org_url(cls, value: str) -> str:
+        normalized = value.rstrip("/")
+        if not normalized.startswith("https://"):
+            raise ValueError("Dataverse org_url must start with https://.")
+        return normalized