contextbase-shared-plugins 0.0.0a1__py3-none-any.whl

shared_plugins/naming.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Mapping
+from functools import lru_cache
+from pathlib import Path
+
+
+def _slugify(value: str) -> str:
+    return re.sub(r"[^a-zA-Z0-9]+", "_", value).strip("_").lower() or "value"
+
+
+@lru_cache(maxsize=None)
+def _plugin_id_from_dir(plugin_dir: str) -> str:
+    manifest_path = Path(plugin_dir) / "plugin.json"
+    with manifest_path.open("r", encoding="utf-8") as manifest_file:
+        manifest = json.load(manifest_file)
+    return str(manifest["plugin_id"]).strip()
+
+
+def plugin_id_from_module(module_file: str) -> str:
+    module_path = Path(module_file).resolve()
+    current = module_path.parent
+
+    while True:
+        if (current / "plugin.json").exists():
+            return _plugin_id_from_dir(str(current))
+        if current == current.parent:
+            break
+        current = current.parent
+
+    raise FileNotFoundError(
+        f"Could not resolve plugin.json for module '{module_file}'."
+    )
+
+
+def dlt_dataset_name(plugin_id: str) -> str:
+    return plugin_id.strip()
+
+
+def dagster_asset_group_name(plugin_id: str) -> str:
+    return plugin_id.strip()
+
+
+def dagster_partition_def_name(plugin_id: str) -> str:
+    return f"{plugin_id.strip()}_bindings"
+
+
+def dagster_pool_name(plugin_id: str) -> str:
+    return f"{plugin_id.strip()}_pool"
+
+
+def dlt_source_name(plugin_id: str, job: str) -> str:
+    return f"{_slugify(plugin_id)}_{_slugify(job)}"
+
+
+def dlt_resource_name(entity: str) -> str:
+    return entity.strip()
+
+
+def dlt_pipeline_name(plugin_id: str, binding_id: str, job: str) -> str:
+    return f"{_slugify(plugin_id)}__{_slugify(binding_id)}__{_slugify(job)}"
+
+
+def dagster_dlt_asset_key(source_name: str, entity: str) -> str:
+    """Default asset key produced by DagsterDltTranslator for a dlt resource."""
+    return f"dlt_{source_name}_{entity}"
+
+
+def dagster_airbyte_sync_asset_key(plugin_id: str) -> str:
+    return f"ab_{_slugify(plugin_id)}_sync"
+
+
+def dagster_asset_tags(
+    plugin_id: str,
+    *,
+    extra_tags: Mapping[str, str] | None = None,
+) -> dict[str, str]:
+    tags = {"plugin_id": plugin_id.strip()}
+    if extra_tags:
+        tags.update(dict(extra_tags))
+    return tags

shared_plugins/pg_column_comments.py

@@ -0,0 +1,59 @@
+"""Monkey-patch PostgresClient to emit COMMENT ON COLUMN for dlt column descriptions.
+
+dlt's ``TColumnSchema`` supports a ``description`` field, but the Postgres destination
+does not act on it.  Databricks and Snowflake do (inline ``COMMENT`` in DDL).  Postgres
+uses a separate ``COMMENT ON COLUMN`` statement, so we hook into
+PostgresClient's ``_build_schema_update_sql`` to append COMMENT statements for the
+same tables dlt is updating — regardless of whether those tables have new columns.
+
+ctxb docs reads ``COMMENT ON COLUMN`` metadata and surfaces it in generated schema docs
+that are injected into the agent context.  This closes the loop:
+
+    dlt schema (description) → COMMENT ON COLUMN → ctxb docs → agent prompt
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, List, Tuple
+
+from dlt.common.data_writers.escape import (
+    escape_postgres_identifier,
+    escape_postgres_literal,
+)
+from dlt.common.schema.typing import (
+    TSchemaTables,
+    TTableSchemaColumns,
+)
+from dlt.destinations.impl.postgres.postgres import PostgresClient
+
+_original_build_schema_update_sql = PostgresClient._build_schema_update_sql
+
+
+def _build_schema_update_sql_with_comments(
+    self: PostgresClient,
+    storage_tables: Iterable[Tuple[str, TTableSchemaColumns]],
+) -> Tuple[List[str], TSchemaTables]:
+    storage_tables = list(storage_tables)
+    sql_updates, schema_update = _original_build_schema_update_sql(self, storage_tables)
+
+    dlt_table_names = set(self.schema.dlt_table_names())
+    for table_name, _storage_columns in storage_tables:
+        if table_name in dlt_table_names:
+            continue
+
+        full_table = self.prepare_load_table(table_name)
+        qualified_table = self.sql_client.make_qualified_table_name(table_name)
+        for col in full_table["columns"].values():
+            description = col.get("description")
+            comment_literal = (
+                escape_postgres_literal(description) if description else "NULL"
+            )
+            col_name = escape_postgres_identifier(col["name"])
+            sql_updates.append(
+                f"COMMENT ON COLUMN {qualified_table}.{col_name} IS {comment_literal}"
+            )
+
+    return sql_updates, schema_update
+
+
+PostgresClient._build_schema_update_sql = _build_schema_update_sql_with_comments  # type: ignore[method-assign]

shared_plugins/provider_token.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import httpx
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from base_client import BindingAuth_AuthenticatedAccount
+from shared_plugins.env import (
+    CTXB_SCRATCH_DIR_ENV_VAR,
+    normalize_optional_absolute_path,
+    normalize_required_string,
+)
+from shared_plugins.machine_token import (
+    CTX_MACHINE_CLIENT_ID_ENV_VAR,
+    CTX_MACHINE_CLIENT_SECRET_ENV_VAR,
+    CTX_WEB_URL_ENV_VAR,
+    MachineTokenMinter,
+    raise_on_error,
+)
+
+# Redeems third-party provider tokens server-side (the Python parallel of web's
+# `lib/ctx-token.ts` redemption endpoint). The daemon presents its `aud=ctx`
+# machine token — minted and cached by `shared_plugins.machine_token`, the one
+# audited copy of the mint — to web's `/provider-token` endpoint, which returns
+# a fresh Google/Slack access token. Host-side only: the machine secret and
+# the minted token never enter the synthesis guest, where a prompt-injected
+# synthesizer could exfiltrate them.
+#
+# One extra hardening the TS side deferred: on a 401 from `/provider-token` the
+# Python caller CAN observe the downstream status, so it invalidates the failed
+# machine token and retries once with a fresh one — re-minted, or adopted from
+# the deployment's shared cache when another process already replaced it (the
+# TS `() => Promise<string>` seam could not).
+
+
+class ProviderTokenSettings(BaseSettings):
+    """Machine-auth config for provider-token redemption — all required, no
+    fallback. CTX_WEB_URL has NO localhost default here (unlike the CLI env): a
+    daemon that redeems provider tokens is always platform-managed, so a missing or
+    blank var is a deploy mistake, not a local-dev mode — fail loud."""
+
+    model_config = SettingsConfigDict(extra="ignore")
+
+    ctx_web_url: str = Field(alias=CTX_WEB_URL_ENV_VAR)
+    ctx_machine_client_id: str = Field(alias=CTX_MACHINE_CLIENT_ID_ENV_VAR)
+    ctx_machine_client_secret: str = Field(alias=CTX_MACHINE_CLIENT_SECRET_ENV_VAR)
+    # The deployment-shared machine-token cache lives here (see machine_token.py);
+    # required for the same reason as the creds — a redeeming daemon without it
+    # would regress to per-process minting.
+    ctx_scratch_dir: Path = Field(alias=CTXB_SCRATCH_DIR_ENV_VAR)
+
+    @field_validator("ctx_web_url", mode="before")
+    @classmethod
+    def _validate_web_url(cls, value: object) -> str:
+        normalized = normalize_required_string(value, env_var_name=CTX_WEB_URL_ENV_VAR)
+        if not normalized.startswith(("http://", "https://")):
+            raise ValueError(f"{CTX_WEB_URL_ENV_VAR} must use http:// or https://.")
+        return normalized.rstrip("/")
+
+    @field_validator("ctx_machine_client_id", mode="before")
+    @classmethod
+    def _validate_client_id(cls, value: object) -> str:
+        return normalize_required_string(
+            value, env_var_name=CTX_MACHINE_CLIENT_ID_ENV_VAR
+        )
+
+    @field_validator("ctx_scratch_dir", mode="before")
+    @classmethod
+    def _validate_scratch_dir(cls, value: object) -> Path:
+        # The one canonical CTXB_SCRATCH_DIR normalization (absolute, resolved).
+        normalized = normalize_optional_absolute_path(
+            value, env_var_name=CTXB_SCRATCH_DIR_ENV_VAR
+        )
+        if normalized is None:
+            raise ValueError(f"{CTXB_SCRATCH_DIR_ENV_VAR} cannot be blank.")
+        return normalized
+
+    @field_validator("ctx_machine_client_secret", mode="before")
+    @classmethod
+    def _validate_client_secret(cls, value: object) -> str:
+        return normalize_required_string(
+            value, env_var_name=CTX_MACHINE_CLIENT_SECRET_ENV_VAR
+        )
+
+
+class ProviderTokenLease(BaseModel):
+    """A redeemed provider access token. `accessTokenExpiresAt` is the
+    provider-DECLARED expiry: ISO-8601 with an offset → parsed to a tz-aware
+    datetime (a naive value would make `_normalize_google_expiry` raise
+    downstream), or None for providers that declare no expiry (notion never
+    sends expires_in). None means "valid until proven otherwise" — the caller's
+    recovery is a `refresh=True` redemption, which web maps to the provider's
+    real refresh grant.
+
+    TS twin: packages/base-schemas/lib/provider-token-lease.ts (the slate-bridge
+    MCP gateway's copy of this contract) — a contract change updates BOTH."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    access_token: str = Field(alias="accessToken", min_length=1)
+    access_token_expires_at: datetime | None = Field(alias="accessTokenExpiresAt")
+
+    @field_validator("access_token_expires_at")
+    @classmethod
+    def _require_tz_aware(cls, value: datetime | None) -> datetime | None:
+        if value is None:
+            return value
+        if value.tzinfo is None or value.utcoffset() is None:
+            raise ValueError("accessTokenExpiresAt must be timezone-aware.")
+        return value
+
+
+class ProviderTokenClient:
+    """Redeems provider tokens through web's `/provider-token`, authenticated by
+    the `aud=ctx` machine token from `MachineTokenMinter`; a 401 from
+    `/provider-token` invalidates the failed token and retries once with a fresh
+    one (re-minted, or adopted from the deployment's shared cache)."""
+
+    def __init__(
+        self,
+        settings: ProviderTokenSettings,
+        *,
+        http_client: httpx.Client | None = None,
+    ) -> None:
+        self._redeem_url = f"{settings.ctx_web_url}/api/auth/provider-token"
+        # One plain (auth-less) client serves both the redeem POSTs and the
+        # minter's token POSTs — headers are stamped explicitly per call.
+        self._http = http_client or httpx.Client(timeout=30.0)
+        self._minter = MachineTokenMinter(
+            web_url=settings.ctx_web_url,
+            client_id=settings.ctx_machine_client_id,
+            client_secret=settings.ctx_machine_client_secret,
+            http_client=self._http,
+            # The same client credentials mint the same aud=ctx token, so the
+            # redemption path shares the deployment's one cached token instead
+            # of minting its own per run/step.
+            shared_cache_dir=settings.ctx_scratch_dir,
+        )
+
+    def redeem(
+        self,
+        *,
+        provider_id: str,
+        account_id: str,
+        refresh: bool = False,
+    ) -> ProviderTokenLease:
+        response = self._post_redeem(provider_id, account_id, refresh)
+        if response.status_code == httpx.codes.UNAUTHORIZED:
+            # The machine token may have lapsed / been revoked RS-side. Invalidate
+            # it and retry once with a fresh token before surfacing the failure.
+            self._minter.invalidate()
+            response = self._post_redeem(provider_id, account_id, refresh)
+        raise_on_error(response, context="Failed to redeem the provider token")
+        return ProviderTokenLease.model_validate(response.json())
+
+    def _post_redeem(
+        self, provider_id: str, account_id: str, refresh: bool
+    ) -> httpx.Response:
+        return self._http.post(
+            self._redeem_url,
+            json={
+                "accountId": account_id,
+                "providerId": provider_id,
+                "refresh": refresh,
+            },
+            headers={"authorization": f"Bearer {self._minter.token()}"},
+        )
+
+
+# Re-redeem this long before the stated expiry so a run that outlives one
+# delegated token (a cold-start mail backfill can outlive a single token) never
+# stamps a bearer that lapses mid-request. Matches the intent of azure.identity's
+# own near-expiry refresh on the client_credentials path.
+_DELEGATED_REFRESH_SKEW = timedelta(minutes=5)
+
+
+class DelegatedTokenProvider:
+    """Caching access-token source for an ``authenticated_account`` binding — the
+    delegated sibling of the plugins' ``ClientSecretCredential`` path. Both yield a
+    bearer per call; this one redeems the binding's linked account server-side
+    through ``/provider-token`` (``ProviderTokenClient``) rather than holding a
+    client secret, so a plugin's sync code stays credential-agnostic (the
+    write-side MCP spec's R7 "one plugin, one auth" — a dual-tolerant seam).
+
+    Caches the redeemed lease and re-redeems within ``_DELEGATED_REFRESH_SKEW`` of
+    expiry, so a run that outlives one token rotates transparently — parity with
+    azure.identity's own get_token caching on the client_credentials path. The
+    first redemption is ``refresh=False`` (web heals a stale stored token
+    transparently); a near-expiry re-redemption is ``refresh=True`` to force real
+    rotation, mirroring the google refresh path.
+    """
+
+    def __init__(
+        self,
+        *,
+        auth: BindingAuth_AuthenticatedAccount,
+        provider_token_client: ProviderTokenClient,
+    ) -> None:
+        self._auth = auth
+        self._client = provider_token_client
+        self._lease: ProviderTokenLease | None = None
+
+    def current_lease(self) -> ProviderTokenLease:
+        """The cached lease — redeemed on first use, re-redeemed within the refresh
+        skew of expiry. The shared ``DelegatedTokenCredential`` reads its expiry to
+        hand azure a real ``AccessToken.expires_on``; Dataverse's ``__call__`` needs
+        only the token."""
+        now = datetime.now(timezone.utc)
+        lease = self._lease
+        if lease is not None and now < _lease_expiry(lease) - _DELEGATED_REFRESH_SKEW:
+            return lease
+
+        lease = self._client.redeem(
+            provider_id=self._auth.provider_id,
+            account_id=self._auth.account_id,
+            refresh=self._lease is not None,
+        )
+        self._lease = lease
+        return lease
+
+    def __call__(self) -> str:
+        return self.current_lease().access_token
+
+
+def _lease_expiry(lease: ProviderTokenLease) -> datetime:
+    # DelegatedTokenProvider manages refresh off the lease's declared expiry; a
+    # no-expiry lease (a notion-shaped provider) gives no refresh horizon to
+    # manage. The microsoft providers this serves always declare one — fail loud
+    # rather than silently ride a token we cannot refresh.
+    if lease.access_token_expires_at is None:
+        raise ValueError(
+            "delegated provider-token lease declares no expiry; "
+            "DelegatedTokenProvider cannot manage refresh without one."
+        )
+    return lease.access_token_expires_at