durable-sync 0.1.0__py3-none-any.whl

durable_sync/__init__.py

@@ -0,0 +1,26 @@
+"""durable-sync: durable, idempotent source -> destination sync on Temporal.
+
+Public API — implement `Source` for your data, `Destination` for your target;
+the spine (entity workflow, idempotent upsert, OAuth refresh, backoff) is
+inherited. See `connectors/` (one subpackage per system — GitHub/Luma/YouTube/
+Contentful sources, Notion/Asana destinations) for reference implementations.
+"""
+from __future__ import annotations
+
+from durable_sync.core import (
+    Destination,
+    DestinationSession,
+    Record,
+    Source,
+    SourceSpec,
+)
+
+__all__ = [
+    "Record",
+    "SourceSpec",
+    "Source",
+    "Destination",
+    "DestinationSession",
+]
+
+__version__ = "0.1.0"

durable_sync/activities.py

@@ -0,0 +1,156 @@
+"""Generic, source/destination-agnostic activities.
+
+A library can't hardcode `from pipeline import SOURCE, DESTINATION` the way a
+single app would, so the activities are produced by a FACTORY the app calls once
+with its wired Source + Destination:
+
+    worker = Worker(..., activities=make_activities(SOURCE, DESTINATION))
+
+The activities are registered under stable string names (FETCH_SOURCE /
+SYNC_RECORDS); the workflow refers to them by those names, so it never imports
+these closures and stays sandbox-clean.
+"""
+from __future__ import annotations
+
+import datetime as dt
+import inspect
+from dataclasses import dataclass, field
+from typing import Awaitable, Callable, Union
+
+from temporalio import activity
+from temporalio.exceptions import ApplicationError
+
+from durable_sync.core import Destination, Record, Source, SourceSpec
+
+# Stable activity names — the workflow executes by name (see workflows/sync.py).
+FETCH_SOURCE = "fetch_source"
+SYNC_RECORDS = "sync_records"
+
+
+@dataclass
+class FetchPage:
+    """One page of fetched+transformed records, plus the cursor for the NEXT page
+    (None on the last page). A Source that implements `fetch_page` lets the spine
+    bound Temporal history for arbitrarily large sources — neither the fetch result
+    nor the upsert payload is ever the whole dataset. A Source without it returns
+    everything as a single page (next_cursor=None), exactly as before."""
+    records: list[Record] = field(default_factory=list)
+    next_cursor: str | None = None
+
+# The GENERIC transform seam: Record -> Record (mutate/derive/rename) or None
+# (drop it — so transform doubles as a filter). Source- and destination-agnostic;
+# may be sync or async. For transforms that need source internals use the source's
+# enrich hook; for ones that read the destination use its session_enrich.
+Transform = Callable[[Record], Union[Record, None, Awaitable[Union[Record, None]]]]
+
+
+def make_activities(
+    source: Source, destination: Destination, *, transform: Transform | None = None
+) -> list:
+    """Build the two generic activities, closed over the app's Source +
+    Destination (+ optional generic transform). Returns a list ready to hand to a
+    Temporal Worker."""
+
+    async def _apply_transform(records: list[Record]) -> list[Record]:
+        if transform is None:
+            return records
+        out: list[Record] = []
+        for rec in records:
+            res = transform(rec)
+            if inspect.isawaitable(res):
+                res = await res
+            if res is not None:
+                out.append(res)
+        return out
+
+    @activity.defn(name=FETCH_SOURCE)
+    async def fetch_source(
+        spec: SourceSpec, only_items: list[str] | None = None, cursor: str | None = None
+    ) -> FetchPage:
+        """Fetch ONE page of a source unit (optionally just specific items), apply
+        the generic transform (which may drop records by returning None), and return
+        the records + the next-page cursor.
+
+        Adapts to the Source: if it implements `fetch_page(spec, only_items, cursor)
+        -> (records, next_cursor)` the spine paginates (bounded history for big
+        sources); otherwise it calls `fetch()` once and returns everything as a
+        single page (next_cursor=None) — unchanged behavior."""
+        fetch_page = getattr(source, "fetch_page", None)
+        if callable(fetch_page):
+            records, next_cursor = await fetch_page(spec, only_items, cursor)
+        else:
+            records = await source.fetch(spec, only_items)
+            next_cursor = None
+        return FetchPage(records=await _apply_transform(records), next_cursor=next_cursor)
+
+    @activity.defn(name=SYNC_RECORDS)
+    async def sync_records(records: list[Record]) -> dict:
+        """Idempotent upsert into the Destination, keyed on each primary_key."""
+        if not destination.configured:
+            raise ApplicationError(
+                f"Destination is not configured ({destination.config_hint})",
+                type="ConfigError", non_retryable=True,
+            )
+
+        synced_at = dt.datetime.now(dt.timezone.utc)
+        created = updated = skipped = 0
+
+        # Guard the idempotency key BEFORE any write (counts fold into `skipped`
+        # so created+updated+skipped == total still holds):
+        #  * a falsy primary_key can't be idempotent — every keyless record
+        #    collides on the same empty key (the first creates a row, the rest
+        #    "update" that one row, or all overwrite one link). Drop them.
+        #  * a duplicate primary_key WITHIN one batch would double-create:
+        #    `existing` is queried once up front, so a 2nd occurrence is still
+        #    "not existing" and creates a second row. Collapse to the LAST
+        #    occurrence (freshest data) so the upsert stays idempotent.
+        deduped: dict[str, Record] = {}
+        for rec in records:
+            if not rec.primary_key:
+                skipped += 1
+                activity.logger.warning(
+                    "Skipping record with empty primary_key (not idempotent): %r",
+                    rec.properties,
+                )
+                continue
+            deduped[rec.primary_key] = rec
+        to_sync = list(deduped.values())
+        in_batch_dupes = (len(records) - skipped) - len(to_sync)
+        if in_batch_dupes:
+            skipped += in_batch_dupes
+            activity.logger.warning(
+                "Collapsed %d in-batch duplicate primary_key(s) (last-wins)", in_batch_dupes
+            )
+
+        try:
+            async with destination.connect() as session:
+                existing = await session.query_existing_ids()  # primary_key -> dest id
+                for rec in to_sync:
+                    existing_id = existing.get(rec.primary_key)
+                    if existing_id:
+                        wrote = await session.update(existing_id, rec, synced_at)
+                        updated += 1 if wrote else 0
+                    else:
+                        wrote = await session.create(rec, synced_at)
+                        created += 1 if wrote else 0
+                    skipped += 0 if wrote else 1   # dropped by a destination-side filter
+                    activity.heartbeat(rec.primary_key)
+        except ApplicationError:
+            raise
+        except Exception as e:
+            # Auth failures are NOT retryable — only a human re-auth fixes them,
+            # so the workflow can pause instead of hammering a dead credential.
+            # Everything else stays retryable (transient).
+            if destination.is_auth_error(e):
+                raise ApplicationError(
+                    "Destination authorization is no longer valid (token refresh "
+                    "failed or was revoked). Re-authorize, then send `resume`.",
+                    type="AuthError", non_retryable=True,
+                ) from e
+            raise
+
+        stats = {"total": len(records), "created": created, "updated": updated, "skipped": skipped}
+        activity.logger.info("Sync complete: %s", stats)
+        return stats
+
+    return [fetch_source, sync_records]

durable_sync/auth/__init__.py

@@ -0,0 +1,8 @@
+"""Authentication mechanisms for destinations.
+
+A cross-cutting toolkit (not a connector), organized by mechanism. Today there's
+one: `oauth/` — the OAuth-as-a-workflow toolkit (token-owner workflow + flow +
+store) for no-admin providers. Mechanisms that need NO shared code (e.g. a
+self-serve PAT, which is a one-liner) live inline in their connector, so they get
+no package here until there's something to share.
+"""

durable_sync/auth/oauth/__init__.py

@@ -0,0 +1,18 @@
+"""Generic OAuth-as-a-workflow toolkit (provider-agnostic).
+
+For destinations whose API offers no admin-free static token: authorize as an
+individual via OAuth 2.1 + PKCE + dynamic client registration, then let a Temporal
+workflow OWN the rotating refresh token — refreshing on a timer and serving fresh
+access tokens via query (so the secret never enters event history). Standards-based
+(RFC 8414 discovery, RFC 7591 dynamic registration, PKCE).
+
+Import from the SUBMODULES, not this package. This __init__ deliberately imports
+nothing: `flow` pulls in `requests`, and the Temporal workflow sandbox forbids
+that — so it must only ever be loaded via the workflow's pass-through import, never
+eagerly here (an eager re-export here breaks `OAuthTokenWorkflow` sandbox validation).
+
+    from durable_sync.auth.oauth.workflow import OAuthTokenWorkflow, AuthParams
+    from durable_sync.auth.oauth.refresh  import refresh_oauth_token
+    from durable_sync.auth.oauth.token    import current_access_token
+    from durable_sync.auth.oauth import flow, store
+"""

durable_sync/auth/oauth/flow.py

@@ -0,0 +1,183 @@
+"""OAuth 2.1 (PKCE + dynamic client registration) — provider-agnostic HTTP
+helpers. No Temporal, no browser, no file IO, no hardcoded provider: every
+endpoint is passed in (discover() takes the server base URL). Reusable from an
+interactive bootstrap AND from the refresh activity.
+
+Public clients (token_endpoint_auth_method="none"): no client secret, PKCE
+mandatory. Endpoints are discovered, not hardcoded, so this keeps working if a
+provider moves them.
+
+Deliberately NOT the MCP SDK's OAuthClientProvider: we own the token lifecycle
+(the auth workflow does) and pass a plain Bearer header to the transport, which
+sidesteps that SDK's cross-version auth API churn.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import os
+import secrets
+from typing import Any
+from urllib.parse import urlsplit
+
+import requests
+
+_TIMEOUT = 30
+DEFAULT_CLIENT_NAME = "durable-sync"
+
+
+def _registrable_domain(host: str) -> str:
+    """Last two labels of a host (heuristic, no PSL): notion.com, contentful.com.
+    Good enough to pin discovered OAuth endpoints to the provider's own domain;
+    the hard guarantee is the https check in _validate_endpoint."""
+    labels = host.split(".")
+    return ".".join(labels[-2:]) if len(labels) >= 2 else host
+
+
+def _validate_endpoint(url: str, base_url: str, *, same_site: bool) -> str:
+    """Reject a discovered OAuth endpoint that could exfiltrate the refresh token.
+
+    We POST the refresh token to whatever the discovery documents name, on every
+    refresh, unattended — so a tampered/compromised discovery response must not be
+    able to point us at an attacker host. Enforce https always; when `same_site`
+    (the default), also require the same registrable domain as the pinned base URL.
+    Providers whose authorization server is on a different domain pass same_site=False."""
+    parts = urlsplit(url)
+    if parts.scheme != "https":
+        raise ValueError(f"Refusing non-https OAuth endpoint from discovery: {url!r}")
+    if same_site:
+        base_host = urlsplit(base_url).hostname or ""
+        host = parts.hostname or ""
+        if _registrable_domain(host) != _registrable_domain(base_host):
+            raise ValueError(
+                f"Discovered OAuth endpoint {host!r} is off-domain from {base_host!r}; "
+                f"refusing (pass same_site=False if this provider's auth server is "
+                f"intentionally on another domain)."
+            )
+    return url
+
+
+def discover(base_url: str, *, same_site: bool = True) -> dict[str, str]:
+    """Two-step OAuth discovery (RFC 9728 protected-resource -> RFC 8414 AS
+    metadata) against `base_url`. Returns authorization/token/registration
+    endpoints. Every discovered endpoint is validated (https + same-domain) before
+    return, because the token endpoint later receives the refresh token unattended
+    (see _validate_endpoint)."""
+    pr = requests.get(f"{base_url}/.well-known/oauth-protected-resource", timeout=_TIMEOUT)
+    pr.raise_for_status()
+    auth_server = _validate_endpoint(
+        pr.json()["authorization_servers"][0], base_url, same_site=same_site
+    )
+
+    md = requests.get(f"{auth_server}/.well-known/oauth-authorization-server", timeout=_TIMEOUT)
+    md.raise_for_status()
+    data = md.json()
+    return {
+        "authorization_endpoint": _validate_endpoint(data["authorization_endpoint"], base_url, same_site=same_site),
+        "token_endpoint": _validate_endpoint(data["token_endpoint"], base_url, same_site=same_site),
+        "registration_endpoint": _validate_endpoint(data["registration_endpoint"], base_url, same_site=same_site),
+    }
+
+
+def register_client(
+    registration_endpoint: str, redirect_uri: str, *, client_name: str = DEFAULT_CLIENT_NAME
+) -> dict[str, Any]:
+    """Dynamic Client Registration (RFC 7591) — no admin, no pre-approval."""
+    resp = requests.post(
+        registration_endpoint,
+        json={
+            "client_name": client_name,
+            "redirect_uris": [redirect_uri],
+            "grant_types": ["authorization_code", "refresh_token"],
+            "response_types": ["code"],
+            "token_endpoint_auth_method": "none",
+        },
+        timeout=_TIMEOUT,
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+def gen_pkce() -> tuple[str, str]:
+    """Return (verifier, challenge) for PKCE S256."""
+    verifier = base64.urlsafe_b64encode(os.urandom(32)).rstrip(b"=").decode()
+    digest = hashlib.sha256(verifier.encode()).digest()
+    challenge = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
+    return verifier, challenge
+
+
+def new_state() -> str:
+    return secrets.token_urlsafe(16)
+
+
+def build_authorize_url(
+    authorization_endpoint: str, client_id: str, redirect_uri: str,
+    code_challenge: str, state: str,
+) -> str:
+    from urllib.parse import urlencode
+    params = {
+        "response_type": "code",
+        "client_id": client_id,
+        "redirect_uri": redirect_uri,
+        "code_challenge": code_challenge,
+        "code_challenge_method": "S256",
+        "state": state,
+    }
+    return f"{authorization_endpoint}?{urlencode(params)}"
+
+
+def exchange_code(
+    token_endpoint: str, client_id: str, code: str, redirect_uri: str, code_verifier: str
+) -> dict[str, Any]:
+    """Authorization code -> tokens (access_token, refresh_token, expires_in)."""
+    resp = requests.post(
+        token_endpoint,
+        data={
+            "grant_type": "authorization_code",
+            "code": code,
+            "client_id": client_id,
+            "redirect_uri": redirect_uri,
+            "code_verifier": code_verifier,
+        },
+        timeout=_TIMEOUT,
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+def refresh_access_token(token_endpoint: str, client_id: str, refresh_token: str) -> dict[str, Any]:
+    """Refresh token -> a fresh access_token (and possibly a ROTATED refresh_token).
+
+    Providers like Notion rotate the refresh token on every use, so the caller
+    MUST persist the returned refresh_token; an `invalid_grant` means the stored
+    token was already spent -> re-bootstrap.
+    """
+    resp = requests.post(
+        token_endpoint,
+        data={
+            "grant_type": "refresh_token",
+            "refresh_token": refresh_token,
+            "client_id": client_id,
+        },
+        timeout=_TIMEOUT,
+    )
+    if resp.status_code >= 400:
+        # OAuth errors are JSON: {"error": "...", "error_description": "..."}. Surface
+        # the body, and turn the common "your token is dead" cases into a plain-English
+        # hint instead of a bare HTTPError. Keep `invalid_grant`/401 in the message so
+        # is_auth_error still classifies it.
+        body = resp.text[:600]
+        try:
+            err = (resp.json() or {}).get("error", "")
+        except ValueError:
+            err = ""
+        if err in ("invalid_grant", "invalid_client") or resp.status_code in (400, 401):
+            raise RuntimeError(
+                f"OAuth token refresh rejected ({resp.status_code} {err or 'error'}). The stored "
+                f"refresh token is no longer valid — expired, revoked, or already spent (providers "
+                f"that rotate the refresh token on every use, e.g. Notion, invalidate the old one each "
+                f"refresh). Re-authorize to mint a fresh token by re-running your provider's bootstrap. "
+                f"Server said: {body}"
+            )
+        raise RuntimeError(f"OAuth token refresh failed ({resp.status_code}): {body}")
+    return resp.json()

durable_sync/auth/oauth/refresh.py

@@ -0,0 +1,58 @@
+"""The refresh activity — the only IO in the OAuth auth hot path.
+
+Wraps oauth.refresh_access_token so OAuthTokenWorkflow stays deterministic (no
+network in the workflow). Returns the new access token AND the rotated refresh
+token; the workflow persists both in its state. Provider-agnostic — the token
+endpoint + client id come in via the input. Kept in its own module so the
+workflow imports it pass-through without dragging `requests` into the sandbox.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from temporalio import activity
+from temporalio.exceptions import ApplicationError
+
+from durable_sync.auth.oauth import flow as oauth
+from durable_sync.core import auth_error_in_chain
+
+
+@dataclass
+class RefreshInput:
+    client_id: str
+    token_endpoint: str
+    refresh_token: str
+
+
+@dataclass
+class RefreshOutput:
+    access_token: str
+    refresh_token: str  # rotated — the workflow MUST store this
+    expires_in: int
+
+
+@activity.defn
+def refresh_oauth_token(inp: RefreshInput) -> RefreshOutput:
+    try:
+        tokens = oauth.refresh_access_token(inp.token_endpoint, inp.client_id, inp.refresh_token)
+    except Exception as e:
+        # A revoked/expired/spent refresh token can't be fixed by retrying — only a
+        # human re-bootstrap mints a new one. Mark it non-retryable + typed so the
+        # OAuthTokenWorkflow PAUSES (stays queryable + resumable) instead of burning
+        # retries and then terminating. Transient failures stay retryable (re-raise).
+        if auth_error_in_chain(e):
+            raise ApplicationError(
+                "OAuth refresh token is no longer valid (expired, revoked, or spent). "
+                "Re-bootstrap to mint a fresh token, then send the `reauthorize` signal.",
+                type="AuthError", non_retryable=True,
+            ) from e
+        raise
+    return RefreshOutput(
+        access_token=tokens["access_token"],
+        # Not every provider rotates the refresh token on each refresh — many omit
+        # `refresh_token` from the response when it's unchanged. Falling back to the
+        # one we sent keeps the chain alive instead of KeyError-ing the activity
+        # (which, after retries, would kill the token workflow and break auth).
+        refresh_token=tokens.get("refresh_token") or inp.refresh_token,
+        expires_in=int(tokens.get("expires_in", 3600)),
+    )

durable_sync/auth/oauth/store.py

@@ -0,0 +1,36 @@
+"""Local credential store for the OAuth bootstrap handoff (provider-agnostic).
+
+BOOTSTRAP/PROOF persistence only. In the running system the refresh token lives
+in OAuthTokenWorkflow's state (durable, single-owner) — but bootstrap needs
+somewhere to hand off the initial token, and prove reads it back. The file is
+gitignored; never commit it. Each provider passes its own `file` path.
+"""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+
+def load(file: str) -> dict[str, Any] | None:
+    p = Path(file)
+    if not p.exists():
+        return None
+    return json.loads(p.read_text())
+
+
+def save(file: str, data: dict[str, Any]) -> None:
+    """Write atomically so a crash mid-write can't corrupt the rotating token."""
+    p = Path(file)
+    tmp = p.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data, indent=2))
+    try:
+        tmp.chmod(0o600)
+    except OSError:
+        pass
+    os.replace(tmp, p)
+
+
+def resolve(file: str) -> Path:
+    return Path(file)

durable_sync/auth/oauth/token.py

@@ -0,0 +1,36 @@
+"""Access-token accessor for use INSIDE activities (a destination's session).
+
+current_access_token(workflow_id) queries the OAuthTokenWorkflow with that id for
+a valid token. The query result is used locally and never returned from the
+activity, so the token stays out of Temporal event history.
+
+Not workflow code — safe to do IO and cache a client.
+"""
+from __future__ import annotations
+
+from temporalio.client import Client
+
+from durable_sync.auth.oauth.workflow import OAuthTokenWorkflow
+from durable_sync.temporal_client import connect
+
+_client: Client | None = None
+
+
+async def _get_client() -> Client:
+    global _client
+    if _client is None:
+        _client = await connect()
+    return _client
+
+
+async def current_access_token(workflow_id: str) -> str:
+    """Query the OAuthTokenWorkflow `workflow_id` for a fresh access token."""
+    client = await _get_client()
+    handle = client.get_workflow_handle(workflow_id)
+    token = await handle.query(OAuthTokenWorkflow.get_access_token)
+    if not token:
+        raise RuntimeError(
+            f"OAuthTokenWorkflow '{workflow_id}' returned an empty access token — "
+            f"is it running? Bootstrap + start it (see the destination's docs)."
+        )
+    return token