durable-sync 0.1.0__py3-none-any.whl

durable_sync/connectors/notion/destination.py

@@ -0,0 +1,270 @@
+"""Reference Destination: Notion via the hosted MCP server.
+
+Merges the two lineages:
+  * clean neutral-Record encoding + paginated idempotent upsert (ex-devrel-demos),
+  * Bearer-token transport (NO MCP SDK OAuthClientProvider), 429 backoff, and
+    inter-write pacing (ex-devrel-ships).
+
+Auth: the access token comes from `token_provider` (an async () -> str). The
+default queries OAuthTokenWorkflow, which owns the rotating refresh token; the
+token never enters event history. We pass it as a plain `Authorization: Bearer`
+header to the streamable-HTTP transport.
+
+Property encoding quirks (live-server facts): dates expand to
+`date:{prop}:start` (+ `:is_datetime`); multi-selects are JSON arrays (options
+must pre-exist); checkboxes are `__YES__`/`__NO__`; a property literally named
+`id`/`url` must be addressed `userDefined:{name}` (declare via
+`user_defined_properties`).
+
+Requires the `notion` extra:  pip install "durable-sync[notion]"
+"""
+from __future__ import annotations
+
+import asyncio
+import datetime as dt
+import json
+from contextlib import asynccontextmanager
+from typing import Any, Awaitable, Callable, AsyncIterator
+
+from mcp.client.session import ClientSession
+
+from durable_sync.core import Record, auth_error_in_chain
+from durable_sync.connectors.notion import client as mcp
+from durable_sync.connectors.notion.client import NotionMCP, TokenProvider
+from durable_sync.connectors.notion.token import current_access_token
+
+_MAX_BODY = 50000          # cap page body length to keep create snappy
+_MAX_TEXT = 2000           # Notion's hard per-rich_text limit; a longer value is
+                           # rejected by the API and the whole record silently
+                           # fails every sync, so the destination truncates.
+
+# Optional hooks (app-supplied), kept out of the generic core:
+# TokenProvider is imported from client.py (shared with the source).
+# Runs inside the open MCP session before each write — for DESTINATION-SIDE
+# enrichment that must read Notion (e.g. resolving author handles to a relation).
+# Gets the live session + the record; returns the (possibly mutated) record.
+SessionEnrich = Callable[[ClientSession, Record], Awaitable[Record]]
+# Maps a record to a page icon (emoji or URL), or None. Keeps Notion's icon
+# concept off the neutral Record.
+IconFor = Callable[[Record], "str | None"]
+
+
+class NotionDestination:
+    """Notion-MCP Destination. Configure with the target data source id and which
+    property is the title / idempotency key / sync heartbeat."""
+
+    name = "notion"
+
+    def __init__(
+        self,
+        data_source_id: str,
+        *,
+        title_property: str = "Name",
+        key_property: str = "Repo ID",
+        synced_property: str | None = "Last synced",
+        date_properties: set[str] | None = None,
+        create_only_properties: set[str] | None = None,
+        user_defined_properties: set[str] | None = None,
+        token_provider: TokenProvider | None = None,
+        session_enrich: SessionEnrich | None = None,
+        icon_for: IconFor | None = None,
+        pacing_seconds: float = 0.3,
+        resolve_data_source: bool = True,
+    ):
+        # `data_source_id` may be a data source id OR a database id/URL — with
+        # resolve_data_source on (default) the latter is resolved automatically.
+        self.data_source_id = data_source_id
+        self._resolve_ds = resolve_data_source
+        self.title_property = title_property
+        self.key_property = key_property
+        self.synced_property = synced_property
+        self.date_properties = date_properties or set()
+        # Written only on CREATE (enrichment seeds): objective fields refresh every
+        # run, but these are seeded once so human edits stick.
+        self.create_only_properties = create_only_properties or set()
+        # Property names that must be addressed as `userDefined:{name}` (Notion
+        # reserves bare `id`/`url`). Ours deliberately avoid those, but a BYO
+        # schema may need e.g. {"URL"}.
+        self.user_defined_properties = user_defined_properties or set()
+        self._token_provider = token_provider or current_access_token
+        self._session_enrich = session_enrich
+        self._icon_for = icon_for
+        self.pacing_seconds = pacing_seconds
+
+    @property
+    def configured(self) -> bool:
+        return bool(self.data_source_id)
+
+    @property
+    def config_hint(self) -> str:
+        return "NOTION_DATA_SOURCE_ID unset"
+
+    @asynccontextmanager
+    async def connect(self) -> AsyncIterator["_NotionSession"]:
+        async with mcp.open_session(self._token_provider) as session:
+            ds = self.data_source_id
+            if self._resolve_ds:
+                ds = await mcp.resolve_data_source_id(session, ds)
+            yield _NotionSession(session, self, data_source_id=ds)
+
+    # The worker auto-registers these so the token-owner workflow runs alongside
+    # the sync. (Optional hook; destinations without aux work omit it.)
+    def aux_workflows(self) -> list:
+        from durable_sync.auth.oauth.workflow import OAuthTokenWorkflow
+        return [OAuthTokenWorkflow]
+
+    def aux_activities(self) -> list:
+        from durable_sync.auth.oauth.refresh import refresh_oauth_token
+        return [refresh_oauth_token]
+
+    @staticmethod
+    def is_auth_error(err: BaseException) -> bool:
+        """A rejected Bearer token / broken refresh chain (revoked or expired) ->
+        re-bootstrap. The default signatures (401/403, unauthorized, forbidden,
+        invalid_token/grant) cover every Notion auth failure we've seen, so we
+        delegate to the shared, word-boundary-correct matcher in the spine."""
+        return auth_error_in_chain(err)
+
+
+class _NotionSession:
+    """One open MCP connection. Implements the DestinationSession protocol."""
+
+    def __init__(self, session: NotionMCP, destination: NotionDestination, *, data_source_id: str):
+        self._mcp = session
+        self._destination = destination
+        self._ds = data_source_id          # already resolved (database id -> data source id)
+
+    async def call(self, name: str, arguments: dict[str, Any]) -> str:
+        return await self._mcp.call(name, arguments)
+
+    async def query_existing_ids(self) -> dict[str, str]:
+        """{ key-property value -> page id } for rows already in the DB.
+
+        Paginates LIMIT/OFFSET with ORDER BY the key property; unordered OFFSET
+        reshuffles under concurrent edits and skips rows -> duplicates, so the
+        ORDER BY is REQUIRED."""
+        ds = self._ds
+        key = self._destination.key_property
+        PAGE = 100
+        mapping: dict[str, str] = {}
+        offset = 0
+        while True:
+            sql = mcp.query_sql(ds, order_by=key, limit=PAGE, offset=offset)
+            raw = await self.call(
+                "notion-query-data-sources",
+                {"data": {"data_source_urls": [f"collection://{ds}"], "query": sql}},
+            )
+            rows = mcp.rows_from_result(raw)
+            for row in rows:
+                kval = str(row.get(key) or "").strip()
+                page_id = mcp.page_id_from_row(row)
+                if kval and page_id:
+                    mapping[kval] = page_id
+            if len(rows) < PAGE:
+                break
+            offset += PAGE
+        return mapping
+
+    async def create(self, record: Record, synced_at: dt.datetime) -> bool:
+        record = await self._maybe_enrich(record)
+        if record is None:
+            return False  # session_enrich dropped it (out of scope)
+        page: dict[str, Any] = {"properties": self._encode(record.properties, synced_at)}
+        if record.body:
+            page["content"] = record.body[:_MAX_BODY]
+        icon = self._icon(record)
+        if icon:
+            page["icon"] = icon
+        await self.call(
+            "notion-create-pages",
+            {"parent": {"data_source_id": self._ds}, "pages": [page]},
+        )
+        await self._pace()
+        return True
+
+    async def update(self, existing_id: str, record: Record, synced_at: dt.datetime) -> bool:
+        record = await self._maybe_enrich(record)
+        if record is None:
+            return False  # session_enrich dropped it (out of scope)
+        # Skip create-only seeds (enrichment) so human edits to them survive;
+        # refresh the rest. Page body is written on create, not refreshed.
+        props = {
+            k: v for k, v in record.properties.items()
+            if k not in self._destination.create_only_properties
+        }
+        args: dict[str, Any] = {
+            "page_id": existing_id,
+            "command": "update_properties",
+            "properties": self._encode(props, synced_at),
+        }
+        icon = self._icon(record)
+        if icon:
+            args["icon"] = icon
+        await self.call("notion-update-page", args)
+        await self._pace()
+        return True
+
+    async def _maybe_enrich(self, record: Record) -> Record | None:
+        """Run the destination-side enrich hook (if any). It may return None to
+        DROP the record (an out-of-scope filter)."""
+        if self._destination._session_enrich is not None:
+            return await self._destination._session_enrich(self._mcp.session, record)
+        return record
+
+    def _icon(self, record: Record) -> str | None:
+        fn = self._destination._icon_for
+        return fn(record) if fn else None
+
+    async def _pace(self) -> None:
+        # Stay under Notion's MCP rate limit (~few req/s). Backoff handles the
+        # residual; this keeps us from hitting it in the first place.
+        if self._destination.pacing_seconds > 0:
+            await asyncio.sleep(self._destination.pacing_seconds)
+
+    def _encode(self, properties: dict[str, Any], synced_at: dt.datetime) -> dict[str, Any]:
+        """Neutral Python values -> Notion MCP wire format. bool is checked before
+        int because bool subclasses int."""
+        dest = self._destination
+        out: dict[str, Any] = {}
+        for name, val in properties.items():
+            if val is None:
+                continue
+            if name in dest.date_properties:
+                if val:
+                    start, is_dt = _encode_date(val)
+                    out[f"date:{name}:start"] = start
+                    out[f"date:{name}:is_datetime"] = is_dt
+            elif isinstance(val, bool):
+                out[_key(name, dest)] = "__YES__" if val else "__NO__"
+            elif isinstance(val, (int, float)):
+                out[_key(name, dest)] = val
+            elif isinstance(val, (list, tuple)):
+                if val:  # multi-selects are JSON arrays; options must pre-exist
+                    out[_key(name, dest)] = json.dumps(list(val))
+            else:
+                # Notion rejects any rich_text/title over 2000 chars; a long
+                # value (e.g. a verbose repo description) would 400 and the record
+                # would silently fail to sync every run. Truncate as a backstop —
+                # the destination owns wire limits (per the core contract).
+                out[_key(name, dest)] = str(val)[:_MAX_TEXT]
+        # Sync heartbeat: "Last synced" is a DATE column -> stamp the UTC date.
+        if dest.synced_property:
+            out[f"date:{dest.synced_property}:start"] = synced_at.date().isoformat()
+            out[f"date:{dest.synced_property}:is_datetime"] = 0
+        return out
+
+
+def _key(name: str, dest: NotionDestination) -> str:
+    """Prefix props that collide with Notion's reserved id/url addressing."""
+    return f"userDefined:{name}" if name in dest.user_defined_properties else name
+
+
+def _encode_date(val: Any) -> tuple[str, int]:
+    """Return (start-string, is_datetime). A datetime, or an ISO string with a
+    'T', carries time -> is_datetime=1; a plain date -> 0."""
+    if isinstance(val, dt.datetime):
+        return val.isoformat(), 1
+    if isinstance(val, dt.date):
+        return val.isoformat(), 0
+    s = str(val)
+    return s, (1 if "T" in s else 0)

durable_sync/connectors/notion/oauth.py

@@ -0,0 +1,25 @@
+"""Notion binding of the generic OAuth client (durable_sync.auth.oauth).
+
+The OAuth 2.1 + PKCE + DCR flow is provider-agnostic and lives in auth/oauth/flow.py;
+this module just pins Notion's hosted MCP server and re-exports the flow so the
+Notion bootstrap/prove/destination can keep importing `oauth.*`.
+"""
+from __future__ import annotations
+
+from durable_sync.auth.oauth.flow import (  # noqa: F401  (re-exported for callers)
+    build_authorize_url,
+    exchange_code,
+    gen_pkce,
+    new_state,
+    refresh_access_token,
+    register_client,
+)
+from durable_sync.auth.oauth import flow as _generic
+
+MCP_BASE = "https://mcp.notion.com"
+MCP_ENDPOINT = f"{MCP_BASE}/mcp"  # Streamable HTTP transport
+
+
+def discover() -> dict[str, str]:
+    """Discover Notion's OAuth endpoints (generic flow, Notion base URL)."""
+    return _generic.discover(MCP_BASE)

durable_sync/connectors/notion/prove.py

@@ -0,0 +1,57 @@
+"""Headless proof: NO browser. Loads the saved refresh token, mints a fresh
+access token, and uses it to actually talk to the Notion MCP server.
+
+    PYTHONPATH=. python -m durable_sync.connectors.notion.prove
+
+The de-risking step for the whole architecture: if this works, the Temporal
+auth workflow can do exactly the same on a timer with no human present.
+"""
+from __future__ import annotations
+
+import asyncio
+
+from mcp import ClientSession
+from mcp.client.streamable_http import streamablehttp_client
+
+from durable_sync.connectors.notion import oauth, store
+
+
+async def _call_mcp(access_token: str) -> list[str]:
+    """Connect to Notion MCP with the access token; return tool names. list_tools()
+    succeeding proves the token authenticated the session."""
+    headers = {"Authorization": f"Bearer {access_token}"}
+    async with streamablehttp_client(oauth.MCP_ENDPOINT, headers=headers) as (read, write, _):
+        async with ClientSession(read, write) as session:
+            await session.initialize()
+            tools = await session.list_tools()
+            return [t.name for t in tools.tools]
+
+
+def main() -> None:
+    creds = store.load()
+    if not creds:
+        raise SystemExit(
+            f"No credentials at {store.path()}. Run the bootstrap first:\n"
+            f"  PYTHONPATH=. python -m durable_sync.connectors.notion.bootstrap"
+        )
+
+    print("Refreshing access token (headless, no browser)...")
+    tokens = oauth.refresh_access_token(
+        creds["token_endpoint"], creds["client_id"], creds["refresh_token"]
+    )
+    # Notion ROTATES the refresh token on every use — persist the new one now,
+    # atomically, or the next run fails with invalid_grant.
+    creds["refresh_token"] = tokens["refresh_token"]
+    store.save(creds)
+    print(f"  Got access token (expires in {tokens.get('expires_in')}s). Rotated refresh token persisted.")
+
+    print("Calling Notion MCP with the minted access token...")
+    tool_names = asyncio.run(_call_mcp(tokens["access_token"]))
+    print(f"\nSUCCESS — authenticated headlessly. MCP exposed {len(tool_names)} tools:")
+    for name in tool_names:
+        print(f"  - {name}")
+    print("\nHeadless auth proven — the Temporal auth workflow can run this unattended.")
+
+
+if __name__ == "__main__":
+    main()

durable_sync/connectors/notion/source.py

@@ -0,0 +1,136 @@
+"""NotionSource — read rows from a Notion data source -> Records.
+
+The read half of the Notion connector; shares the MCP client + OAuth with
+NotionDestination (see client.py), which is the whole reason connectors are
+grouped by system. Each row becomes a Record keyed on its Notion page id — the
+immutable, sync-safe id when Notion is the system of record.
+
+Column values come back as the query renders them (text); for precise typing or
+to pull page body content, use the `enrich` hook — it gets the raw row plus the
+live MCP session for extra calls. Requires the `notion` extra.
+"""
+from __future__ import annotations
+
+import inspect
+import logging
+from dataclasses import dataclass
+from typing import Awaitable, Callable, Union
+
+from temporalio import activity
+
+from durable_sync.core import Record, SourceSpec
+from durable_sync.connectors.notion import client as mcp
+from durable_sync.connectors.notion.client import NotionMCP, TokenProvider
+from durable_sync.connectors.notion.token import current_access_token
+
+log = logging.getLogger("durable_sync.connectors.notion.source")
+
+EnrichHook = Callable[[Record, "NotionRowContext"], Union[Record, Awaitable[Record]]]
+
+_PAGE = 100
+
+
+@dataclass
+class NotionRowContext:
+    """Handed to the enrich hook: the raw queried row + the live MCP session, so
+    enrich can type-coerce columns or fetch page content without re-connecting."""
+    raw_row: dict
+    session: NotionMCP
+
+
+def _heartbeat(detail: str) -> None:
+    if activity.in_activity():
+        activity.heartbeat(detail)
+
+
+class NotionSource:
+    name = "notion"
+
+    def __init__(
+        self,
+        data_source_id: str,
+        *,
+        order_property: str | None = None,
+        interval_minutes: int = 30,
+        token_provider: TokenProvider | None = None,
+        enrich: EnrichHook | None = None,
+        resolve_data_source: bool = True,
+        decode: bool = True,
+    ):
+        # `data_source_id` may be a data source id OR a database id/URL — with
+        # resolve_data_source on (default) the latter is resolved automatically.
+        self.data_source_id = data_source_id
+        # Pagination is LIMIT/OFFSET; ordering by a STABLE column keeps pages from
+        # reshuffling under concurrent edits (else a run can skip/dupe rows — self-
+        # corrects next run since the upsert is idempotent, but order if you can).
+        self.order_property = order_property
+        self.interval_minutes = interval_minutes
+        self._token_provider = token_provider or current_access_token
+        self._enrich = enrich
+        self._resolve_ds = resolve_data_source
+        self._decode = decode
+        self._resolved_ds: str | None = None
+
+    def specs(self) -> list[SourceSpec]:
+        return [SourceSpec(key=f"ds:{self.data_source_id}", interval_minutes=self.interval_minutes,
+                           params={"data_source_id": self.data_source_id})]
+
+    # The Notion OAuth token workflow must run alongside ANY route that touches
+    # Notion — source or destination. The worker registers a source's aux work too
+    # (and dedupes, so a Notion->Notion route registers it once).
+    def aux_workflows(self) -> list:
+        from durable_sync.auth.oauth.workflow import OAuthTokenWorkflow
+        return [OAuthTokenWorkflow]
+
+    def aux_activities(self) -> list:
+        from durable_sync.auth.oauth.refresh import refresh_oauth_token
+        return [refresh_oauth_token]
+
+    async def fetch(self, spec: SourceSpec, only_items: list[str] | None = None) -> list[Record]:
+        ds = spec.params.get("data_source_id", self.data_source_id)
+        targeted = set(only_items or [])   # page ids for a targeted refresh
+        out: list[Record] = []
+        async with mcp.open_session(self._token_provider) as session:
+            if self._resolve_ds:
+                if self._resolved_ds is None:
+                    self._resolved_ds = await mcp.resolve_data_source_id(session, ds)
+                    if self._resolved_ds != ds:
+                        log.info("Resolved database %s -> data source %s", ds, self._resolved_ds)
+                ds = self._resolved_ds
+            offset = 0
+            while True:
+                sql = mcp.query_sql(ds, order_by=self.order_property, limit=_PAGE, offset=offset)
+                raw = await session.call(
+                    "notion-query-data-sources",
+                    {"data": {"data_source_urls": [f"collection://{ds}"], "query": sql}},
+                )
+                rows = mcp.rows_from_result(raw)
+                for row in rows:
+                    record = self._to_record(row)
+                    if record is None:
+                        continue
+                    if targeted and record.primary_key not in targeted:
+                        continue
+                    if self._enrich is not None:
+                        ctx = NotionRowContext(raw_row=row, session=session)
+                        result = self._enrich(record, ctx)
+                        record = await result if inspect.isawaitable(result) else result
+                    out.append(record)
+                    _heartbeat(record.primary_key)
+                if len(rows) < _PAGE:
+                    break
+                offset += _PAGE
+        log.info("Fetched %d Notion rows for %s", len(out), spec.key)
+        return out
+
+    def _to_record(self, row: dict) -> Record | None:
+        """Map one queried Notion row to a neutral Record. Pure (no IO). Returns
+        None for a row with no resolvable page id — it can't be keyed idempotently
+        (primary_key must be the immutable page id, never a column value)."""
+        page_id = mcp.page_id_from_row(row)
+        if not page_id:
+            return None
+        columns = mcp.row_columns(row)
+        if self._decode:
+            columns = mcp.decode_row(columns)
+        return Record(primary_key=page_id, properties=columns)

durable_sync/connectors/notion/start.py

@@ -0,0 +1,46 @@
+"""Launch OAuthTokenWorkflow from the credentials bootstrap saved.
+
+    PYTHONPATH=. python -m durable_sync.connectors.notion.start
+
+Reads the bootstrap creds, starts the single long-running auth workflow, and
+hands ownership of the refresh token to it. After this, the worker keeps access
+tokens fresh unattended; the local file is no longer the source of truth.
+"""
+from __future__ import annotations
+
+import asyncio
+
+from durable_sync import config
+from durable_sync.auth.oauth.workflow import AuthParams, OAuthTokenWorkflow
+from durable_sync.connectors.notion import store
+from durable_sync.temporal_client import connect
+
+
+async def main() -> None:
+    creds = store.load()
+    if not creds:
+        raise SystemExit(
+            f"No credentials at {store.path()}. Run the bootstrap first:\n"
+            f"  PYTHONPATH=. python -m durable_sync.connectors.notion.bootstrap"
+        )
+
+    client = await connect()
+    handle = await client.start_workflow(
+        OAuthTokenWorkflow.run,
+        AuthParams(
+            client_id=creds["client_id"],
+            token_endpoint=creds["token_endpoint"],
+            refresh_token=creds["refresh_token"],
+        ),
+        id=config.NOTION_AUTH_WORKFLOW_ID,
+        task_queue=config.TASK_QUEUE,
+    )
+    print(
+        f"Started OAuthTokenWorkflow (id={handle.id}). It now owns the refresh "
+        f"token and keeps access tokens fresh.\n"
+        f"Verify:  temporal workflow query --workflow-id {handle.id} --type get_access_token"
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

durable_sync/connectors/notion/store.py

@@ -0,0 +1,25 @@
+"""Notion binding of the generic creds store (durable_sync.auth.store).
+
+Pins Notion's auth file path; bootstrap/prove/start call load()/save()/path().
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+from durable_sync.auth.oauth import store as _store
+
+_FILE = os.getenv("DURABLE_SYNC_NOTION_AUTH_FILE", ".notion_auth.json")
+
+
+def load() -> dict[str, Any] | None:
+    return _store.load(_FILE)
+
+
+def save(data: dict[str, Any]) -> None:
+    _store.save(_FILE, data)
+
+
+def path() -> Path:
+    return _store.resolve(_FILE)

durable_sync/connectors/notion/token.py

@@ -0,0 +1,13 @@
+"""Notion binding of the generic token accessor (durable_sync.auth.oauth.token).
+
+The default token_provider for NotionDestination: query the OAuthTokenWorkflow
+running under config.NOTION_AUTH_WORKFLOW_ID for a fresh access token.
+"""
+from __future__ import annotations
+
+from durable_sync import config
+from durable_sync.auth.oauth.token import current_access_token as _current
+
+
+async def current_access_token() -> str:
+    return await _current(config.NOTION_AUTH_WORKFLOW_ID)

durable_sync/connectors/youtube/__init__.py

@@ -0,0 +1,13 @@
+"""YouTube source: a channel's uploads -> Records.
+
+YouTube exposes no per-video author, so attribution (if you need it) is an
+app-side concern: the Record carries a "Scan Text" field and the enrich hook gets
+a YouTubeVideoContext for inverted name-matching against your own directory of people.
+
+Requires the `youtube` extra:  pip install "durable-sync[youtube]"
+"""
+from __future__ import annotations
+
+from durable_sync.connectors.youtube.source import YouTubeConfig, YouTubeSource, YouTubeVideoContext
+
+__all__ = ["YouTubeSource", "YouTubeConfig", "YouTubeVideoContext"]