parse-sdk 0.1.0__py3-none-any.whl

parse_sdk/__init__.py

@@ -0,0 +1,57 @@
+"""Parse SDK engine — errors, runtime helpers, and the `parse` CLI.
+
+Typed clients are generated by `parse sync` into a project-local
+`parse_apis/` package and imported directly:
+
+    import parse_sdk as parse                      # errors / runtime helpers
+    from parse_apis.reddit import Reddit           # generated typed client
+
+    try:
+        reddit = Reddit()                          # picks up `parse login` creds
+        for post in reddit.subreddit("smallbusiness").search_posts(query="hiring", limit=10):
+            print(post.title)
+    except parse.RateLimitError:
+        ...
+
+`parse_apis` is a real on-disk namespace package (pyright-resolvable, no
+import magic). `parse_sdk` no longer resolves generated submodules.
+"""
+
+from __future__ import annotations
+
+from parse_sdk._runtime import (
+    AuthError,
+    BadRequestError,
+    BaseAPI,
+    Collection,
+    ForbiddenError,
+    NotFoundError,
+    PaginationError,
+    PaginationLimitError,
+    Paginator,
+    ParseError,
+    QuotaExceededError,
+    RateLimitError,
+    Resource,
+    RequestMeta,
+    UpstreamError,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "BaseAPI",
+    "Resource",
+    "Collection",
+    "Paginator",
+    "RequestMeta",
+    "ParseError",
+    "AuthError",
+    "ForbiddenError",
+    "NotFoundError",
+    "BadRequestError",
+    "QuotaExceededError",
+    "RateLimitError",
+    "UpstreamError",
+    "PaginationError",
+    "PaginationLimitError",
+]

parse_sdk/_labels.py

@@ -0,0 +1,51 @@
+"""Shared parsing for the closed type-label grammar codegen emits
+(``str`` | ``List[X]`` | ``Optional[X]`` | ``Dict[K, V]`` | ``_Paginator[X]`` |
+unions). Single source: ``codegen_reconcile``, ``sample_gate``, and the
+runtime previously each hand-rolled a splitter/unwrapper and drifted
+(``Paginator[`` vs ``_Paginator[``). Stdlib-only — ``_runtime`` imports it."""
+from __future__ import annotations
+
+from typing import List, Optional
+
+_WRAPPERS = ("List[", "Optional[", "_Paginator[")
+
+
+def strip_wrapper(s: str, prefix: str) -> Optional[str]:
+    """The inner of a single ``<prefix>[...]`` layer, or ``None`` if ``s`` is not
+    that shape: ``strip_wrapper('Optional[X]', 'Optional[')`` → ``'X'``. The
+    one-layer primitive that ``unwrap_label`` loops over and ``sample_gate``
+    applies to a known prefix — single source for the ``s[len(prefix):-1]``
+    slice so the grammar parsers cannot drift."""
+    if s.startswith(prefix) and s.endswith("]"):
+        return s[len(prefix):-1].strip()
+    return None
+
+
+def split_top(s: str) -> List[str]:
+    """Split on commas at bracket-depth 0: ``'str, Dict[a, b]'`` →
+    ``['str', 'Dict[a, b]']``."""
+    parts, depth, start = [], 0, 0
+    for i, ch in enumerate(s):
+        if ch == "[":
+            depth += 1
+        elif ch == "]":
+            depth -= 1
+        elif ch == "," and depth == 0:
+            parts.append(s[start:i].strip())
+            start = i + 1
+    parts.append(s[start:].strip())
+    return parts
+
+
+def unwrap_label(s: str, *, max_depth: int = 5) -> str:
+    """Strip wrapper layers down to the leaf: ``'List[Optional[X]]'`` → ``'X'``."""
+    s = s.strip()
+    for _ in range(max_depth):
+        for pfx in _WRAPPERS:
+            inner = strip_wrapper(s, pfx)
+            if inner is not None:
+                s = inner
+                break
+        else:
+            return s
+    return s

parse_sdk/_oauth.py

@@ -0,0 +1,301 @@
+"""Browser (loopback) OAuth 2.1 + PKCE login for `parse login --web`.
+
+Reuses the backend's existing OAuth endpoints (the same ones the MCP server
+advertises): dynamic client registration → /oauth/authorize (browser consent on
+the Parse frontend) → loopback redirect with ?code → /oauth/token exchange. The
+minted token (aud=parse-mcp) authorizes /sdk + /scraper once the executor accepts
+it (backend Parse#666).
+
+The pure, I/O-free pieces (PKCE, URL build, token-response parse, expiry math)
+are split out so they unit-test without a browser, a socket, or the network; the
+orchestration (`run_web_login`) wires them to a real loopback listener + the
+system browser. Loopback redirects are the only ones the backend allowlists for
+http, and the credential-store host gate accepts loopback — so a token captured
+on 127.0.0.1 can be persisted and then used against the configured Parse host.
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import http.server
+import secrets
+import threading
+import time
+import urllib.parse
+import webbrowser
+from dataclasses import dataclass
+from typing import Optional, Tuple
+
+import httpx
+
+# Default scope the backend advertises (scopes_supported: ["mcp"]).
+_SCOPE = "mcp"
+_REGISTER_TIMEOUT = 30.0
+_TOKEN_TIMEOUT = 30.0
+# How long to wait for the human to finish the browser consent before giving up.
+_CALLBACK_WAIT_SECONDS = 300.0
+
+
+@dataclass
+class TokenSet:
+    access_token: str
+    refresh_token: Optional[str]
+    expires_at: Optional[int]   # unix seconds; None when the server omits expires_in
+    client_id: Optional[str] = None
+
+
+# --------------------------------------------------------------------------- #
+# pure helpers (unit-tested without I/O)
+# --------------------------------------------------------------------------- #
+
+def generate_pkce() -> Tuple[str, str]:
+    """(code_verifier, code_challenge) for PKCE S256 — the exact transform the
+    backend verifies (urlsafe-b64 of sha256(verifier), '=' stripped)."""
+    verifier = base64.urlsafe_b64encode(secrets.token_bytes(32)).rstrip(b"=").decode()
+    challenge = base64.urlsafe_b64encode(
+        hashlib.sha256(verifier.encode()).digest()
+    ).rstrip(b"=").decode()
+    return verifier, challenge
+
+
+def build_authorize_url(
+    authorize_endpoint: str, *, client_id: str, redirect_uri: str,
+    code_challenge: str, state: str, scope: str = _SCOPE,
+) -> str:
+    """The full /oauth/authorize URL to open in the browser."""
+    q = urllib.parse.urlencode({
+        "response_type": "code",
+        "client_id": client_id,
+        "redirect_uri": redirect_uri,
+        "code_challenge": code_challenge,
+        "code_challenge_method": "S256",
+        "state": state,
+        "scope": scope,
+    })
+    sep = "&" if "?" in authorize_endpoint else "?"
+    return f"{authorize_endpoint}{sep}{q}"
+
+
+def parse_token_response(payload: object, *, client_id: Optional[str],
+                         now: int) -> TokenSet:
+    """Validate + normalize an /oauth/token JSON response into a TokenSet.
+
+    Raises ValueError on a missing/blank access_token (so a malformed exchange is
+    a clean error, not a TokenSet carrying None). expires_at is derived from
+    expires_in relative to ``now`` (caller passes time.time() — keeps this pure)."""
+    if not isinstance(payload, dict):
+        raise ValueError("token endpoint returned a non-object response")
+    access = payload.get("access_token")
+    if not isinstance(access, str) or not access:
+        raise ValueError("token endpoint response had no access_token")
+    refresh = payload.get("refresh_token")
+    expires_in = payload.get("expires_in")
+    expires_at = (now + int(expires_in)) if isinstance(expires_in, (int, float)) else None
+    return TokenSet(
+        access_token=access,
+        refresh_token=refresh if isinstance(refresh, str) and refresh else None,
+        expires_at=expires_at,
+        client_id=client_id,
+    )
+
+
+def is_expiring(expires_at: Optional[int], *, now: int, skew: int = 60) -> bool:
+    """True when a token is absent-of-expiry-safe to keep, or within ``skew`` of
+    expiry. None expiry → never proactively refreshed (treat as long-lived)."""
+    if expires_at is None:
+        return False
+    return now >= (expires_at - skew)
+
+
+# --------------------------------------------------------------------------- #
+# transport (thin; mocked in tests)
+# --------------------------------------------------------------------------- #
+
+def discover(base_url: str) -> dict:
+    """Fetch the authorization-server metadata (RFC 8414). Returns the endpoint
+    map; raises httpx.HTTPError / ValueError on failure."""
+    r = httpx.get(f"{base_url}/.well-known/oauth-authorization-server",
+                  timeout=_REGISTER_TIMEOUT)
+    r.raise_for_status()
+    data = r.json()
+    if not isinstance(data, dict):
+        raise ValueError("discovery returned a non-object")
+    return data
+
+
+def register_client(register_endpoint: str, redirect_uri: str,
+                    *, client_name: str = "Parse CLI") -> str:
+    """Dynamic client registration (RFC 7591) → client_id. The backend requires a
+    client whose redirect_uris include the exact loopback URI; registrations are
+    short-lived, so the CLI registers per-login."""
+    r = httpx.post(register_endpoint,
+                   json={"client_name": client_name, "redirect_uris": [redirect_uri]},
+                   timeout=_REGISTER_TIMEOUT)
+    r.raise_for_status()
+    data = r.json()
+    cid = data.get("client_id") if isinstance(data, dict) else None
+    if not isinstance(cid, str) or not cid:
+        raise ValueError("registration response had no client_id")
+    return cid
+
+
+def exchange_code(token_endpoint: str, *, code: str, code_verifier: str,
+                  redirect_uri: str, client_id: str) -> TokenSet:
+    r = httpx.post(token_endpoint, data={
+        "grant_type": "authorization_code", "code": code,
+        "code_verifier": code_verifier, "redirect_uri": redirect_uri,
+        "client_id": client_id,
+    }, timeout=_TOKEN_TIMEOUT)
+    r.raise_for_status()
+    return parse_token_response(r.json(), client_id=client_id, now=int(time.time()))
+
+
+def refresh_access_token(token_endpoint: str, *, refresh_token: str,
+                         client_id: Optional[str]) -> TokenSet:
+    data = {"grant_type": "refresh_token", "refresh_token": refresh_token}
+    if client_id:
+        data["client_id"] = client_id
+    r = httpx.post(token_endpoint, data=data, timeout=_TOKEN_TIMEOUT)
+    r.raise_for_status()
+    ts = parse_token_response(r.json(), client_id=client_id, now=int(time.time()))
+    # Some servers omit a rotated refresh token on refresh — keep the old one.
+    if ts.refresh_token is None:
+        ts.refresh_token = refresh_token
+    return ts
+
+
+# --------------------------------------------------------------------------- #
+# loopback callback server
+# --------------------------------------------------------------------------- #
+
+class _CallbackResult:
+    code: Optional[str] = None
+    state: Optional[str] = None
+    error: Optional[str] = None
+    # True once the loopback callback actually fired (a GET reached the handler),
+    # distinct from `handle_request()` returning on its socket timeout. Without
+    # this an abandoned login (browser closed) is indistinguishable from a
+    # successful-but-codeless callback, surfacing a misleading "no code" error.
+    handled: bool = False
+
+
+def _make_handler(result: _CallbackResult, expected_state: str):
+    class _Handler(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):  # noqa: N802 — http.server API
+            # RFC 8252 §8.3: reject any callback whose Host header is not the
+            # loopback interface — belt-and-suspenders against DNS-rebinding a
+            # malicious page onto the ephemeral callback port. The genuine
+            # browser redirect always carries Host: 127.0.0.1[:port] / localhost.
+            host = (self.headers.get("Host") or "").rsplit(":", 1)[0].strip("[]")
+            if host not in ("127.0.0.1", "localhost", "::1"):
+                self.send_response(400)
+                self.end_headers()
+                return
+            result.handled = True
+            parsed = urllib.parse.urlsplit(self.path)
+            qs = urllib.parse.parse_qs(parsed.query)
+            err = qs.get("error", [None])[0]
+            code = qs.get("code", [None])[0]
+            state = qs.get("state", [None])[0]
+            ok = False
+            if err:
+                result.error = err
+            elif not code:
+                result.error = "no authorization code in callback"
+            elif state != expected_state:
+                result.error = "state mismatch (possible CSRF) — login aborted"
+            else:
+                result.code, result.state, ok = code, state, True
+            body = (b"<html><body><h2>Parse login "
+                    + (b"complete" if ok else b"failed")
+                    + b"</h2>You can close this tab and return to the terminal.</body></html>")
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+        def log_message(self, format, *args):  # noqa: A002 — match base signature; silence access log
+            return
+
+    return _Handler
+
+
+def run_web_login(base_url: str, *, open_browser: bool = True) -> TokenSet:
+    """Drive the full loopback PKCE login against ``base_url`` and return a
+    TokenSet. Raises RuntimeError with an actionable message on any failure.
+
+    Requires a browser + the hosted consent page (the backend's /oauth/approve
+    needs a logged-in Parse session), so this is inherently interactive — the
+    pure/transport helpers above are what the unit tests exercise.
+    """
+    meta = discover(base_url)
+    authorize_ep = meta.get("authorization_endpoint")
+    token_ep = meta.get("token_endpoint")
+    register_ep = meta.get("registration_endpoint")
+    if not (isinstance(authorize_ep, str) and isinstance(token_ep, str)):
+        raise RuntimeError(f"{base_url} did not advertise OAuth authorize/token endpoints.")
+
+    # Bind a loopback listener on an ephemeral port FIRST so the redirect_uri is
+    # exact at registration time (the backend allowlists http://127.0.0.1[:port]).
+    result = _CallbackResult()
+    state = secrets.token_urlsafe(24)
+    server = http.server.HTTPServer(("127.0.0.1", 0), _make_handler(result, state))
+    port = server.server_address[1]
+    redirect_uri = f"http://127.0.0.1:{port}/callback"
+
+    try:
+        if not isinstance(register_ep, str):
+            raise RuntimeError(f"{base_url} did not advertise a registration endpoint.")
+        client_id = register_client(register_ep, redirect_uri)
+        verifier, challenge = generate_pkce()
+        url = build_authorize_url(authorize_ep, client_id=client_id,
+                                  redirect_uri=redirect_uri, code_challenge=challenge,
+                                  state=state)
+
+        print("Opening your browser to log in to Parse…")
+        print(f"  If it doesn't open, visit:\n  {url}")
+        if open_browser:
+            try:
+                webbrowser.open(url)
+            except Exception:
+                pass  # the printed URL is the fallback
+
+        # Serve exactly one request (the redirect), bounded by a wait timeout so a
+        # closed/abandoned browser doesn't hang the CLI forever. `server.timeout`
+        # makes `handle_request()` return on its socket select timeout when no
+        # request arrives — so `done` is set whether the callback fired or it
+        # timed out. The authoritative timeout signal is therefore `result.handled`
+        # (set inside the handler), NOT `done.wait()` (which all but always
+        # returns True here once the inner socket timeout elapses).
+        server.timeout = _CALLBACK_WAIT_SECONDS
+        done = threading.Event()
+
+        def _serve():
+            # Keep serving until a real callback is handled, bounded by the same
+            # wall-clock deadline — a rejected/stray GET (Host check, prefetch)
+            # must NOT consume the one login slot and abort the genuine redirect.
+            deadline = time.monotonic() + _CALLBACK_WAIT_SECONDS
+            while not result.handled and time.monotonic() < deadline:
+                server.handle_request()
+            done.set()
+
+        threading.Thread(target=_serve, daemon=True).start()
+        # +5 is a small slack over the socket timeout so the worker thread has
+        # exited cleanly; if even that elapses with the thread still alive, treat
+        # it as a timeout too.
+        done.wait(_CALLBACK_WAIT_SECONDS + 5)
+        if not result.handled:
+            raise RuntimeError(
+                f"login timed out after {int(_CALLBACK_WAIT_SECONDS)}s — no "
+                f"callback was received. Did you complete the login in the "
+                f"browser? Re-run `parse login --web` to try again.")
+        if result.error:
+            raise RuntimeError(f"login failed: {result.error}")
+        if not result.code:
+            raise RuntimeError("login did not return an authorization code.")
+
+        return exchange_code(token_ep, code=result.code, code_verifier=verifier,
+                             redirect_uri=redirect_uri, client_id=client_id)
+    finally:
+        server.server_close()

parse_sdk/_project.py

@@ -0,0 +1,184 @@
+"""The committed, declarative project config — the ``[tool.parse]`` table in the
+consumer ``pyproject.toml`` that records which APIs this repo wants.
+
+This is the DESIRED-state manifest: committed, reproducible across clones/CI,
+and the source of truth ``parse sync`` reconciles the generated tree to. It is
+distinct from ``parse_apis/_manifest.json`` (the GENERATED uuid→slug map, which
+is gitignored and rebuilt every sync).
+
+  * ``apis`` — account API *server slugs* this repo wants. The key being ABSENT
+    means "all your account APIs" (the post-``init`` default); an explicit list
+    (including the empty list) means exactly those.
+  * ``marketplace`` — locally-downloaded marketplace canonicals, each pinned to a
+    version (wired in the marketplace-add slice). Read-tolerant here so an older
+    CLI never trips over a newer field.
+
+We OWN the ``[tool.parse]`` table: writes regenerate it wholesale from the values
+we just read back, so a hand-edited value (e.g. a slug you added to ``apis``)
+survives, but formatting/comments *inside* the table are normalized. We never
+touch anything outside it.
+"""
+from __future__ import annotations
+
+import re
+try:
+    import tomllib  # Python 3.11+
+except ModuleNotFoundError:  # 3.10 — tomllib landed in 3.11; tomli is its backport
+    import tomli as tomllib  # type: ignore[no-redef]
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional
+
+
+@dataclass
+class MarketplaceEntry:
+    """A locally-downloaded marketplace canonical recorded for reproducible
+    re-sync. ``scraper_id`` (canonical_scraper_id) is the execution + schema
+    primary key; ``version`` pins execution via the ``API-Snapshot-Version``
+    header."""
+    scraper_id: str
+    slug: str
+    version: Optional[int] = None
+    listing_id: Optional[str] = None
+    source_url: Optional[str] = None
+
+
+@dataclass
+class ProjectConfig:
+    # None  → the `apis` key is absent  → "all your account APIs" (default).
+    # [...] → exactly these account server slugs ([] = no account APIs).
+    apis: Optional[List[str]] = None
+    marketplace: List[MarketplaceEntry] = field(default_factory=list)
+
+    @property
+    def all_account_mode(self) -> bool:
+        """True when the repo tracks ALL account APIs (apis key absent)."""
+        return self.apis is None
+
+    def account_desired(self, all_account_slugs: List[str]) -> List[str]:
+        """The account server-slugs this repo wants, given the full account set."""
+        return list(all_account_slugs) if self.apis is None else list(self.apis)
+
+
+# --------------------------------------------------------------------------- #
+# read
+# --------------------------------------------------------------------------- #
+
+def read_project_config(consumer_pyproject: Path) -> ProjectConfig:
+    """Parse ``[tool.parse]`` from the consumer pyproject. Total over a missing
+    file / malformed TOML / off-type fields → a permissive default (all-account,
+    no marketplace), never a raise: a hand-broken config must degrade to the
+    safe default, not crash ``parse sync``."""
+    try:
+        data = tomllib.loads(consumer_pyproject.read_text())
+    except (OSError, tomllib.TOMLDecodeError):
+        return ProjectConfig()
+    parse_tbl = (((data.get("tool") or {}).get("parse")) or {}) if isinstance(data, dict) else {}
+    if not isinstance(parse_tbl, dict):
+        return ProjectConfig()
+
+    apis: Optional[List[str]] = None
+    if "apis" in parse_tbl:
+        raw = parse_tbl.get("apis")
+        # An off-type `apis` (string, number) degrades to all-account rather than
+        # silently reconciling to an empty/garbage set that would prune the repo.
+        if isinstance(raw, list):
+            str_items = [s for s in raw if isinstance(s, str)]
+            # If the list contained any non-string entries, treat the whole value as
+            # malformed and fall back to all-account rather than silently producing a
+            # narrower (or empty) set the user never intended.
+            apis = str_items if len(str_items) == len(raw) else None
+        else:
+            apis = None
+
+    marketplace: List[MarketplaceEntry] = []
+    raw_mkt = parse_tbl.get("marketplace")
+    if isinstance(raw_mkt, list):
+        for row in raw_mkt:
+            if not isinstance(row, dict):
+                continue
+            sid, slug = row.get("scraper_id"), row.get("slug")
+            if isinstance(sid, str) and sid and isinstance(slug, str) and slug:
+                ver = row.get("version")
+                marketplace.append(MarketplaceEntry(
+                    scraper_id=sid, slug=slug,
+                    version=ver if isinstance(ver, int) else None,
+                    listing_id=row.get("listing_id") if isinstance(row.get("listing_id"), str) else None,
+                    source_url=row.get("source_url") if isinstance(row.get("source_url"), str) else None,
+                ))
+    return ProjectConfig(apis=apis, marketplace=marketplace)
+
+
+# --------------------------------------------------------------------------- #
+# write (regenerate the table we own)
+# --------------------------------------------------------------------------- #
+
+def _toml_str(s: str) -> str:
+    """A TOML basic string. Slugs/uuids are tame, but source_url is freer —
+    escape backslash and quote so the emitted table always re-parses."""
+    return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def render_parse_table(config: ProjectConfig) -> str:
+    """The full ``[tool.parse]`` block (+ any ``[[tool.parse.marketplace]]``
+    sub-tables). Trailing newline included."""
+    lines: List[str] = [
+        "# Managed by the parse CLI (`parse add` / `parse remove` / `parse sync`).",
+        "# Edit `apis` to curate which of your account APIs this repo syncs;",
+        "# an absent `apis` key means all of them.",
+        "[tool.parse]",
+    ]
+    if config.apis is not None:
+        if config.apis:
+            lines.append("apis = [")
+            for slug in config.apis:
+                lines.append(f"    {_toml_str(slug)},")
+            lines.append("]")
+        else:
+            lines.append("apis = []")
+    for m in config.marketplace:
+        lines.append("")
+        lines.append("[[tool.parse.marketplace]]")
+        lines.append(f"scraper_id = {_toml_str(m.scraper_id)}")
+        lines.append(f"slug = {_toml_str(m.slug)}")
+        if m.version is not None:
+            lines.append(f"version = {int(m.version)}")
+        if m.listing_id:
+            lines.append(f"listing_id = {_toml_str(m.listing_id)}")
+        if m.source_url:
+            lines.append(f"source_url = {_toml_str(m.source_url)}")
+    return "\n".join(lines) + "\n"
+
+
+def _strip_parse_tables(text: str) -> str:
+    """Remove the existing ``[tool.parse]`` / ``[tool.parse.*]`` /
+    ``[[tool.parse.*]]`` blocks (header line through the line before the next
+    table header or EOF), leaving everything else byte-for-byte."""
+    out: List[str] = []
+    skipping = False
+    header_re = re.compile(r"^\s*\[\[?\s*([^\]]+?)\s*\]\]?\s*(?:#.*)?$")
+    for line in text.splitlines(keepends=True):
+        m = header_re.match(line)
+        if m:
+            name = m.group(1)
+            skipping = name == "tool.parse" or name.startswith("tool.parse.")
+        if not skipping:
+            out.append(line)
+    return "".join(out)
+
+
+def write_project_config(consumer_pyproject: Path, config: ProjectConfig) -> None:
+    """Persist ``config`` into the consumer pyproject's ``[tool.parse]`` table,
+    regenerating only that table. Raises ValueError if the result wouldn't be
+    valid TOML (invariant: we never write a broken pyproject)."""
+    original = consumer_pyproject.read_text()
+    stripped = _strip_parse_tables(original).rstrip()
+    block = render_parse_table(config)
+    new = (stripped + "\n\n" + block) if stripped else block
+    try:
+        tomllib.loads(new)
+    except tomllib.TOMLDecodeError as e:  # pragma: no cover — defensive invariant
+        raise ValueError(f"refusing to write pyproject.toml — the [tool.parse] edit "
+                         f"would corrupt it ({e}).") from e
+    consumer_pyproject.write_text(new)