data360-autodoc 0.2.0__py3-none-any.whl

data360_autodoc/__init__.py

@@ -0,0 +1 @@
+"""data360-autodoc: auto-generate documentation for Salesforce Data 360 orgs."""

data360_autodoc/cli/__init__.py

@@ -0,0 +1 @@
+"""Command-line interface for data360-autodoc."""

data360_autodoc/cli/main.py

@@ -0,0 +1,209 @@
+"""``data360-autodoc`` command-line entry point.
+
+Orchestrates the one-shot pipeline::
+
+    auth (JWT bearer) -> fetch metadata -> render outputs -> write files
+
+Outputs are selected with ``--format``:
+
+- ``markdown`` — human-readable ``.md`` plus a ``.mmd`` Mermaid diagram
+- ``json``     — deterministic ``.json`` snapshot (the drift-detection seam)
+- ``pdf``      — not yet implemented (stub; warns and skips)
+- ``all``      — markdown + json (+ pdf stub warning)
+
+The ``.json`` snapshot is a first-class output, not a debug artifact: the paid
+drift tier loads a prior snapshot and diffs it against a fresh fetch.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import click
+
+from data360_autodoc.fetcher.auth import (
+    AUD_PRODUCTION,
+    AUD_SANDBOX,
+    DEFAULT_TOKEN_URL,
+    SANDBOX_TOKEN_URL,
+    AuthError,
+    get_access_token,
+)
+from data360_autodoc.fetcher.metadata import MetadataError, fetch_metadata
+from data360_autodoc.generator.markdown import render_markdown
+from data360_autodoc.generator.mermaid import render_mermaid
+from data360_autodoc.generator.snapshot import render_json
+
+#: Valid values for the ``--format`` option.
+FORMATS = ["markdown", "json", "pdf", "all"]
+
+
+@click.group()
+def cli() -> None:
+    """Auto-generate documentation for Salesforce Data 360 orgs."""
+
+
+@cli.command()
+@click.option("--instance-url", required=True, help="Org base URL.")
+@click.option(
+    "--access-token",
+    default=None,
+    help="Use a pre-obtained OAuth access token and skip JWT auth. When set, "
+    "--client-id / --private-key / --username are not needed.",
+)
+@click.option(
+    "--client-id", default=None, help="Connected app consumer key (JWT auth)."
+)
+@click.option(
+    "--private-key",
+    "private_key_path",
+    default=None,
+    type=click.Path(exists=True, dir_okay=False),
+    help="Path to the connected app's PEM private key (JWT auth).",
+)
+@click.option(
+    "--username", default=None, help="Salesforce username to impersonate (JWT auth)."
+)
+@click.option(
+    "--output",
+    "output_dir",
+    default=".",
+    type=click.Path(file_okay=False),
+    help="Directory to write output files into (created if missing).",
+)
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(FORMATS),
+    default="all",
+    show_default=True,
+    help="Which artifacts to generate.",
+)
+@click.option(
+    "--sandbox",
+    is_flag=True,
+    default=False,
+    help="Authenticate against test.salesforce.com (sandbox/scratch orgs).",
+)
+@click.option(
+    "--api-version",
+    "api_version",
+    default=None,
+    help="Data API version (e.g. v62.0). Default: auto-detect the org's highest.",
+)
+@click.option(
+    "--timeout",
+    default=120.0,
+    show_default=True,
+    type=float,
+    help="Per-request timeout (seconds) for metadata calls. Data Cloud is slow.",
+)
+def generate(
+    instance_url: str,
+    access_token: str | None,
+    client_id: str | None,
+    private_key_path: str | None,
+    username: str | None,
+    output_dir: str,
+    output_format: str,
+    sandbox: bool,
+    api_version: str | None,
+    timeout: float,
+) -> None:
+    """Fetch an org's metadata and write documentation artifacts."""
+    if access_token:
+        token = access_token
+        org_url = instance_url
+    else:
+        missing = [
+            flag
+            for flag, value in (
+                ("--client-id", client_id),
+                ("--private-key", private_key_path),
+                ("--username", username),
+            )
+            if not value
+        ]
+        if missing:
+            raise click.UsageError(
+                "Provide --access-token, or all of --client-id, --private-key, "
+                f"--username for JWT auth. Missing: {', '.join(missing)}."
+            )
+        token_url = SANDBOX_TOKEN_URL if sandbox else DEFAULT_TOKEN_URL
+        audience = AUD_SANDBOX if sandbox else AUD_PRODUCTION
+        try:
+            auth = get_access_token(
+                instance_url=instance_url,
+                client_id=client_id,
+                private_key_path=private_key_path,
+                username=username,
+                token_url=token_url,
+                audience=audience,
+            )
+        except AuthError as exc:
+            raise click.ClickException(str(exc)) from exc
+        token = auth["access_token"]
+        org_url = auth["instance_url"]
+
+    def _progress(message: str, inline: bool) -> None:
+        # Progress goes to stderr so it never pollutes the doc output. inline
+        # uses a carriage return to update the DMO counter in place.
+        if inline:
+            click.echo(f"\r{message}", nl=False, err=True)
+        else:
+            click.echo(message, err=True)
+
+    try:
+        schema = fetch_metadata(
+            instance_url=org_url,
+            access_token=token,
+            api_version=api_version,
+            timeout=timeout,
+            progress=_progress,
+        )
+    except MetadataError as exc:
+        # Surface a clean one-line error and exit non-zero — never a traceback.
+        raise click.ClickException(str(exc)) from exc
+
+    out_dir = Path(output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    base = _slug(schema.org_name) or "data360"
+
+    written: list[str] = []
+    want_markdown = output_format in ("markdown", "all")
+    want_json = output_format in ("json", "all")
+    want_pdf = output_format in ("pdf", "all")
+
+    if want_markdown:
+        md_path = out_dir / f"{base}.md"
+        md_path.write_text(render_markdown(schema), encoding="utf-8")
+        written.append(md_path.name)
+        mmd_path = out_dir / f"{base}.mmd"
+        mmd_path.write_text(render_mermaid(schema) + "\n", encoding="utf-8")
+        written.append(mmd_path.name)
+
+    if want_json:
+        json_path = out_dir / f"{base}.json"
+        json_path.write_text(render_json(schema), encoding="utf-8")
+        written.append(json_path.name)
+
+    if want_pdf:
+        click.echo("PDF output is not yet implemented (stub) — skipping.", err=True)
+
+    for name in written:
+        click.echo(f"Wrote {name}")
+    click.echo(
+        f"Generated docs for {len(schema.dmos)} DMOs, "
+        f"{len(schema.dlos)} DLOs, "
+        f"{len(schema.identity_rulesets)} Identity Rulesets"
+    )
+
+
+def _slug(value: str) -> str:
+    """Slugify an org name for use as an output filename stem."""
+    return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+
+
+if __name__ == "__main__":  # pragma: no cover
+    cli()

data360_autodoc/fetcher/__init__.py

@@ -0,0 +1 @@
+"""Salesforce Data 360 API client package."""

data360_autodoc/fetcher/_http.py

@@ -0,0 +1,110 @@
+"""Shared HTTP helpers for the Data 360 Connect REST API clients.
+
+All ``/ssot/*`` fetchers share the same needs: a bearer-authenticated GET with
+exponential-backoff retry, and ``nextPageUrl`` pagination with a cycle guard.
+Centralizing them here keeps the per-endpoint clients (``metadata.py``,
+``streams.py``) small and consistent.
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any, Final, Iterator
+
+import requests
+
+_MAX_ATTEMPTS: Final = 3
+_BACKOFF_BASE_SECONDS: Final = 1.0
+
+
+class FetchError(RuntimeError):
+    """Raised when a Data 360 API request fails (4xx, or retries exhausted)."""
+
+
+def get_json(url: str, *, access_token: str, timeout: float) -> Any:
+    """GET ``url`` with bearer auth, retrying transient failures.
+
+    Retries transport errors and 5xx responses up to three times with
+    exponential backoff (1s, 2s, 4s). 4xx responses are terminal.
+
+    Args:
+        url: Absolute URL to request.
+        access_token: OAuth bearer token.
+        timeout: Per-request timeout in seconds.
+
+    Returns:
+        The decoded JSON body.
+
+    Raises:
+        FetchError: On a 4xx response or after exhausting retries.
+    """
+    headers = {"Authorization": f"Bearer {access_token}", "Accept": "application/json"}
+    last_error: str | None = None
+    for attempt in range(_MAX_ATTEMPTS):
+        try:
+            response = requests.get(url, headers=headers, timeout=timeout)
+        except requests.RequestException as exc:
+            last_error = str(exc)
+        else:
+            if response.status_code == 200:
+                try:
+                    return response.json()
+                except ValueError as exc:
+                    # A 200 with a non-JSON body (e.g. a proxy/login HTML page)
+                    # must become a FetchError, not leak a raw JSONDecodeError
+                    # past the CLI's clean-error handler.
+                    raise FetchError(
+                        f"Non-JSON 200 response from {url}: {exc}"
+                    ) from exc
+            if 400 <= response.status_code < 500:
+                raise FetchError(
+                    f"Request rejected ({response.status_code}) for {url}: "
+                    f"{response.text[:500]}"
+                )
+            last_error = f"HTTP {response.status_code}: {response.text[:500]}"
+        if attempt < _MAX_ATTEMPTS - 1:
+            time.sleep(_BACKOFF_BASE_SECONDS * (2**attempt))
+    raise FetchError(
+        f"Request to {url} failed after {_MAX_ATTEMPTS} attempts: {last_error}"
+    )
+
+
+def iter_pages(
+    first_url: str, *, base_url: str, access_token: str, timeout: float
+) -> Iterator[dict[str, Any]]:
+    """Yield each page of a paginated ``/ssot/*`` response.
+
+    Follows each page's ``nextPageUrl`` (absolute or relative) until exhausted,
+    guarding against a self-referential or repeating link that would otherwise
+    loop forever.
+
+    Args:
+        first_url: Absolute URL of the first page.
+        base_url: Org base URL, used to resolve relative ``nextPageUrl`` values.
+        access_token: OAuth bearer token.
+        timeout: Per-request timeout in seconds.
+
+    Yields:
+        Each page's decoded JSON body.
+
+    Raises:
+        FetchError: On a request failure or a detected pagination cycle.
+    """
+    seen: set[str] = set()
+    next_url: str | None = first_url
+    while next_url:
+        if next_url in seen:
+            raise FetchError(f"Pagination cycle detected at {next_url}")
+        seen.add(next_url)
+        page = get_json(next_url, access_token=access_token, timeout=timeout)
+        yield page
+        next_url = _resolve_next(page.get("nextPageUrl"), base_url)
+
+
+def _resolve_next(raw: Any, base_url: str) -> str | None:
+    """Normalize a ``nextPageUrl`` value to an absolute URL, or ``None``."""
+    if not raw or not isinstance(raw, str):
+        return None
+    if raw.startswith("http://") or raw.startswith("https://"):
+        return raw
+    return f"{base_url.rstrip('/')}/{raw.lstrip('/')}"

data360_autodoc/fetcher/auth.py

@@ -0,0 +1,161 @@
+"""OAuth 2.0 JWT Bearer Flow for Salesforce Data 360.
+
+This module performs the server-to-server JWT bearer flow only (per project
+policy: never store user passwords or client secrets). The caller supplies a
+connected-app ``client_id``, the path to the app's private key, and the
+``username`` to impersonate; we mint a short-lived signed assertion and exchange
+it for an access token.
+
+Reference: https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_jwt_flow.htm
+"""
+
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Final
+
+import jwt
+import requests
+
+#: OAuth grant type identifier for the JWT bearer flow.
+_GRANT_TYPE: Final = "urn:ietf:params:oauth:grant-type:jwt-bearer"
+#: Audience for production logins.
+AUD_PRODUCTION: Final = "https://login.salesforce.com"
+#: Audience for sandbox / scratch-org logins.
+AUD_SANDBOX: Final = "https://test.salesforce.com"
+#: Default token endpoint for production JWT bearer exchange.
+DEFAULT_TOKEN_URL: Final = "https://login.salesforce.com/services/oauth2/token"
+#: Token endpoint for sandbox / scratch-org JWT bearer exchange.
+SANDBOX_TOKEN_URL: Final = "https://test.salesforce.com/services/oauth2/token"
+#: Assertion lifetime in seconds (Salesforce caps this at 3 minutes).
+_ASSERTION_TTL_SECONDS: Final = 180
+#: Number of token-exchange attempts before giving up.
+_MAX_ATTEMPTS: Final = 3
+#: Base delay (seconds) for exponential backoff: 1s, 2s, 4s.
+_BACKOFF_BASE_SECONDS: Final = 1.0
+
+
+class AuthError(RuntimeError):
+    """Raised when the JWT bearer token exchange ultimately fails."""
+
+
+def build_assertion(
+    *,
+    client_id: str,
+    username: str,
+    private_key: str,
+    audience: str = AUD_PRODUCTION,
+    now: int | None = None,
+) -> str:
+    """Build and sign the JWT assertion for the bearer flow.
+
+    Args:
+        client_id: The connected app's consumer key (``iss`` claim).
+        username: The Salesforce username to impersonate (``sub`` claim).
+        private_key: PEM-encoded RSA private key contents used to sign (RS256).
+        audience: Token endpoint audience; use :data:`AUD_PRODUCTION` or
+            :data:`AUD_SANDBOX`.
+        now: Override for the current epoch seconds (used in tests for
+            deterministic ``exp`` claims).
+
+    Returns:
+        The encoded, signed JWT assertion as a compact string.
+    """
+    issued_at = int(time.time()) if now is None else now
+    claims = {
+        "iss": client_id,
+        "sub": username,
+        "aud": audience,
+        "exp": issued_at + _ASSERTION_TTL_SECONDS,
+    }
+    return jwt.encode(claims, private_key, algorithm="RS256")
+
+
+def get_access_token(
+    *,
+    instance_url: str,
+    client_id: str,
+    private_key_path: str,
+    username: str,
+    token_url: str = DEFAULT_TOKEN_URL,
+    audience: str = AUD_PRODUCTION,
+    timeout: float = 30.0,
+) -> dict[str, str]:
+    """Obtain an access token via the JWT bearer flow.
+
+    Retries the token exchange up to three times with exponential backoff
+    (1s, 2s, 4s) on transient transport errors or 5xx responses. Client errors
+    (4xx) are treated as terminal and surfaced immediately.
+
+    Args:
+        instance_url: Base URL of the Salesforce org (e.g.
+            ``https://mydomain.my.salesforce.com``). Used as the fallback
+            ``instance_url`` in the return value when the org does not echo one.
+        client_id: The connected app's consumer key.
+        private_key_path: Filesystem path to the PEM private key.
+        username: The Salesforce username to impersonate.
+        token_url: The OAuth token endpoint to POST the assertion to. Defaults
+            to :data:`DEFAULT_TOKEN_URL`
+            (``https://login.salesforce.com/services/oauth2/token``). **Sandbox
+            and scratch orgs must override this** with
+            :data:`SANDBOX_TOKEN_URL`
+            (``https://test.salesforce.com/services/oauth2/token``).
+        audience: JWT ``aud`` claim; :data:`AUD_PRODUCTION` (default) or
+            :data:`AUD_SANDBOX` for sandboxes. Should match ``token_url``.
+        timeout: Per-request timeout in seconds.
+
+    Returns:
+        A dict with ``access_token`` and ``instance_url`` keys. The
+        ``instance_url`` reflects the value returned by Salesforce when present
+        (it may differ from the input), otherwise the input value.
+
+    Raises:
+        FileNotFoundError: If ``private_key_path`` does not exist.
+        AuthError: If the token exchange fails after all retries, the org
+            returns a non-retryable error, or the success response is missing
+            ``access_token``.
+    """
+    private_key = Path(private_key_path).read_text(encoding="utf-8")
+    assertion = build_assertion(
+        client_id=client_id,
+        username=username,
+        private_key=private_key,
+        audience=audience,
+    )
+    data = {"grant_type": _GRANT_TYPE, "assertion": assertion}
+
+    last_error: str | None = None
+    for attempt in range(_MAX_ATTEMPTS):
+        try:
+            response = requests.post(token_url, data=data, timeout=timeout)
+        except requests.RequestException as exc:  # transport-level failure
+            last_error = str(exc)
+        else:
+            if response.status_code == 200:
+                body = response.json()
+                try:
+                    access_token = body["access_token"]
+                except KeyError as exc:
+                    raise AuthError(
+                        "Unexpected response: access_token not found"
+                    ) from exc
+                return {
+                    "access_token": access_token,
+                    "instance_url": body.get("instance_url", instance_url),
+                }
+            # 4xx: bad assertion / config — retrying will not help.
+            if 400 <= response.status_code < 500:
+                raise AuthError(
+                    f"JWT bearer exchange rejected ({response.status_code}): "
+                    f"{response.text}"
+                )
+            last_error = f"HTTP {response.status_code}: {response.text}"
+
+        # Back off before the next attempt, but never after the final one.
+        if attempt < _MAX_ATTEMPTS - 1:
+            time.sleep(_BACKOFF_BASE_SECONDS * (2**attempt))
+
+    raise AuthError(
+        f"JWT bearer exchange failed after {_MAX_ATTEMPTS} attempts: {last_error}"
+    )