atlan-application-sdk-api 3.51.0__py3-none-any.whl

application_sdk/clients/ssl_utils.py

@@ -0,0 +1,247 @@
+"""SSL utilities for HTTP clients."""
+
+import functools
+import os
+import ssl
+
+from application_sdk._logging import get_logger
+from application_sdk.constants import SSL_CERT_DIR
+
+logger = get_logger(__name__)
+
+# Supported certificate file extensions
+CERT_FILE_EXTENSIONS = (".pem", ".crt", ".cer", ".ca-bundle")
+
+
+def get_ssl_cert_dir() -> str | None:
+    """
+    Get the SSL certificate directory from configuration.
+
+    Returns:
+        The SSL certificate directory path if SSL_CERT_DIR is set and the directory exists,
+        None otherwise.
+    """
+    if SSL_CERT_DIR and os.path.isdir(SSL_CERT_DIR):
+        logger.debug("Using SSL certificates from directory: %s", SSL_CERT_DIR)
+        return SSL_CERT_DIR
+    return None
+
+
+def get_certificate_files(cert_dir: str) -> list[str]:
+    """
+    Get all certificate files from a directory.
+
+    Args:
+        cert_dir: Directory to search for certificate files
+
+    Returns:
+        List of full paths to certificate files
+    """
+    cert_files: list[str] = []
+    for filename in os.listdir(cert_dir):
+        if filename.lower().endswith(CERT_FILE_EXTENSIONS):
+            cert_files.append(os.path.join(cert_dir, filename))
+    return cert_files
+
+
+def create_ssl_context_with_custom_certs(cert_dir: str) -> ssl.SSLContext:
+    """
+    Create an SSL context that includes both default system certificates and custom certificates.
+
+    This ensures that connections to public services (using well-known CAs) continue to work,
+    while also allowing connections to private services using custom/internal certificates.
+
+    The function loads all .pem, .crt, .cer, and .ca-bundle files from the specified directory.
+
+    Args:
+        cert_dir: Directory containing additional certificate files (.pem, .crt, .cer, .ca-bundle)
+
+    Returns:
+        SSL context with both default and custom certificates loaded
+    """
+    ssl_context = ssl.create_default_context()
+
+    cert_files = get_certificate_files(cert_dir)
+
+    for cert_file in cert_files:
+        try:
+            ssl_context.load_verify_locations(cafile=cert_file)
+            logger.debug("Loaded certificate from: %s", cert_file)
+        except ssl.SSLError:
+            logger.warning(
+                "Failed to load certificate from %s", cert_file, exc_info=True
+            )
+
+    if cert_files:
+        logger.debug(
+            "Created SSL context with default certificates and %d custom certificate(s) from: %s",
+            len(cert_files),
+            cert_dir,
+        )
+    else:
+        logger.debug(
+            "Created SSL context with default certificates (no custom certificates found in %s)",
+            cert_dir,
+        )
+
+    return ssl_context
+
+
+@functools.lru_cache(maxsize=1)
+def get_ssl_context() -> bool | ssl.SSLContext:
+    """
+    Get the SSL verification context for HTTP clients (httpx, aiohttp, etc.).
+
+    If SSL_CERT_DIR is set and points to a valid directory, returns an SSLContext
+    that includes both default system certificates AND custom certificates from that directory.
+    This allows connections to both public services and private services with custom CAs.
+
+    Returns:
+        An SSLContext with combined certificates if SSL_CERT_DIR is set,
+        True otherwise for default SSL verification.
+
+    Example:
+        >>> import httpx
+        >>> ssl_context = get_ssl_context()
+        >>> async with httpx.AsyncClient(verify=ssl_context) as client:
+        ...     response = await client.get("https://example.com")
+
+        >>> import aiohttp
+        >>> ssl_context = get_ssl_context()
+        >>> async with aiohttp.ClientSession() as session:
+        ...     async with session.get("https://example.com", ssl=ssl_context) as response:
+        ...         pass
+    """
+    ssl_cert_dir = get_ssl_cert_dir()
+    if ssl_cert_dir:
+        return create_ssl_context_with_custom_certs(ssl_cert_dir)
+    return True
+
+
+def _get_default_ca_bundle_path() -> str | None:
+    """
+    Get the path to the default CA certificate bundle.
+
+    Tries multiple sources in order:
+    1. certifi package (if available) - provides Mozilla CA bundle
+    2. SSL default verify paths from the system
+
+    Returns:
+        Path to the default CA bundle file, or None if not found.
+    """
+    try:
+        import certifi  # noqa: PLC0415 — optional dep: certifi
+
+        certifi_path = certifi.where()
+        if certifi_path and os.path.isfile(certifi_path):
+            logger.debug("Using certifi CA bundle: %s", certifi_path)
+            return certifi_path
+    except ImportError:
+        logger.debug("certifi not available, falling back to system CA paths")
+
+    default_paths = ssl.get_default_verify_paths()
+
+    if default_paths.cafile and os.path.isfile(default_paths.cafile):
+        logger.debug("Using system CA file: %s", default_paths.cafile)
+        return default_paths.cafile
+
+    if default_paths.openssl_cafile and os.path.isfile(default_paths.openssl_cafile):
+        logger.debug("Using OpenSSL CA file: %s", default_paths.openssl_cafile)
+        return default_paths.openssl_cafile
+
+    logger.debug("No default CA bundle found")
+    return None
+
+
+def _read_default_ca_certs() -> bytes | None:
+    """
+    Read the default system CA certificates as bytes.
+
+    Returns:
+        The default CA certificates as bytes, or None if not available.
+    """
+    ca_bundle_path = _get_default_ca_bundle_path()
+    if not ca_bundle_path:
+        return None
+
+    try:
+        with open(ca_bundle_path, "rb") as f:
+            ca_bytes = f.read()
+            if ca_bytes:
+                logger.debug(
+                    "Read default CA certificates (%d bytes) from: %s",
+                    len(ca_bytes),
+                    ca_bundle_path,
+                )
+                return ca_bytes
+    except OSError:
+        logger.warning(
+            "Failed to read default CA bundle from %s", ca_bundle_path, exc_info=True
+        )
+
+    return None
+
+
+def get_custom_ca_cert_bytes() -> bytes | None:
+    """
+    Get CA certificate bytes combining default system certificates AND custom certificates.
+
+    This is useful for clients like Temporal that use their own TLS implementation
+    and require certificate data as bytes rather than an ssl.SSLContext.
+
+    If SSL_CERT_DIR is set and points to a valid directory containing certificate files,
+    this function reads all certificate files and concatenates them with the default
+    system CA certificates. This ensures that both public services (using well-known CAs)
+    and private services (using custom CAs) are trusted.
+
+    Returns:
+        Combined certificate bytes (default + custom) if SSL_CERT_DIR is set
+        and contains valid certificate files, None otherwise.
+
+    Example:
+        >>> from temporalio.service import TLSConfig
+        >>> ca_cert_bytes = get_custom_ca_cert_bytes()
+        >>> if ca_cert_bytes:
+        ...     tls_config = TLSConfig(server_root_ca_cert=ca_cert_bytes)
+    """
+    ssl_cert_dir = get_ssl_cert_dir()
+    if not ssl_cert_dir:
+        return None
+
+    cert_files = get_certificate_files(ssl_cert_dir)
+    if not cert_files:
+        logger.debug("No certificate files found in %s", ssl_cert_dir)
+        return None
+
+    cert_bytes_list: list[bytes] = []
+
+    default_ca_bytes = _read_default_ca_certs()
+    if default_ca_bytes:
+        cert_bytes_list.append(default_ca_bytes)
+        logger.debug("Added default system CA certificates to trust store")
+
+    custom_cert_count = 0
+    for cert_file in cert_files:
+        try:
+            with open(cert_file, "rb") as f:
+                cert_data = f.read()
+                if cert_data:
+                    cert_bytes_list.append(cert_data)
+                    custom_cert_count += 1
+                    logger.debug("Read custom certificate from: %s", cert_file)
+        except OSError:
+            logger.warning(
+                "Failed to read certificate from %s", cert_file, exc_info=True
+            )
+
+    if not cert_bytes_list:
+        return None
+
+    combined_certs = b"\n".join(cert_bytes_list)
+    logger.debug(
+        "Combined default CA certificates and %d custom certificate(s) from %s (%d bytes total)",
+        custom_cert_count,
+        ssl_cert_dir,
+        len(combined_certs),
+    )
+    return combined_certs

application_sdk/common/__init__.py

@@ -0,0 +1,53 @@
+"""Shared utilities for Atlan application connectors.
+
+Public API::
+
+    from application_sdk.common import (
+        # SQL filter helpers
+        read_sql_files, prepare_query, prepare_filters, normalize_filters,
+        # App-agnostic filter matching (regex vs exact)
+        FilterPattern, filter_matches,
+        # Concurrency helpers
+        get_actual_cpu_count, get_safe_num_threads,
+        # Task models
+        TaskStatistics, TaskResult,
+        # DataFrame type enum
+        DataframeType,
+    )
+
+Note: ``application_sdk.common.error_codes`` is retained for back-compat with
+v2 connectors. Prefer ``application_sdk.errors`` for new code.
+"""
+
+from application_sdk.common.concurrency import (
+    get_actual_cpu_count,
+    get_safe_num_threads,
+)
+from application_sdk.common.filter_matching import FilterPattern, filter_matches
+from application_sdk.common.models import TaskResult, TaskStatistics
+from application_sdk.common.sql_filters import (
+    normalize_filters,
+    prepare_filters,
+    prepare_query,
+    read_sql_files,
+)
+from application_sdk.common.types import DataframeType
+
+__all__ = [
+    # SQL filter helpers
+    "read_sql_files",
+    "prepare_query",
+    "prepare_filters",
+    "normalize_filters",
+    # App-agnostic filter matching (regex vs exact)
+    "FilterPattern",
+    "filter_matches",
+    # Concurrency helpers
+    "get_actual_cpu_count",
+    "get_safe_num_threads",
+    # Task models
+    "TaskStatistics",
+    "TaskResult",
+    # DataFrame type enum
+    "DataframeType",
+]

application_sdk/common/_generated_tree.py

@@ -0,0 +1,321 @@
+"""The shape of the committed ``app/generated/`` tree, in one place.
+
+Three facts about that tree were, until FND-1667, each written down twice:
+
+* **which layout the tree has** — a bundle nests one directory per entry point,
+  while a single-generated-contract app (including a route/card-split app with
+  several ``@entrypoint``\\ s behind one card) emits everything flat.
+  :func:`generated_layout` is the authority, and
+  :mod:`application_sdk.app._artifact_schema_guard` delegates to it.
+* **which of the sibling ``*.json`` files is a setup form** — as opposed to the
+  DAG ``manifest`` or a per-object-store-family credential template.
+  :func:`is_form_configmap` is the authority, and
+  :mod:`application_sdk.handler.service` delegates to it.
+* **where an entry point's form actually lives**, given the two above.
+  :func:`form_configmap` is the authority; the tenant-side route check in
+  :mod:`application_sdk.testing.setup_routes` is its first caller.
+
+Keeping them here is not tidying. The route check added in FND-1667 asserts a
+join between what an app's contract generates and what its tenant serves, and
+the SDK is on *both* sides of that join: ``handler.service.get_configmap`` is
+the endpoint the tenant's ``/api/service/configmaps/<name>`` proxies to. A
+check that re-derived "which file is the form" would be comparing one guess
+against another, and would drift from the server the moment the exclusion
+vocabulary grew a prefix. There is one copy of each fact, and the check reads
+the same copy the server does.
+
+Deliberately dependency-light: stdlib only, no logger, no constants import. It
+is imported by the request path in ``handler.service`` and by a CI-time check,
+so it must cost nothing and pull nothing.
+
+Private (leading underscore), like its siblings ``_artifact_schema_guard`` and
+``_ep_registration``: these are the SDK's own rules about its own generated
+tree, not a surface a consumer app should pin against. Nothing here is
+re-exported from ``application_sdk.app``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Literal
+
+__all__ = [
+    "ARTIFACT_SCHEMAS_STEM",
+    "CREDENTIAL_TEMPLATE_PREFIXES",
+    "GeneratedLayout",
+    "MANIFEST_STEM",
+    "NON_FORM_STEMS",
+    "choose_form_configmap",
+    "eligible_form_configmaps",
+    "entrypoint_dirs",
+    "form_configmap",
+    "generated_layout",
+    "is_form_configmap",
+    "names_entrypoint",
+    "pick_form_configmap",
+]
+
+#: Stem of the DAG manifest that sits alongside the generated setup forms.
+MANIFEST_STEM = "manifest"
+
+#: Stem of the artifact-schema declarations the toolkit emits for an app with an
+#: ``artifactSchemas`` block (conformance K016). Read by
+#: :mod:`application_sdk.validation.sources` and
+#: :mod:`application_sdk.app._artifact_schema_guard`; named here because it is
+#: also a *non-form* sibling and form discovery must skip it.
+ARTIFACT_SCHEMAS_STEM = "artifact_schemas"
+
+#: Whole stems — as opposed to the family prefixes below — that are never a
+#: setup form.
+#:
+#: ``artifact_schemas`` earns its place the hard way (FND-1682): it sorts before
+#: every real form stem, so once an app declared ``artifactSchemas`` the
+#: alphabetical fallback in ``handler.service.get_configmap`` served the schema
+#: document as the form. It has no ``config`` key and no ``properties``, so the
+#: setup wizard rendered blank behind an HTTP 200 — nothing in the logs, the
+#: network tab or pod stderr.
+NON_FORM_STEMS = frozenset({MANIFEST_STEM, ARTIFACT_SCHEMAS_STEM})
+
+#: Non-form JSON siblings that live in the generated dir next to the setup-form
+#: configmaps. Credential templates are emitted per object-store family
+#: (``atlan-connectors-*.json``, ``csa-connectors-*.json``).
+#:
+#: Centralised so the form-discovery exclusion vocabulary is named once instead
+#: of re-spelled at each site: adding the next connector-family prefix here
+#: updates the server's fallback (``handler.service.get_configmap``) and the
+#: tenant-side route check together. They disagreeing is the failure this
+#: constant exists to make impossible — the server would serve one file and the
+#: check would compare against another, and the mismatch would read as a
+#: contract regression rather than as two different exclusion lists.
+CREDENTIAL_TEMPLATE_PREFIXES = ("atlan-connectors-", "csa-connectors-")
+
+#: Shape of the committed ``app/generated/`` tree — the authority on where an
+#: entry point's generated artifacts live. Mirrors conformance K016's contract
+#: scan.
+GeneratedLayout = Literal["multi", "single", "unknown"]
+
+
+def is_form_configmap(stem: str) -> bool:
+    """Whether a generated JSON stem is a setup-form configmap.
+
+    True when *stem* is neither one of the well-known non-form documents
+    (:data:`NON_FORM_STEMS`) nor a credential template.
+
+    **This list can only ever name the siblings the toolkit emits today.**
+    FND-1682 is what a stale one costs: ``artifact_schemas.json`` was not on it,
+    sorted before every real form stem, and so was served as the form by every
+    app that adopted conformance K016. A caller with an entry-point name to go
+    on should ask :func:`pick_form_configmap`, which prefers a form that
+    :func:`names_entrypoint` over anything this filter merely fails to reject.
+
+    Args:
+        stem: A generated file's name without its ``.json`` suffix.
+    """
+    return stem not in NON_FORM_STEMS and not stem.startswith(
+        CREDENTIAL_TEMPLATE_PREFIXES
+    )
+
+
+def names_entrypoint(stem: str, entrypoint: str) -> bool:
+    """Whether a form's *stem* identifies itself as *entrypoint*'s.
+
+    Two spellings are in the field, and both are identifications rather than
+    guesses:
+
+    * ``<entrypoint>`` — a flat app whose form is named for its entry point
+      (``bridge.json`` for ``bridge``).
+    * ``<source>-<entrypoint>`` — the connector convention, where the entry
+      point is the *role* and the file carries the source (``crawler`` →
+      ``snowflake-crawler.json``, ``miner`` → ``postgres-miner.json``).
+
+    The suffix spelling is not a nicety: across the connector fleet the exact
+    spelling matches almost nothing, because the entry points are called
+    ``crawler`` and ``miner`` while the files are named for the source. Matching
+    only the exact form leaves every connector on the last-resort scan.
+
+    Args:
+        stem: A generated file's name without its ``.json`` suffix.
+        entrypoint: The entry point's kebab-case wire name.
+    """
+    return stem == entrypoint or stem.endswith(f"-{entrypoint}")
+
+
+def entrypoint_dirs(generated: Path, entrypoint: str) -> tuple[Path, ...]:
+    """Where *entrypoint*'s generated files may live, most specific first.
+
+    The kebab-case wire name, then its snake_case spelling. The contract toolkit
+    writes a UI-less system entry point's files under the snake_case directory
+    (FND-3563) so ``app.generated.<entrypoint_snake>._input`` imports; every
+    other entry point keeps the wire-name directory. One path when the two
+    spellings agree.
+
+    The kebab-to-snake rule is
+    :func:`application_sdk.app.entrypoint.entrypoint_module_segment`'s, restated
+    because this module must not import ``application_sdk.app``.
+
+    Args:
+        generated: The generated directory, usually ``app/generated``.
+        entrypoint: The entry point's kebab-case wire name.
+    """
+    kebab = generated / entrypoint
+    snake = generated / entrypoint.replace("-", "_")
+    return (kebab,) if snake == kebab else (kebab, snake)
+
+
+def eligible_form_configmaps(directory: Path) -> list[Path]:
+    """Every ``*.json`` in *directory* that could be a setup form, sorted.
+
+    The non-form siblings (:func:`is_form_configmap`) are already gone. What
+    remains is what any pick has to choose between, and a caller that wants to
+    *report* on the choice — the configmap endpoint warns when it had to fall
+    back to alphabetical order among several — needs the list, not just the
+    winner.
+
+    Args:
+        directory: A directory in the generated tree. Missing or not a
+            directory yields ``[]``.
+    """
+    if not directory.is_dir():
+        return []
+    return [
+        candidate
+        for candidate in sorted(directory.glob("*.json"))
+        if is_form_configmap(candidate.stem)
+    ]
+
+
+def choose_form_configmap(candidates: Sequence[Path], entrypoint: str) -> Path | None:
+    """Pick *entrypoint*'s form out of *candidates*, or ``None`` if empty.
+
+    Three steps, in this order:
+
+    1. The candidate whose stem :func:`names_entrypoint` — an identification.
+    2. The only candidate, when there is exactly one. Nothing else it could be.
+    3. Otherwise the alphabetically first, which **is** a guess.
+
+    **Step 3 stays, deliberately.** It is the compatibility path for every app
+    that has not adopted a name the SDK can recognise, and dropping it (or
+    turning ambiguity into a 404) would break apps that work today in order to
+    protect against a sibling the toolkit does not yet emit. FND-1682 was not
+    caused by the guess being *reachable*; it was caused by
+    ``artifact_schemas.json`` being eligible at all, which
+    :data:`NON_FORM_STEMS` now fixes. Steps 1 and 2 shrink how often step 3 has
+    to run — across the connector fleet they answer every generated directory —
+    but the endpoint that calls this logs a warning when step 3 does run with
+    more than one candidate, so a future sibling shows up in the logs instead of
+    only in a blank wizard.
+
+    Args:
+        candidates: Eligible forms, from :func:`eligible_form_configmaps`.
+        entrypoint: The entry point's kebab-case wire name.
+    """
+    if not candidates:
+        return None
+    named = [c for c in candidates if names_entrypoint(c.stem, entrypoint)]
+    if len(named) == 1:
+        return named[0]
+    return candidates[0]
+
+
+def pick_form_configmap(directory: Path, entrypoint: str) -> Path | None:
+    """Return *directory*'s setup form for *entrypoint*, or ``None``.
+
+    :func:`eligible_form_configmaps` then :func:`choose_form_configmap` — the
+    convenience spelling for callers that do not need the candidate list.
+
+    Args:
+        directory: A directory in the generated tree. Missing or not a
+            directory yields ``None``.
+        entrypoint: The entry point's kebab-case wire name.
+    """
+    return choose_form_configmap(eligible_form_configmaps(directory), entrypoint)
+
+
+def generated_layout(generated: Path) -> GeneratedLayout:
+    """Classify the committed generated tree by the shape it actually has.
+
+    ``multi``
+        One or more immediate subdirectories each holding a ``manifest.json``.
+        Each subdirectory name is an entry point's wire name.
+    ``single``
+        A ``manifest.json`` at the root of *generated* and no per-entry-point
+        subdirectories.
+    ``unknown``
+        No generated tree, or one carrying no ``manifest.json`` anywhere (a repo
+        that has not generated yet). Nothing can be inferred about the layout,
+        and every caller says so rather than guessing.
+
+    **Why the tree and not the Python entry-point count.** The layout is a
+    property of what the toolkit emitted, not of how many ``@entrypoint``
+    methods Python happens to see. A **route/card-split** app (BLDX-1342) is
+    exactly where the two disagree: it has several ``@entrypoint``\\ s the DAG
+    invokes by ``workflow_type``, but one marketplace card and therefore one
+    *flat* generated tree. Counting Python entry points calls it a bundle and
+    sends every consumer looking under a wire-name subdirectory the toolkit
+    will never write for that app.
+
+    Args:
+        generated: The generated directory, usually ``app/generated``.
+    """
+    try:
+        children = list(generated.iterdir())
+    except OSError:
+        return "unknown"
+    if any((child / "manifest.json").is_file() for child in children if child.is_dir()):
+        return "multi"
+    if (generated / "manifest.json").is_file():
+        return "single"
+    return "unknown"
+
+
+def form_configmap(
+    generated: Path, entrypoint: str, *, layout: GeneratedLayout | None = None
+) -> Path | None:
+    """Return the setup-form configmap for *entrypoint*, or ``None``.
+
+    Which directory is searched follows the *layout*, not the caller's belief
+    about how many entry points exist — see :func:`generated_layout`:
+
+    ``multi``
+        ``<generated>/<entrypoint>/`` only. A bundle's forms are nested, and
+        falling back to the root would let the root-level credential template
+        or another entry point's form answer for this one.
+    ``single``
+        ``<generated>/`` only, and *entrypoint* is ignored: a flat tree has one
+        form serving every route behind the single card.
+    ``unknown``
+        The nested directory, then the flat one — best effort, since a repo
+        that has not generated has nothing to be right about.
+
+    Within the chosen directory :func:`pick_form_configmap` decides, so a form
+    named after the entry point is identified by name and anything else falls
+    back to the sorted :func:`is_form_configmap` scan. That exclusion is what
+    keeps a flat app's ``atlan-connectors-<source>.json`` from being mistaken
+    for its form: sorted alphabetically the credential template comes *first*,
+    so a glob without the exclusion picks the wrong file on every
+    single-entrypoint connector.
+
+    Args:
+        generated: The generated directory, usually ``app/generated``.
+        entrypoint: The entry point's kebab-case wire name, which is also its
+            directory name in a bundle's generated tree.
+        layout: The tree's shape. Computed from *generated* when omitted.
+
+    Returns:
+        The form's path, or ``None`` when the chosen directories hold no file
+        that qualifies.
+    """
+    resolved = generated_layout(generated) if layout is None else layout
+    if resolved == "single":
+        search: tuple[Path, ...] = (generated,)
+    elif resolved == "multi":
+        search = (generated / entrypoint,)
+    else:
+        search = (generated / entrypoint, generated)
+
+    for directory in search:
+        found = pick_form_configmap(directory, entrypoint)
+        if found is not None:
+            return found
+    return None