airbyte-cdk 6.48.2.post1.dev14733922675__tar.gz → 6.48.4__tar.gz

{airbyte_cdk-6.48.2.post1.dev14733922675 → airbyte_cdk-6.48.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.48.2.post1.dev14733922675
+Version: 6.48.4
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT
@@ -66,6 +66,7 @@ Requires-Dist: pytz (==2024.2)
 Requires-Dist: rapidfuzz (>=3.10.1,<4.0.0)
 Requires-Dist: requests
 Requires-Dist: requests_cache
+Requires-Dist: rich
 Requires-Dist: rich-click (>=1.8.8,<2.0.0)
 Requires-Dist: serpyco-rs (>=1.10.2,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0,<3.0,!=2.0.36) ; extra == "sql"

airbyte_cdk-6.48.4/airbyte_cdk/cli/airbyte_cdk/_secrets.py

@@ -0,0 +1,424 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""**Secret management commands.**
+
+This module provides commands for managing secrets for Airbyte connectors.
+
+**Usage:**
+
+```bash
+# Fetch secrets
+airbyte-cdk secrets fetch --connector-name source-github
+airbyte-cdk secrets fetch --connector-directory /path/to/connector
+airbyte-cdk secrets fetch  # Run from within a connector directory
+
+# List secrets (without fetching)
+airbyte-cdk secrets list --connector-name source-github
+airbyte-cdk secrets list --connector-directory /path/to/connector
+```
+
+**Usage without pre-installing (stateless):**
+
+```bash
+pipx run airbyte-cdk secrets fetch ...
+uvx airbyte-cdk secrets fetch ...
+```
+
+The command retrieves secrets from Google Secret Manager based on connector
+labels and writes them to the connector's `secrets` directory.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from functools import lru_cache
+from pathlib import Path
+from typing import Any, cast
+
+import requests
+import rich_click as click
+import yaml
+from click import style
+from rich.console import Console
+from rich.table import Table
+
+from airbyte_cdk.cli.airbyte_cdk._util import (
+    resolve_connector_name,
+    resolve_connector_name_and_directory,
+)
+
+AIRBYTE_INTERNAL_GCP_PROJECT = "dataline-integration-testing"
+CONNECTOR_LABEL = "connector"
+GLOBAL_MASK_KEYS_URL = "https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml"
+
+logger = logging.getLogger("airbyte-cdk.cli.secrets")
+
+try:
+    from google.cloud import secretmanager_v1 as secretmanager
+    from google.cloud.secretmanager_v1 import Secret
+except ImportError:
+    # If the package is not installed, we will raise an error in the CLI command.
+    secretmanager = None  # type: ignore
+    Secret = None  # type: ignore
+
+
+@click.group(
+    name="secrets",
+    help=__doc__.replace("\n", "\n\n"),  # Render docstring as help text (markdown) # type: ignore
+)
+def secrets_cli_group() -> None:
+    """Secret management commands."""
+    pass
+
+
+@secrets_cli_group.command()
+@click.option(
+    "--connector-name",
+    type=str,
+    help="Name of the connector to fetch secrets for. Ignored if --connector-directory is provided.",
+)
+@click.option(
+    "--connector-directory",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Path to the connector directory.",
+)
+@click.option(
+    "--gcp-project-id",
+    type=str,
+    default=AIRBYTE_INTERNAL_GCP_PROJECT,
+    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+)
+@click.option(
+    "--print-ci-secrets-masks",
+    help="Print GitHub CI mask for secrets.",
+    type=bool,
+    is_flag=True,
+    default=False,
+)
+def fetch(
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+    print_ci_secrets_masks: bool = False,
+) -> None:
+    """Fetch secrets for a connector from Google Secret Manager.
+
+    This command fetches secrets for a connector from Google Secret Manager and writes them
+    to the connector's secrets directory.
+
+    If no connector name or directory is provided, we will look within the current working
+    directory. If the current working directory is not a connector directory (e.g. starting
+    with 'source-') and no connector name or path is provided, the process will fail.
+
+    The `--print-ci-secrets-masks` option will print the GitHub CI mask for the secrets.
+    This is useful for masking secrets in CI logs.
+
+    WARNING: This action causes the secrets to be printed in clear text to `STDOUT`. For security
+    reasons, this function will only execute if the `CI` environment variable is set. Otherwise,
+    masks will not be printed.
+    """
+    click.echo("Fetching secrets...", err=True)
+
+    client = _get_gsm_secrets_client()
+    connector_name, connector_directory = resolve_connector_name_and_directory(
+        connector_name=connector_name,
+        connector_directory=connector_directory,
+    )
+    secrets_dir = _get_secrets_dir(
+        connector_directory=connector_directory,
+        connector_name=connector_name,
+        ensure_exists=True,
+    )
+    secrets = _fetch_secret_handles(
+        connector_name=connector_name,
+        gcp_project_id=gcp_project_id,
+    )
+    # Fetch and write secrets
+    secret_count = 0
+    for secret in secrets:
+        secret_file_path = _get_secret_filepath(
+            secrets_dir=secrets_dir,
+            secret=secret,
+        )
+        _write_secret_file(
+            secret=secret,
+            client=client,
+            file_path=secret_file_path,
+        )
+        click.echo(f"Secret written to: {secret_file_path.absolute()!s}", err=True)
+        secret_count += 1
+
+    if secret_count == 0:
+        click.echo(
+            f"No secrets found for connector: '{connector_name}'",
+            err=True,
+        )
+
+    if not print_ci_secrets_masks:
+        return
+
+    if not os.environ.get("CI", None):
+        click.echo(
+            "The `--print-ci-secrets-masks` option is only available in CI environments. "
+            "The `CI` env var is either not set or not set to a truthy value. "
+            "Skipping printing secret masks.",
+            err=True,
+        )
+        return
+
+    # Else print the CI mask
+    _print_ci_secrets_masks(
+        secrets_dir=secrets_dir,
+    )
+
+
+@secrets_cli_group.command("list")
+@click.option(
+    "--connector-name",
+    type=str,
+    help="Name of the connector to fetch secrets for. Ignored if --connector-directory is provided.",
+)
+@click.option(
+    "--connector-directory",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    help="Path to the connector directory.",
+)
+@click.option(
+    "--gcp-project-id",
+    type=str,
+    default=AIRBYTE_INTERNAL_GCP_PROJECT,
+    help=f"GCP project ID. Defaults to '{AIRBYTE_INTERNAL_GCP_PROJECT}'.",
+)
+def list_(
+    connector_name: str | None = None,
+    connector_directory: Path | None = None,
+    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+) -> None:
+    """List secrets for a connector from Google Secret Manager.
+
+    This command fetches secrets for a connector from Google Secret Manager and prints
+    them as a table.
+
+    If no connector name or directory is provided, we will look within the current working
+    directory. If the current working directory is not a connector directory (e.g. starting
+    with 'source-') and no connector name or path is provided, the process will fail.
+    """
+    click.echo("Scanning secrets...", err=True)
+
+    connector_name = connector_name or resolve_connector_name(
+        connector_directory=connector_directory or Path().resolve().absolute(),
+    )
+    secrets: list[Secret] = _fetch_secret_handles(  # type: ignore
+        connector_name=connector_name,
+        gcp_project_id=gcp_project_id,
+    )
+
+    if not secrets:
+        click.echo(
+            f"No secrets found for connector: '{connector_name}'",
+            err=True,
+        )
+        return
+    # print a rich table with the secrets
+    click.echo(
+        style(
+            f"Secrets for connector '{connector_name}' in project '{gcp_project_id}':",
+            fg="green",
+        )
+    )
+
+    console = Console()
+    table = Table(title=f"'{connector_name}' Secrets")
+    table.add_column("Name", justify="left", style="cyan", overflow="fold")
+    table.add_column("Labels", justify="left", style="magenta", overflow="fold")
+    table.add_column("Created", justify="left", style="blue", overflow="fold")
+    for secret in secrets:
+        full_secret_name = secret.name
+        secret_name = full_secret_name.split("/secrets/")[-1]  # Removes project prefix
+        # E.g. https://console.cloud.google.com/security/secret-manager/secret/SECRET_SOURCE-SHOPIFY__CREDS/versions?hl=en&project=<gcp_project_id>
+        secret_url = f"https://console.cloud.google.com/security/secret-manager/secret/{secret_name}/versions?hl=en&project={gcp_project_id}"
+        table.add_row(
+            f"[link={secret_url}]{secret_name}[/link]",
+            "\n".join([f"{k}={v}" for k, v in secret.labels.items()]),
+            str(secret.create_time),
+        )
+
+    console.print(table)
+
+
+def _fetch_secret_handles(
+    connector_name: str,
+    gcp_project_id: str = AIRBYTE_INTERNAL_GCP_PROJECT,
+) -> list["Secret"]:  # type: ignore
+    """Fetch secrets from Google Secret Manager."""
+    if not secretmanager:
+        raise ImportError(
+            "google-cloud-secret-manager package is required for Secret Manager integration. "
+            "Install it with 'pip install airbyte-cdk[dev]' "
+            "or 'pip install google-cloud-secret-manager'."
+        )
+
+    client = _get_gsm_secrets_client()
+
+    # List all secrets with the connector label
+    parent = f"projects/{gcp_project_id}"
+    filter_string = f"labels.{CONNECTOR_LABEL}={connector_name}"
+    secrets = client.list_secrets(
+        request=secretmanager.ListSecretsRequest(
+            parent=parent,
+            filter=filter_string,
+        )
+    )
+    return [s for s in secrets]
+
+
+def _write_secret_file(
+    secret: "Secret",  # type: ignore
+    client: "secretmanager.SecretManagerServiceClient",  # type: ignore
+    file_path: Path,
+) -> None:
+    version_name = f"{secret.name}/versions/latest"
+    response = client.access_secret_version(name=version_name)
+    file_path.write_text(response.payload.data.decode("UTF-8"))
+    file_path.chmod(0o600)  # default to owner read/write only
+
+
+def _get_secrets_dir(
+    connector_directory: Path,
+    connector_name: str,
+    ensure_exists: bool = True,
+) -> Path:
+    try:
+        connector_name, connector_directory = resolve_connector_name_and_directory(
+            connector_name=connector_name,
+            connector_directory=connector_directory,
+        )
+    except FileNotFoundError as e:
+        raise FileNotFoundError(
+            f"Could not find connector directory for '{connector_name}'. "
+            "Please provide the --connector-directory option with the path to the connector. "
+            "Note: This command requires either running from within a connector directory, "
+            "being in the airbyte monorepo, or explicitly providing the connector directory path."
+        ) from e
+    except ValueError as e:
+        raise ValueError(str(e))
+
+    secrets_dir = connector_directory / "secrets"
+    if ensure_exists:
+        secrets_dir.mkdir(parents=True, exist_ok=True)
+
+        gitignore_path = secrets_dir / ".gitignore"
+        if not gitignore_path.exists():
+            gitignore_path.write_text("*")
+
+    return secrets_dir
+
+
+def _get_secret_filepath(
+    secrets_dir: Path,
+    secret: Secret,  # type: ignore
+) -> Path:
+    """Get the file path for a secret based on its labels."""
+    if secret.labels and "filename" in secret.labels:
+        return secrets_dir / f"{secret.labels['filename']}.json"
+
+    return secrets_dir / "config.json"  # Default filename
+
+
+def _get_gsm_secrets_client() -> "secretmanager.SecretManagerServiceClient":  # type: ignore
+    """Get the Google Secret Manager client."""
+    if not secretmanager:
+        raise ImportError(
+            "google-cloud-secret-manager package is required for Secret Manager integration. "
+            "Install it with 'pip install airbyte-cdk[dev]' "
+            "or 'pip install google-cloud-secret-manager'."
+        )
+
+    credentials_json = os.environ.get("GCP_GSM_CREDENTIALS")
+    if not credentials_json:
+        raise ValueError(
+            "No Google Cloud credentials found. "
+            "Please set the `GCP_GSM_CREDENTIALS` environment variable."
+        )
+
+    return cast(
+        "secretmanager.SecretManagerServiceClient",
+        secretmanager.SecretManagerServiceClient.from_service_account_info(
+            json.loads(credentials_json)
+        ),
+    )
+
+
+def _print_ci_secrets_masks(
+    secrets_dir: Path,
+) -> None:
+    """Print GitHub CI mask for secrets.
+
+    https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#example-masking-an-environment-variable
+
+    The env var `CI` is set to a truthy value in GitHub Actions, so we can use it to
+    determine if we are in a CI environment. If not, we don't want to print the masks,
+    as it will cause the secrets to be printed in clear text to STDOUT.
+    """
+    if not os.environ.get("CI", None):
+        click.echo(
+            "The `--print-ci-secrets-masks` option is only available in CI environments. "
+            "The `CI` env var is either not set or not set to a truthy value. "
+            "Skipping printing secret masks.",
+            err=True,
+        )
+        return
+
+    for secret_file_path in secrets_dir.glob("*.json"):
+        config_dict = json.loads(secret_file_path.read_text())
+        _print_ci_secrets_masks_for_config(config=config_dict)
+
+
+def _print_ci_secrets_masks_for_config(
+    config: dict[str, str] | list[Any] | Any,
+) -> None:
+    """Print GitHub CI mask for secrets config, navigating child nodes recursively."""
+    if isinstance(config, list):
+        for item in config:
+            _print_ci_secrets_masks_for_config(item)
+
+    if isinstance(config, dict):
+        for key, value in config.items():
+            if _is_secret_property(key):
+                logger.debug(f"Masking secret for config key: {key}")
+                print(f"::add-mask::{value!s}")
+                if isinstance(value, dict):
+                    # For nested dicts, we also need to mask the json-stringified version
+                    print(f"::add-mask::{json.dumps(value)!s}")
+
+            if isinstance(value, (dict, list)):
+                _print_ci_secrets_masks_for_config(config=value)
+
+
+def _is_secret_property(property_name: str) -> bool:
+    """Check if the property name is in the list of properties to mask.
+
+    To avoid false negatives, we perform a case-insensitive check, and we include any property name
+    that contains a rule entry, even if it is not an exact match.
+
+    For example, if the rule entry is "password", we will also match "PASSWORD" and "my_password".
+    """
+    names_to_mask: list[str] = _get_spec_mask()
+    if any([mask.lower() in property_name.lower() for mask in names_to_mask]):
+        return True
+
+    return False
+
+
+@lru_cache
+def _get_spec_mask() -> list[str]:
+    """Get the list of properties to mask from the spec mask file."""
+    response = requests.get(GLOBAL_MASK_KEYS_URL, allow_redirects=True)
+    if not response.ok:
+        logger.error(f"Failed to fetch spec mask: {response.content.decode('utf-8')}")
+    try:
+        return cast(list[str], yaml.safe_load(response.content)["properties"])
+    except Exception as e:
+        logger.error(f"Failed to parse spec mask: {e}")
+        raise

{airbyte_cdk-6.48.2.post1.dev14733922675 → airbyte_cdk-6.48.4}/airbyte_cdk/cli/airbyte_cdk/_util.py

@@ -41,3 +41,29 @@ def resolve_connector_name_and_directory(
         raise ValueError("Either connector_name or connector_directory must be provided.")
 
     return connector_name, connector_directory
+
+
+def resolve_connector_name(
+    connector_directory: Path,
+) -> str:
+    """Resolve the connector name.
+
+    This function will resolve the connector name based on the provided connector directory.
+    If the current working directory is not a connector directory
+    (e.g. starting with 'source-'), the process will fail.
+
+    Raises:
+        FileNotFoundError: If the connector directory does not exist or cannot be found.
+    """
+    if not connector_directory:
+        raise FileNotFoundError(
+            "Connector directory does not exist or cannot be found. Please provide a valid "
+            "connector directory."
+        )
+    connector_name = connector_directory.absolute().name
+    if not connector_name.startswith("source-") and not connector_name.startswith("destination-"):
+        raise ValueError(
+            f"Connector directory '{connector_name}' does not look like a valid connector directory. "
+            f"Full path: {connector_directory.absolute()}"
+        )
+    return connector_name

{airbyte_cdk-6.48.2.post1.dev14733922675 → airbyte_cdk-6.48.4}/airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py

@@ -33,19 +33,44 @@ class HttpRequesterRequestBodyJsonDataToRequestBody(ManifestMigration):
     def migrate(self, manifest: ManifestType) -> None:
         for key in self.original_keys:
             if key == self.body_json_key and key in manifest:
-                manifest[self.replacement_key] = {
-                    "type": "RequestBodyJson",
-                    "value": manifest[key],
-                }
-                manifest.pop(key, None)
+                self._migrate_body_json(manifest, key)
             elif key == self.body_data_key and key in manifest:
-                manifest[self.replacement_key] = {
-                    "type": "RequestBodyData",
-                    "value": manifest[key],
-                }
-                manifest.pop(key, None)
+                self._migrate_body_data(manifest, key)
 
     def validate(self, manifest: ManifestType) -> bool:
         return self.replacement_key in manifest and all(
             key not in manifest for key in self.original_keys
         )
+
+    def _migrate_body_json(self, manifest: ManifestType, key: str) -> None:
+        """
+        Migrate the value of the request_body_json.
+        """
+        query_key = "query"
+        text_type = "RequestBodyPlainText"
+        graph_ql_type = "RequestBodyGraphQL"
+        json_object_type = "RequestBodyJsonObject"
+
+        if isinstance(manifest[key], str):
+            self._migrate_value(manifest, key, text_type)
+        elif isinstance(manifest[key], dict):
+            if manifest[key].get(query_key) is not None:
+                self._migrate_value(manifest, key, graph_ql_type)
+            else:
+                self._migrate_value(manifest, key, json_object_type)
+
+    def _migrate_body_data(self, manifest: ManifestType, key: str) -> None:
+        """
+        Migrate the value of the request_body_data.
+        """
+        self._migrate_value(manifest, key, "RequestBodyUrlEncodedForm")
+
+    def _migrate_value(self, manifest: ManifestType, key: str, type: str) -> None:
+        """
+        Migrate the value of the key to a specific type and update the manifest.
+        """
+        manifest[self.replacement_key] = {
+            "type": type,
+            "value": manifest[key],
+        }
+        manifest.pop(key, None)

{airbyte_cdk-6.48.2.post1.dev14733922675 → airbyte_cdk-6.48.4}/airbyte_cdk/manifest_migrations/migrations/registry.yaml

@@ -3,7 +3,7 @@
 #
 
 manifest_migrations:
-  - version: 6.48.2
+  - version: 6.48.3
     migrations: 
       - name: http_requester_url_base_to_url
         order: 1

{airbyte_cdk-6.48.2.post1.dev14733922675 → airbyte_cdk-6.48.4}/airbyte_cdk/sources/declarative/auth/oauth.py

@@ -2,10 +2,9 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-import logging
 from dataclasses import InitVar, dataclass, field
 from datetime import datetime, timedelta
-from typing import Any, List, Mapping, Optional, Union
+from typing import Any, List, Mapping, MutableMapping, Optional, Union
 
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
@@ -20,7 +19,6 @@ from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import (
 )
 from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse
 
-logger = logging.getLogger("airbyte")
 
 @dataclass
 class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAuthenticator):
@@ -203,8 +201,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
             self._client_secret.eval(self.config) if self._client_secret else self._client_secret
         )
         if not client_secret:
-            # We've seen some APIs allowing empty client_secret so we will only log here
-            logger.warning("OAuthAuthenticator was unable to evaluate client_secret parameter hence it'll be empty")
+            raise ValueError("OAuthAuthenticator was unable to evaluate client_secret parameter")
         return client_secret  # type: ignore # value will be returned as a string, or an error will be raised
 
     def get_refresh_token_name(self) -> str: