airbyte-cdk 6.48.0__py3-none-any.whl → 6.48.2__py3-none-any.whl

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -56,6 +56,16 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits:
     return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
 
+def should_migrate_manifest(config: Mapping[str, Any]) -> bool:
+    """
+    Determines whether the manifest should be migrated,
+    based on the presence of the "__should_migrate" key in the config.
+
+    This flag is set by the UI.
+    """
+    return config.get("__should_migrate", False)
+
+
 def should_normalize_manifest(config: Mapping[str, Any]) -> bool:
     """
     Check if the manifest should be normalized.
@@ -71,6 +81,7 @@ def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDecl
         config=config,
         emit_connector_builder_messages=True,
         source_config=manifest,
+        migrate_manifest=should_migrate_manifest(config),
         normalize_manifest=should_normalize_manifest(config),
         component_factory=ModelToComponentFactory(
             emit_connector_builder_messages=True,

airbyte_cdk/connector_builder/test_reader/reader.py

@@ -4,6 +4,7 @@
 
 
 import logging
+from math import log
 from typing import Any, ClassVar, Dict, Iterator, List, Mapping, Optional, Union
 
 from airbyte_cdk.connector_builder.models import (
@@ -112,11 +113,16 @@ class TestReader:
         record_limit = self._check_record_limit(record_limit)
         # The connector builder currently only supports reading from a single stream at a time
         stream = source.streams(config)[0]
+
+        # get any deprecation warnings during the component creation
+        deprecation_warnings: List[LogMessage] = source.deprecation_warnings()
+
         schema_inferrer = SchemaInferrer(
             self._pk_to_nested_and_composite_field(stream.primary_key),
             self._cursor_field_to_nested_and_composite_field(stream.cursor_field),
         )
         datetime_format_inferrer = DatetimeFormatInferrer()
+
         message_group = get_message_groups(
             self._read_stream(source, config, configured_catalog, state),
             schema_inferrer,
@@ -127,6 +133,10 @@ class TestReader:
         slices, log_messages, auxiliary_requests, latest_config_update = self._categorise_groups(
             message_group
         )
+
+        # extend log messages with deprecation warnings
+        log_messages.extend(deprecation_warnings)
+
         schema, log_messages = self._get_infered_schema(
             configured_catalog, schema_inferrer, log_messages
         )
@@ -269,6 +279,7 @@ class TestReader:
         auxiliary_requests = []
         latest_config_update: Optional[AirbyteControlMessage] = None
 
+        # process the message groups first
         for message_group in message_groups:
             match message_group:
                 case AirbyteLogMessage():

airbyte_cdk/manifest_migrations/README.md

@@ -0,0 +1,73 @@
+# Manifest Migrations
+
+This directory contains the logic and registry for manifest migrations in the Airbyte CDK. Migrations are used to update or transform manifest components to newer formats or schemas as the CDK evolves.
+
+## Adding a New Migration
+
+1. **Create a Migration File:**
+   - Add a new Python file in the `migrations/` subdirectory.
+   - Name the file using the pattern: `<description_of_the_migration>.py`.
+     - Example: `http_requester_url_base_to_url.py`
+   - The filename should be unique and descriptive.
+
+2. **Define the Migration Class:**
+   - The migration class must inherit from `ManifestMigration`.
+   - Name the class using a descriptive name (e.g., `HttpRequesterUrlBaseToUrl`).
+   - Implement the following methods:
+     - `should_migrate(self, manifest: ManifestType) -> bool`
+     - `migrate(self, manifest: ManifestType) -> None`
+     - `validate(self, manifest: ManifestType) -> bool`
+
+3. **Register the Migration:**
+   - Open `migrations/registry.yaml`.
+   - Add an entry under the appropriate version, or create a new version section if needed.
+   - Each migration entry should include:
+     - `name`: The filename (without `.py`)
+     - `order`: The order in which this migration should be applied for the version
+     - `description`: A short description of the migration
+
+   Example:
+   ```yaml
+   manifest_migrations:
+     - version: 6.45.2
+       migrations:
+         - name: http_requester_url_base_to_url
+           order: 1
+           description: |
+             This migration updates the `url_base` field in the `HttpRequester` component spec to `url`.
+   ```
+
+4. **Testing:**
+   - Ensure your migration is covered by unit tests.
+   - Tests should verify both `should_migrate`, `migrate`, and `validate` behaviors.
+
+## Migration Discovery
+
+- Migrations are discovered and registered automatically based on the entries in `migrations/registry.yaml`.
+- Do not modify the migration registry in code manually.
+- If you need to skip certain component types, use the `NON_MIGRATABLE_TYPES` list in `manifest_migration.py`.
+
+## Example Migration Skeleton
+
+```python
+from airbyte_cdk.manifest_migrations.manifest_migration import TYPE_TAG, ManifestMigration, ManifestType
+
+class ExampleMigration(ManifestMigration):
+    component_type = "ExampleComponent"
+    original_key = "old_key"
+    replacement_key = "new_key"
+
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        return manifest[TYPE_TAG] == self.component_type and self.original_key in manifest
+
+    def migrate(self, manifest: ManifestType) -> None:
+        manifest[self.replacement_key] = manifest[self.original_key]
+        manifest.pop(self.original_key, None)
+
+    def validate(self, manifest: ManifestType) -> bool:
+        return self.replacement_key in manifest and self.original_key not in manifest
+```
+
+---
+
+For more details, see the docstrings in `manifest_migration.py` and the examples in the `migrations/` folder.

airbyte_cdk/manifest_migrations/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#

airbyte_cdk/manifest_migrations/exceptions.py

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+class ManifestMigrationException(Exception):
+    """
+    Raised when a migration error occurs in the manifest.
+    """
+
+    def __init__(self, message: str) -> None:
+        super().__init__(f"Failed to migrate the manifest: {message}")

airbyte_cdk/manifest_migrations/manifest_migration.py

@@ -0,0 +1,134 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass
+from typing import Any, Dict
+
+ManifestType = Dict[str, Any]
+
+
+TYPE_TAG = "type"
+
+NON_MIGRATABLE_TYPES = [
+    # more info here: https://github.com/airbytehq/airbyte-internal-issues/issues/12423
+    "DynamicDeclarativeStream",
+]
+
+
+@dataclass
+class MigrationTrace:
+    """
+    This class represents a migration that has been applied to the manifest.
+    It contains information about the migration, including the version it was applied from,
+    the version it was applied to, and the time it was applied.
+    """
+
+    from_version: str
+    to_version: str
+    migration: str
+    migrated_at: str
+
+    def as_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+class ManifestMigration(ABC):
+    """
+    Base class for manifest migrations.
+    This class provides a framework for migrating manifest components.
+    It defines the structure for migration classes, including methods for checking if a migration is needed,
+    performing the migration, and validating the migration.
+    """
+
+    def __init__(self) -> None:
+        self.is_migrated: bool = False
+
+    @abstractmethod
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        """
+        Check if the manifest should be migrated.
+
+        :param manifest: The manifest to potentially migrate
+
+        :return: true if the manifest is of the expected format and should be migrated. False otherwise.
+        """
+
+    @abstractmethod
+    def migrate(self, manifest: ManifestType) -> None:
+        """
+        Migrate the manifest. Assumes should_migrate(manifest) returned True.
+
+        :param manifest: The manifest to migrate
+        """
+
+    @abstractmethod
+    def validate(self, manifest: ManifestType) -> bool:
+        """
+        Validate the manifest to ensure the migration was successfully applied.
+
+        :param manifest: The manifest to validate
+        """
+
+    def _is_component(self, obj: Dict[str, Any]) -> bool:
+        """
+        Check if the object is a component.
+
+        :param obj: The object to check
+        :return: True if the object is a component, False otherwise
+        """
+        return TYPE_TAG in obj.keys()
+
+    def _is_migratable_type(self, obj: Dict[str, Any]) -> bool:
+        """
+        Check if the object is a migratable component,
+        based on the Type of the component and the migration version.
+
+        :param obj: The object to check
+        :return: True if the object is a migratable component, False otherwise
+        """
+        return obj[TYPE_TAG] not in NON_MIGRATABLE_TYPES
+
+    def _process_manifest(self, obj: Any) -> None:
+        """
+        Recursively processes a manifest object, migrating components that match the migration criteria.
+
+        This method traverses the entire manifest structure (dictionaries and lists) and applies
+        migrations to components that:
+        1. Have a type tag
+        2. Are not in the list of non-migratable types
+        3. Meet the conditions defined in the should_migrate method
+
+        Parameters:
+            obj (Any): The object to process, which can be a dictionary, list, or any other type.
+                       Dictionary objects are checked for component type tags and potentially migrated.
+                       List objects have each of their items processed recursively.
+                       Other types are ignored.
+
+        Returns:
+            None, since we process the manifest in place.
+        """
+        if isinstance(obj, dict):
+            # Check if the object is a component
+            if self._is_component(obj):
+                # Check if the object is allowed to be migrated
+                if not self._is_migratable_type(obj):
+                    return
+
+                # Check if the object should be migrated
+                if self.should_migrate(obj):
+                    # Perform the migration, if needed
+                    self.migrate(obj)
+                    # validate the migration
+                    self.is_migrated = self.validate(obj)
+
+            # Process all values in the dictionary
+            for value in list(obj.values()):
+                self._process_manifest(value)
+
+        elif isinstance(obj, list):
+            # Process all items in the list
+            for item in obj:
+                self._process_manifest(item)

airbyte_cdk/manifest_migrations/migration_handler.py

@@ -0,0 +1,163 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+import copy
+import logging
+from datetime import datetime, timezone
+from typing import Type
+
+from packaging.version import Version
+
+from airbyte_cdk.manifest_migrations.exceptions import (
+    ManifestMigrationException,
+)
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    ManifestMigration,
+    ManifestType,
+    MigrationTrace,
+)
+from airbyte_cdk.manifest_migrations.migrations_registry import (
+    MANIFEST_MIGRATIONS,
+)
+
+METADATA_TAG = "metadata"
+MANIFEST_VERSION_TAG = "version"
+APPLIED_MIGRATIONS_TAG = "applied_migrations"
+
+LOGGER = logging.getLogger("airbyte.cdk.manifest_migrations")
+
+
+class ManifestMigrationHandler:
+    """
+    This class is responsible for handling migrations in the manifest.
+    """
+
+    def __init__(self, manifest: ManifestType) -> None:
+        self._manifest = manifest
+        self._migrated_manifest: ManifestType = copy.deepcopy(self._manifest)
+
+    def apply_migrations(self) -> ManifestType:
+        """
+        Apply all registered migrations to the manifest.
+
+        This method iterates through all migrations in the migrations registry and applies
+        them sequentially to the current manifest. If any migration fails with a
+        ManifestMigrationException, the original unmodified manifest is returned instead.
+
+        Returns:
+            ManifestType: The migrated manifest if all migrations succeeded, or the original
+                          manifest if any migration failed.
+        """
+        try:
+            manifest_version = self._get_manifest_version()
+            for migration_version, migrations in MANIFEST_MIGRATIONS.items():
+                for migration_cls in migrations:
+                    self._handle_migration(migration_cls, manifest_version, migration_version)
+            return self._migrated_manifest
+        except ManifestMigrationException:
+            # if any errors occur we return the original resolved manifest
+            return self._manifest
+
+    def _handle_migration(
+        self,
+        migration_class: Type[ManifestMigration],
+        manifest_version: str,
+        migration_version: str,
+    ) -> None:
+        """
+        Handles a single manifest migration by instantiating the migration class and processing the manifest.
+
+        Args:
+            migration_class (Type[ManifestMigration]): The migration class to apply to the manifest.
+
+        Raises:
+            ManifestMigrationException: If the migration process encounters any errors.
+        """
+        try:
+            migration_instance = migration_class()
+            if self._version_is_valid_for_migration(manifest_version, migration_version):
+                migration_instance._process_manifest(self._migrated_manifest)
+                if migration_instance.is_migrated:
+                    # set the updated manifest version, after migration has been applied
+                    self._set_manifest_version(migration_version)
+                    self._set_migration_trace(migration_class, manifest_version, migration_version)
+            else:
+                LOGGER.info(
+                    f"Manifest migration: `{self._get_migration_name(migration_class)}` is not supported for the given manifest version `{manifest_version}`.",
+                )
+        except Exception as e:
+            raise ManifestMigrationException(str(e)) from e
+
+    def _get_migration_name(self, migration_class: Type[ManifestMigration]) -> str:
+        """
+        Get the name of the migration instance.
+
+        Returns:
+            str: The name of the migration.
+        """
+        return migration_class.__name__
+
+    def _get_manifest_version(self) -> str:
+        """
+        Get the manifest version from the manifest.
+
+        :param manifest: The manifest to get the version from
+        :return: The manifest version
+        """
+        return str(self._migrated_manifest.get(MANIFEST_VERSION_TAG, "0.0.0"))
+
+    def _version_is_valid_for_migration(
+        self,
+        manifest_version: str,
+        migration_version: str,
+    ) -> bool:
+        """
+        Checks if the given manifest version is less than or equal to the specified migration version.
+
+        Args:
+            manifest_version (str): The version of the manifest to check.
+            migration_version (str): The migration version to compare against.
+
+        Returns:
+            bool: True if the manifest version is less than or equal to the migration version, False otherwise.
+        """
+        return Version(manifest_version) <= Version(migration_version)
+
+    def _set_manifest_version(self, version: str) -> None:
+        """
+        Set the manifest version in the manifest.
+
+        :param version: The version to set
+        """
+        self._migrated_manifest[MANIFEST_VERSION_TAG] = version
+
+    def _set_migration_trace(
+        self,
+        migration_instance: Type[ManifestMigration],
+        manifest_version: str,
+        migration_version: str,
+    ) -> None:
+        """
+        Set the migration trace in the manifest, under the `metadata.applied_migrations` property object.
+
+        :param migration_instance: The migration instance to set
+        :param manifest_version: The manifest version before migration
+        :param migration_version: The manifest version after migration
+        """
+
+        if METADATA_TAG not in self._migrated_manifest:
+            self._migrated_manifest[METADATA_TAG] = {}
+        if APPLIED_MIGRATIONS_TAG not in self._migrated_manifest[METADATA_TAG]:
+            self._migrated_manifest[METADATA_TAG][APPLIED_MIGRATIONS_TAG] = []
+
+        migration_trace = MigrationTrace(
+            from_version=manifest_version,
+            to_version=migration_version,
+            migration=self._get_migration_name(migration_instance),
+            migrated_at=datetime.now(tz=timezone.utc).isoformat(),
+        ).as_dict()
+
+        if migration_version not in self._migrated_manifest[METADATA_TAG][APPLIED_MIGRATIONS_TAG]:
+            self._migrated_manifest[METADATA_TAG][APPLIED_MIGRATIONS_TAG].append(migration_trace)

airbyte_cdk/manifest_migrations/migrations/__init__.py

@@ -0,0 +1,4 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+

airbyte_cdk/manifest_migrations/migrations/http_requester_path_to_url.py

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from urllib.parse import urljoin
+
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    TYPE_TAG,
+    ManifestMigration,
+    ManifestType,
+)
+from airbyte_cdk.sources.types import EmptyString
+
+
+class HttpRequesterPathToUrl(ManifestMigration):
+    """
+    This migration is responsible for migrating the `path` key to `url` in the HttpRequester component.
+    The `path` key is expected to be a relative path, and the `url` key is expected to be a full URL.
+    The migration will concatenate the `url_base` and `path` to form a full URL.
+    """
+
+    component_type = "HttpRequester"
+    original_key = "path"
+    replacement_key = "url"
+
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        return manifest[TYPE_TAG] == self.component_type and self.original_key in list(
+            manifest.keys()
+        )
+
+    def migrate(self, manifest: ManifestType) -> None:
+        original_key_value = manifest.get(self.original_key, EmptyString).lstrip("/")
+        replacement_key_value = manifest[self.replacement_key]
+
+        # return a full-url if provided directly from interpolation context
+        if original_key_value == EmptyString or original_key_value is None:
+            manifest[self.replacement_key] = replacement_key_value
+            manifest.pop(self.original_key, None)
+        else:
+            # since we didn't provide a full-url, the url_base might not have a trailing slash
+            # so we join the url_base and path correctly
+            if not replacement_key_value.endswith("/"):
+                replacement_key_value += "/"
+
+            manifest[self.replacement_key] = urljoin(replacement_key_value, original_key_value)
+            manifest.pop(self.original_key, None)
+
+    def validate(self, manifest: ManifestType) -> bool:
+        """
+        Validate the migration by checking if the `url` key is present and the `path` key is not.
+        """
+        return (
+            self.replacement_key in manifest
+            and self.original_key not in manifest
+            and manifest[self.replacement_key] is not None
+        )

airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py

@@ -0,0 +1,51 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    TYPE_TAG,
+    ManifestMigration,
+    ManifestType,
+)
+
+
+class HttpRequesterRequestBodyJsonDataToRequestBody(ManifestMigration):
+    """
+    This migration is responsible for migrating the `request_body_json` and `request_body_data` keys
+    to a unified `request_body` key in the HttpRequester component.
+    The migration will copy the value of either original key to `request_body` and remove the original key.
+    """
+
+    component_type = "HttpRequester"
+
+    body_json_key = "request_body_json"
+    body_data_key = "request_body_data"
+    original_keys = (body_json_key, body_data_key)
+
+    replacement_key = "request_body"
+
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        return manifest[TYPE_TAG] == self.component_type and any(
+            key in list(manifest.keys()) for key in self.original_keys
+        )
+
+    def migrate(self, manifest: ManifestType) -> None:
+        for key in self.original_keys:
+            if key == self.body_json_key and key in manifest:
+                manifest[self.replacement_key] = {
+                    "type": "RequestBodyJson",
+                    "value": manifest[key],
+                }
+                manifest.pop(key, None)
+            elif key == self.body_data_key and key in manifest:
+                manifest[self.replacement_key] = {
+                    "type": "RequestBodyData",
+                    "value": manifest[key],
+                }
+                manifest.pop(key, None)
+
+    def validate(self, manifest: ManifestType) -> bool:
+        return self.replacement_key in manifest and all(
+            key not in manifest for key in self.original_keys
+        )

airbyte_cdk/manifest_migrations/migrations/http_requester_url_base_to_url.py

@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    TYPE_TAG,
+    ManifestMigration,
+    ManifestType,
+)
+
+
+class HttpRequesterUrlBaseToUrl(ManifestMigration):
+    """
+    This migration is responsible for migrating the `url_base` key to `url` in the HttpRequester component.
+    The `url_base` key is expected to be a base URL, and the `url` key is expected to be a full URL.
+    The migration will copy the value of `url_base` to `url`.
+    """
+
+    component_type = "HttpRequester"
+    original_key = "url_base"
+    replacement_key = "url"
+
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        return manifest[TYPE_TAG] == self.component_type and self.original_key in list(
+            manifest.keys()
+        )
+
+    def migrate(self, manifest: ManifestType) -> None:
+        manifest[self.replacement_key] = manifest[self.original_key]
+        manifest.pop(self.original_key, None)
+
+    def validate(self, manifest: ManifestType) -> bool:
+        """
+        Validate the migration by checking if the `url` key is present and the `url_base` key is not.
+        """
+        return (
+            self.replacement_key in manifest
+            and self.original_key not in manifest
+            and manifest[self.replacement_key] is not None
+        )

airbyte_cdk/manifest_migrations/migrations/registry.yaml

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+manifest_migrations:
+  - version: 6.48.2
+    migrations: 
+      - name: http_requester_url_base_to_url
+        order: 1
+        description: |
+          This migration updates the `url_base` field in the `http_requester` spec to `url`.
+          The `url_base` field is deprecated and will be removed in a future version.
+      - name: http_requester_path_to_url
+        order: 2
+        description: |
+          This migration updates the `path` field in the `http_requester` spec to `url`.
+          The `path` field is deprecated and will be removed in a future version.
+      - name: http_requester_request_body_json_data_to_request_body
+        order: 3
+        description: |
+          This migration updates the `request_body_json_data` field in the `http_requester` spec to `request_body`.
+          The `request_body_json_data` field is deprecated and will be removed in a future version.