airbyte-cdk 6.48.0__py3-none-any.whl → 6.48.1__py3-none-any.whl

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -56,6 +56,16 @@ def get_limits(config: Mapping[str, Any]) -> TestLimits:
     return TestLimits(max_records, max_pages_per_slice, max_slices, max_streams)
 
 
+def should_migrate_manifest(config: Mapping[str, Any]) -> bool:
+    """
+    Determines whether the manifest should be migrated,
+    based on the presence of the "__should_migrate" key in the config.
+
+    This flag is set by the UI.
+    """
+    return config.get("__should_migrate", False)
+
+
 def should_normalize_manifest(config: Mapping[str, Any]) -> bool:
     """
     Check if the manifest should be normalized.
@@ -71,6 +81,7 @@ def create_source(config: Mapping[str, Any], limits: TestLimits) -> ManifestDecl
         config=config,
         emit_connector_builder_messages=True,
         source_config=manifest,
+        migrate_manifest=should_migrate_manifest(config),
         normalize_manifest=should_normalize_manifest(config),
         component_factory=ModelToComponentFactory(
             emit_connector_builder_messages=True,

airbyte_cdk/manifest_migrations/README.md

@@ -0,0 +1,73 @@
+# Manifest Migrations
+
+This directory contains the logic and registry for manifest migrations in the Airbyte CDK. Migrations are used to update or transform manifest components to newer formats or schemas as the CDK evolves.
+
+## Adding a New Migration
+
+1. **Create a Migration File:**
+   - Add a new Python file in the `migrations/` subdirectory.
+   - Name the file using the pattern: `<description_of_the_migration>.py`.
+     - Example: `http_requester_url_base_to_url.py`
+   - The filename should be unique and descriptive.
+
+2. **Define the Migration Class:**
+   - The migration class must inherit from `ManifestMigration`.
+   - Name the class using a descriptive name (e.g., `HttpRequesterUrlBaseToUrl`).
+   - Implement the following methods:
+     - `should_migrate(self, manifest: ManifestType) -> bool`
+     - `migrate(self, manifest: ManifestType) -> None`
+     - `validate(self, manifest: ManifestType) -> bool`
+
+3. **Register the Migration:**
+   - Open `migrations/registry.yaml`.
+   - Add an entry under the appropriate version, or create a new version section if needed.
+   - Each migration entry should include:
+     - `name`: The filename (without `.py`)
+     - `order`: The order in which this migration should be applied for the version
+     - `description`: A short description of the migration
+
+   Example:
+   ```yaml
+   manifest_migrations:
+     - version: 6.45.2
+       migrations:
+         - name: http_requester_url_base_to_url
+           order: 1
+           description: |
+             This migration updates the `url_base` field in the `HttpRequester` component spec to `url`.
+   ```
+
+4. **Testing:**
+   - Ensure your migration is covered by unit tests.
+   - Tests should verify both `should_migrate`, `migrate`, and `validate` behaviors.
+
+## Migration Discovery
+
+- Migrations are discovered and registered automatically based on the entries in `migrations/registry.yaml`.
+- Do not modify the migration registry in code manually.
+- If you need to skip certain component types, use the `NON_MIGRATABLE_TYPES` list in `manifest_migration.py`.
+
+## Example Migration Skeleton
+
+```python
+from airbyte_cdk.manifest_migrations.manifest_migration import TYPE_TAG, ManifestMigration, ManifestType
+
+class ExampleMigration(ManifestMigration):
+    component_type = "ExampleComponent"
+    original_key = "old_key"
+    replacement_key = "new_key"
+
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        return manifest[TYPE_TAG] == self.component_type and self.original_key in manifest
+
+    def migrate(self, manifest: ManifestType) -> None:
+        manifest[self.replacement_key] = manifest[self.original_key]
+        manifest.pop(self.original_key, None)
+
+    def validate(self, manifest: ManifestType) -> bool:
+        return self.replacement_key in manifest and self.original_key not in manifest
+```
+
+---
+
+For more details, see the docstrings in `manifest_migration.py` and the examples in the `migrations/` folder.

airbyte_cdk/manifest_migrations/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#

airbyte_cdk/manifest_migrations/exceptions.py

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+class ManifestMigrationException(Exception):
+    """
+    Raised when a migration error occurs in the manifest.
+    """
+
+    def __init__(self, message: str) -> None:
+        super().__init__(f"Failed to migrate the manifest: {message}")

airbyte_cdk/manifest_migrations/manifest_migration.py

@@ -0,0 +1,134 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass
+from typing import Any, Dict
+
+ManifestType = Dict[str, Any]
+
+
+TYPE_TAG = "type"
+
+NON_MIGRATABLE_TYPES = [
+    # more info here: https://github.com/airbytehq/airbyte-internal-issues/issues/12423
+    "DynamicDeclarativeStream",
+]
+
+
+@dataclass
+class MigrationTrace:
+    """
+    This class represents a migration that has been applied to the manifest.
+    It contains information about the migration, including the version it was applied from,
+    the version it was applied to, and the time it was applied.
+    """
+
+    from_version: str
+    to_version: str
+    migration: str
+    migrated_at: str
+
+    def as_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+class ManifestMigration(ABC):
+    """
+    Base class for manifest migrations.
+    This class provides a framework for migrating manifest components.
+    It defines the structure for migration classes, including methods for checking if a migration is needed,
+    performing the migration, and validating the migration.
+    """
+
+    def __init__(self) -> None:
+        self.is_migrated: bool = False
+
+    @abstractmethod
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        """
+        Check if the manifest should be migrated.
+
+        :param manifest: The manifest to potentially migrate
+
+        :return: true if the manifest is of the expected format and should be migrated. False otherwise.
+        """
+
+    @abstractmethod
+    def migrate(self, manifest: ManifestType) -> None:
+        """
+        Migrate the manifest. Assumes should_migrate(manifest) returned True.
+
+        :param manifest: The manifest to migrate
+        """
+
+    @abstractmethod
+    def validate(self, manifest: ManifestType) -> bool:
+        """
+        Validate the manifest to ensure the migration was successfully applied.
+
+        :param manifest: The manifest to validate
+        """
+
+    def _is_component(self, obj: Dict[str, Any]) -> bool:
+        """
+        Check if the object is a component.
+
+        :param obj: The object to check
+        :return: True if the object is a component, False otherwise
+        """
+        return TYPE_TAG in obj.keys()
+
+    def _is_migratable_type(self, obj: Dict[str, Any]) -> bool:
+        """
+        Check if the object is a migratable component,
+        based on the Type of the component and the migration version.
+
+        :param obj: The object to check
+        :return: True if the object is a migratable component, False otherwise
+        """
+        return obj[TYPE_TAG] not in NON_MIGRATABLE_TYPES
+
+    def _process_manifest(self, obj: Any) -> None:
+        """
+        Recursively processes a manifest object, migrating components that match the migration criteria.
+
+        This method traverses the entire manifest structure (dictionaries and lists) and applies
+        migrations to components that:
+        1. Have a type tag
+        2. Are not in the list of non-migratable types
+        3. Meet the conditions defined in the should_migrate method
+
+        Parameters:
+            obj (Any): The object to process, which can be a dictionary, list, or any other type.
+                       Dictionary objects are checked for component type tags and potentially migrated.
+                       List objects have each of their items processed recursively.
+                       Other types are ignored.
+
+        Returns:
+            None, since we process the manifest in place.
+        """
+        if isinstance(obj, dict):
+            # Check if the object is a component
+            if self._is_component(obj):
+                # Check if the object is allowed to be migrated
+                if not self._is_migratable_type(obj):
+                    return
+
+                # Check if the object should be migrated
+                if self.should_migrate(obj):
+                    # Perform the migration, if needed
+                    self.migrate(obj)
+                    # validate the migration
+                    self.is_migrated = self.validate(obj)
+
+            # Process all values in the dictionary
+            for value in list(obj.values()):
+                self._process_manifest(value)
+
+        elif isinstance(obj, list):
+            # Process all items in the list
+            for item in obj:
+                self._process_manifest(item)

airbyte_cdk/manifest_migrations/migration_handler.py

@@ -0,0 +1,163 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+import copy
+import logging
+from datetime import datetime, timezone
+from typing import Type
+
+from packaging.version import Version
+
+from airbyte_cdk.manifest_migrations.exceptions import (
+    ManifestMigrationException,
+)
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    ManifestMigration,
+    ManifestType,
+    MigrationTrace,
+)
+from airbyte_cdk.manifest_migrations.migrations_registry import (
+    MANIFEST_MIGRATIONS,
+)
+
+METADATA_TAG = "metadata"
+MANIFEST_VERSION_TAG = "version"
+APPLIED_MIGRATIONS_TAG = "applied_migrations"
+
+LOGGER = logging.getLogger("airbyte.cdk.manifest_migrations")
+
+
+class ManifestMigrationHandler:
+    """
+    This class is responsible for handling migrations in the manifest.
+    """
+
+    def __init__(self, manifest: ManifestType) -> None:
+        self._manifest = manifest
+        self._migrated_manifest: ManifestType = copy.deepcopy(self._manifest)
+
+    def apply_migrations(self) -> ManifestType:
+        """
+        Apply all registered migrations to the manifest.
+
+        This method iterates through all migrations in the migrations registry and applies
+        them sequentially to the current manifest. If any migration fails with a
+        ManifestMigrationException, the original unmodified manifest is returned instead.
+
+        Returns:
+            ManifestType: The migrated manifest if all migrations succeeded, or the original
+                          manifest if any migration failed.
+        """
+        try:
+            manifest_version = self._get_manifest_version()
+            for migration_version, migrations in MANIFEST_MIGRATIONS.items():
+                for migration_cls in migrations:
+                    self._handle_migration(migration_cls, manifest_version, migration_version)
+            return self._migrated_manifest
+        except ManifestMigrationException:
+            # if any errors occur we return the original resolved manifest
+            return self._manifest
+
+    def _handle_migration(
+        self,
+        migration_class: Type[ManifestMigration],
+        manifest_version: str,
+        migration_version: str,
+    ) -> None:
+        """
+        Handles a single manifest migration by instantiating the migration class and processing the manifest.
+
+        Args:
+            migration_class (Type[ManifestMigration]): The migration class to apply to the manifest.
+
+        Raises:
+            ManifestMigrationException: If the migration process encounters any errors.
+        """
+        try:
+            migration_instance = migration_class()
+            if self._version_is_valid_for_migration(manifest_version, migration_version):
+                migration_instance._process_manifest(self._migrated_manifest)
+                if migration_instance.is_migrated:
+                    # set the updated manifest version, after migration has been applied
+                    self._set_manifest_version(migration_version)
+                    self._set_migration_trace(migration_class, manifest_version, migration_version)
+            else:
+                LOGGER.info(
+                    f"Manifest migration: `{self._get_migration_name(migration_class)}` is not supported for the given manifest version `{manifest_version}`.",
+                )
+        except Exception as e:
+            raise ManifestMigrationException(str(e)) from e
+
+    def _get_migration_name(self, migration_class: Type[ManifestMigration]) -> str:
+        """
+        Get the name of the migration instance.
+
+        Returns:
+            str: The name of the migration.
+        """
+        return migration_class.__name__
+
+    def _get_manifest_version(self) -> str:
+        """
+        Get the manifest version from the manifest.
+
+        :param manifest: The manifest to get the version from
+        :return: The manifest version
+        """
+        return str(self._migrated_manifest.get(MANIFEST_VERSION_TAG, "0.0.0"))
+
+    def _version_is_valid_for_migration(
+        self,
+        manifest_version: str,
+        migration_version: str,
+    ) -> bool:
+        """
+        Checks if the given manifest version is less than or equal to the specified migration version.
+
+        Args:
+            manifest_version (str): The version of the manifest to check.
+            migration_version (str): The migration version to compare against.
+
+        Returns:
+            bool: True if the manifest version is less than or equal to the migration version, False otherwise.
+        """
+        return Version(manifest_version) <= Version(migration_version)
+
+    def _set_manifest_version(self, version: str) -> None:
+        """
+        Set the manifest version in the manifest.
+
+        :param version: The version to set
+        """
+        self._migrated_manifest[MANIFEST_VERSION_TAG] = version
+
+    def _set_migration_trace(
+        self,
+        migration_instance: Type[ManifestMigration],
+        manifest_version: str,
+        migration_version: str,
+    ) -> None:
+        """
+        Set the migration trace in the manifest, under the `metadata.applied_migrations` property object.
+
+        :param migration_instance: The migration instance to set
+        :param manifest_version: The manifest version before migration
+        :param migration_version: The manifest version after migration
+        """
+
+        if METADATA_TAG not in self._migrated_manifest:
+            self._migrated_manifest[METADATA_TAG] = {}
+        if APPLIED_MIGRATIONS_TAG not in self._migrated_manifest[METADATA_TAG]:
+            self._migrated_manifest[METADATA_TAG][APPLIED_MIGRATIONS_TAG] = []
+
+        migration_trace = MigrationTrace(
+            from_version=manifest_version,
+            to_version=migration_version,
+            migration=self._get_migration_name(migration_instance),
+            migrated_at=datetime.now(tz=timezone.utc).isoformat(),
+        ).as_dict()
+
+        if migration_version not in self._migrated_manifest[METADATA_TAG][APPLIED_MIGRATIONS_TAG]:
+            self._migrated_manifest[METADATA_TAG][APPLIED_MIGRATIONS_TAG].append(migration_trace)

airbyte_cdk/manifest_migrations/migrations/__init__.py

@@ -0,0 +1,4 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+

airbyte_cdk/manifest_migrations/migrations/http_requester_path_to_url.py

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from urllib.parse import urljoin
+
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    TYPE_TAG,
+    ManifestMigration,
+    ManifestType,
+)
+from airbyte_cdk.sources.types import EmptyString
+
+
+class HttpRequesterPathToUrl(ManifestMigration):
+    """
+    This migration is responsible for migrating the `path` key to `url` in the HttpRequester component.
+    The `path` key is expected to be a relative path, and the `url` key is expected to be a full URL.
+    The migration will concatenate the `url_base` and `path` to form a full URL.
+    """
+
+    component_type = "HttpRequester"
+    original_key = "path"
+    replacement_key = "url"
+
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        return manifest[TYPE_TAG] == self.component_type and self.original_key in list(
+            manifest.keys()
+        )
+
+    def migrate(self, manifest: ManifestType) -> None:
+        original_key_value = manifest.get(self.original_key, EmptyString).lstrip("/")
+        replacement_key_value = manifest[self.replacement_key]
+
+        # return a full-url if provided directly from interpolation context
+        if original_key_value == EmptyString or original_key_value is None:
+            manifest[self.replacement_key] = replacement_key_value
+            manifest.pop(self.original_key, None)
+        else:
+            # since we didn't provide a full-url, the url_base might not have a trailing slash
+            # so we join the url_base and path correctly
+            if not replacement_key_value.endswith("/"):
+                replacement_key_value += "/"
+
+            manifest[self.replacement_key] = urljoin(replacement_key_value, original_key_value)
+            manifest.pop(self.original_key, None)
+
+    def validate(self, manifest: ManifestType) -> bool:
+        """
+        Validate the migration by checking if the `url` key is present and the `path` key is not.
+        """
+        return (
+            self.replacement_key in manifest
+            and self.original_key not in manifest
+            and manifest[self.replacement_key] is not None
+        )

airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py

@@ -0,0 +1,51 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    TYPE_TAG,
+    ManifestMigration,
+    ManifestType,
+)
+
+
+class HttpRequesterRequestBodyJsonDataToRequestBody(ManifestMigration):
+    """
+    This migration is responsible for migrating the `request_body_json` and `request_body_data` keys
+    to a unified `request_body` key in the HttpRequester component.
+    The migration will copy the value of either original key to `request_body` and remove the original key.
+    """
+
+    component_type = "HttpRequester"
+
+    body_json_key = "request_body_json"
+    body_data_key = "request_body_data"
+    original_keys = (body_json_key, body_data_key)
+
+    replacement_key = "request_body"
+
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        return manifest[TYPE_TAG] == self.component_type and any(
+            key in list(manifest.keys()) for key in self.original_keys
+        )
+
+    def migrate(self, manifest: ManifestType) -> None:
+        for key in self.original_keys:
+            if key == self.body_json_key and key in manifest:
+                manifest[self.replacement_key] = {
+                    "type": "RequestBodyJson",
+                    "value": manifest[key],
+                }
+                manifest.pop(key, None)
+            elif key == self.body_data_key and key in manifest:
+                manifest[self.replacement_key] = {
+                    "type": "RequestBodyData",
+                    "value": manifest[key],
+                }
+                manifest.pop(key, None)
+
+    def validate(self, manifest: ManifestType) -> bool:
+        return self.replacement_key in manifest and all(
+            key not in manifest for key in self.original_keys
+        )

airbyte_cdk/manifest_migrations/migrations/http_requester_url_base_to_url.py

@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    TYPE_TAG,
+    ManifestMigration,
+    ManifestType,
+)
+
+
+class HttpRequesterUrlBaseToUrl(ManifestMigration):
+    """
+    This migration is responsible for migrating the `url_base` key to `url` in the HttpRequester component.
+    The `url_base` key is expected to be a base URL, and the `url` key is expected to be a full URL.
+    The migration will copy the value of `url_base` to `url`.
+    """
+
+    component_type = "HttpRequester"
+    original_key = "url_base"
+    replacement_key = "url"
+
+    def should_migrate(self, manifest: ManifestType) -> bool:
+        return manifest[TYPE_TAG] == self.component_type and self.original_key in list(
+            manifest.keys()
+        )
+
+    def migrate(self, manifest: ManifestType) -> None:
+        manifest[self.replacement_key] = manifest[self.original_key]
+        manifest.pop(self.original_key, None)
+
+    def validate(self, manifest: ManifestType) -> bool:
+        """
+        Validate the migration by checking if the `url` key is present and the `url_base` key is not.
+        """
+        return (
+            self.replacement_key in manifest
+            and self.original_key not in manifest
+            and manifest[self.replacement_key] is not None
+        )

airbyte_cdk/manifest_migrations/migrations/registry.yaml

@@ -0,0 +1,22 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+manifest_migrations:
+  - version: 6.47.1
+    migrations: 
+      - name: http_requester_url_base_to_url
+        order: 1
+        description: |
+          This migration updates the `url_base` field in the `http_requester` spec to `url`.
+          The `url_base` field is deprecated and will be removed in a future version.
+      - name: http_requester_path_to_url
+        order: 2
+        description: |
+          This migration updates the `path` field in the `http_requester` spec to `url`.
+          The `path` field is deprecated and will be removed in a future version.
+      - name: http_requester_request_body_json_data_to_request_body
+        order: 3
+        description: |
+          This migration updates the `request_body_json_data` field in the `http_requester` spec to `request_body`.
+          The `request_body_json_data` field is deprecated and will be removed in a future version.

airbyte_cdk/manifest_migrations/migrations_registry.py

@@ -0,0 +1,76 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+import importlib
+import inspect
+import os
+from pathlib import Path
+from types import ModuleType
+from typing import Dict, List, Type
+
+import yaml
+
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    ManifestMigration,
+)
+
+DiscoveredMigrations = Dict[str, List[Type[ManifestMigration]]]
+
+MIGRATIONS_PATH = Path(__file__).parent / "migrations"
+REGISTRY_PATH = MIGRATIONS_PATH / "registry.yaml"
+
+
+def _find_migration_module(name: str) -> str:
+    """
+    Finds the migration module by name in the migrations directory.
+    The name should match the file name of the migration module (without the .py extension).
+    Raises ImportError if the module is not found.
+    """
+
+    for migration_file in os.listdir(MIGRATIONS_PATH):
+        migration_name = name + ".py"
+        if migration_file == migration_name:
+            return migration_file.replace(".py", "")
+
+    raise ImportError(f"Migration module '{name}' not found in {MIGRATIONS_PATH}.")
+
+
+def _get_migration_class(module: ModuleType) -> Type[ManifestMigration]:
+    """
+    Returns the ManifestMigration subclass defined in the module.
+    """
+    for _, obj in inspect.getmembers(module, inspect.isclass):
+        if issubclass(obj, ManifestMigration):
+            return obj
+
+    raise ImportError(f"No ManifestMigration subclass found in module {module.__name__}.")
+
+
+def _discover_migrations() -> DiscoveredMigrations:
+    """
+    Discovers and returns a list of ManifestMigration subclasses in the order specified by registry.yaml.
+    """
+    with open(REGISTRY_PATH, "r") as f:
+        registry = yaml.safe_load(f)
+        migrations: DiscoveredMigrations = {}
+        # Iterate through the registry and import the migration classes
+        # based on the version and order specified in the registry.yaml
+        for version_entry in registry.get("manifest_migrations", []):
+            migration_version = version_entry.get("version", "0.0.0")
+            if not migration_version in migrations:
+                migrations[migration_version] = []
+
+            for migration in sorted(version_entry.get("migrations", []), key=lambda m: m["order"]):
+                module = importlib.import_module(
+                    f"airbyte_cdk.manifest_migrations.migrations.{_find_migration_module(migration['name'])}"
+                )
+                migration_class = _get_migration_class(module)
+                migrations[migration_version].append(migration_class)
+
+    return migrations
+
+
+# registered migrations
+MANIFEST_MIGRATIONS: DiscoveredMigrations = _discover_migrations()

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -15,6 +15,9 @@ from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
 from packaging.version import InvalidVersion, Version
 
+from airbyte_cdk.manifest_migrations.migration_handler import (
+    ManifestMigrationHandler,
+)
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteMessage,
@@ -91,6 +94,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         debug: bool = False,
         emit_connector_builder_messages: bool = False,
         component_factory: Optional[ModelToComponentFactory] = None,
+        migrate_manifest: Optional[bool] = False,
         normalize_manifest: Optional[bool] = False,
     ) -> None:
         """
@@ -104,12 +108,11 @@ class ManifestDeclarativeSource(DeclarativeSource):
         """
         self.logger = logging.getLogger(f"airbyte.{self.name}")
         self._should_normalize = normalize_manifest
+        self._should_migrate = migrate_manifest
         self._declarative_component_schema = _get_declarative_component_schema()
         # If custom components are needed, locate and/or register them.
         self.components_module: ModuleType | None = get_registered_components_module(config=config)
-        # resolve all components in the manifest
-        self._source_config = self._preprocess_manifest(dict(source_config))
-
+        # set additional attributes
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._constructor = (
@@ -126,11 +129,12 @@ class ManifestDeclarativeSource(DeclarativeSource):
         )
         self._config = config or {}
 
+        # resolve all components in the manifest
+        self._source_config = self._pre_process_manifest(dict(source_config))
         # validate resolved manifest against the declarative component schema
         self._validate_source()
-
         # apply additional post-processing to the manifest
-        self._postprocess_manifest()
+        self._post_process_manifest()
 
     @property
     def resolved_manifest(self) -> Mapping[str, Any]:
@@ -145,7 +149,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         """
         return self._source_config
 
-    def _preprocess_manifest(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
+    def _pre_process_manifest(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
         """
         Preprocesses the provided manifest dictionary by resolving any manifest references.
 
@@ -169,12 +173,14 @@ class ManifestDeclarativeSource(DeclarativeSource):
 
         return propagated_manifest
 
-    def _postprocess_manifest(self) -> None:
+    def _post_process_manifest(self) -> None:
         """
         Post-processes the manifest after validation.
         This method is responsible for any additional modifications or transformations needed
         after the manifest has been validated and before it is used in the source.
         """
+        # apply manifest migration, if required
+        self._migrate_manifest()
         # apply manifest normalization, if required
         self._normalize_manifest()
 
@@ -190,6 +196,19 @@ class ManifestDeclarativeSource(DeclarativeSource):
             normalizer = ManifestNormalizer(self._source_config, self._declarative_component_schema)
             self._source_config = normalizer.normalize()
 
+    def _migrate_manifest(self) -> None:
+        """
+        This method is used to migrate the manifest. It should be called after the manifest has been validated.
+        The migration is done in place, so the original manifest is modified.
+
+        The original manifest is returned if any error occurs during migration.
+        """
+        if self._should_migrate:
+            manifest_migrator = ManifestMigrationHandler(self._source_config)
+            self._source_config = manifest_migrator.apply_migrations()
+            # validate migrated manifest against the declarative component schema
+            self._validate_source()
+
     def _fix_source_type(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
         """
         Fix the source type in the manifest. This is necessary because the source type is not always set in the manifest.

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -2833,13 +2833,9 @@ class ModelToComponentFactory:
             else None
         )
 
-        if model.transform_before_filtering is None:
-            # default to False if not set
-            model.transform_before_filtering = False
-
-        assert model.transform_before_filtering is not None  # for mypy
-
-        transform_before_filtering = model.transform_before_filtering
+        transform_before_filtering = (
+            False if model.transform_before_filtering is None else model.transform_before_filtering
+        )
         if client_side_incremental_sync:
             record_filter = ClientSideIncrementalRecordFilterDecorator(
                 config=config,
@@ -2849,7 +2845,11 @@ class ModelToComponentFactory:
                 else None,
                 **client_side_incremental_sync,
             )
-            transform_before_filtering = True
+            transform_before_filtering = (
+                True
+                if model.transform_before_filtering is None
+                else model.transform_before_filtering
+            )
 
         if model.schema_normalization is None:
             # default to no schema normalization if not set

{airbyte_cdk-6.48.0.dist-info → airbyte_cdk-6.48.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.48.0
+Version: 6.48.1
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.48.0.dist-info → airbyte_cdk-6.48.1.dist-info}/RECORD

@@ -14,7 +14,7 @@ airbyte_cdk/config_observation.py,sha256=7SSPxtN0nXPkm4euGNcTTr1iLbwUL01jy-24V1H
 airbyte_cdk/connector.py,sha256=bO23kdGRkl8XKFytOgrrWFc_VagteTHVEF6IsbizVkM,4224
 airbyte_cdk/connector_builder/README.md,sha256=Hw3wvVewuHG9-QgsAq1jDiKuLlStDxKBz52ftyNRnBw,1665
 airbyte_cdk/connector_builder/__init__.py,sha256=4Hw-PX1-VgESLF16cDdvuYCzGJtHntThLF4qIiULWeo,61
-airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=bJ7ufXOWLXN4i8QQ6WhAsnQY2AqlZq0NVL5G-TAPa1g,6241
+airbyte_cdk/connector_builder/connector_builder_handler.py,sha256=OFTzxyfAevI3Um8fXTOLTgoCc4Sx9NzF0boqYkAATfM,6590
 airbyte_cdk/connector_builder/main.py,sha256=j1pP5N8RsnvQZ4iYxhLdLEHsJ5Ui7IVFBUi6wYMGBkM,3839
 airbyte_cdk/connector_builder/models.py,sha256=9pIZ98LW_d6fRS39VdnUOf3cxGt4TkC5MJ0_OrzcCRk,1578
 airbyte_cdk/connector_builder/test_reader/__init__.py,sha256=iTwBMoI9vaJotEgpqZbFjlxRcbxXYypSVJ9YxeHk7wc,120
@@ -36,6 +36,17 @@ airbyte_cdk/destinations/vector_db_based/writer.py,sha256=nZ00xPiohElJmYktEZZIhr
 airbyte_cdk/entrypoint.py,sha256=NRJv5BNZRSUEVTmNBa9N7ih6fW5sg4DwL0nkB9kI99Y,18570
 airbyte_cdk/exception_handler.py,sha256=D_doVl3Dt60ASXlJsfviOCswxGyKF2q0RL6rif3fNks,2013
 airbyte_cdk/logger.py,sha256=1cURbvawbunCAV178q-XhTHcbAQZTSf07WhU7U9AXWU,3744
+airbyte_cdk/manifest_migrations/README.md,sha256=PvnbrW1gyzhlkeucd0YAOXcXVxi0xBUUynzs4DMqjDo,2942
+airbyte_cdk/manifest_migrations/__init__.py,sha256=0eq9ic_6GGXMwzE31eAOSA7PLtBauMfgM9XshjYHF84,61
+airbyte_cdk/manifest_migrations/exceptions.py,sha256=mmMZaCVEkYSGykVL5jKA0xsDWWkybRdQwnh9pGb7VG0,300
+airbyte_cdk/manifest_migrations/manifest_migration.py,sha256=4ohLfbj2PeuPSgCMVbCArb0d-YdaZIllX4ieXQNiRRw,4420
+airbyte_cdk/manifest_migrations/migration_handler.py,sha256=CF8in-Eb45TGzFBxEJrXSzqVr8Lgv0vqvZlbuz1rbQk,6096
+airbyte_cdk/manifest_migrations/migrations/__init__.py,sha256=SJ7imfOgCRYOVaFkW2bVEnSUxbYPlkryWwYT2semsF0,62
+airbyte_cdk/manifest_migrations/migrations/http_requester_path_to_url.py,sha256=IIn2SjRh1v2yaSBFUCDyBHpX6mBhlckhvbsSg55mREI,2153
+airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py,sha256=70md8yDu8SWl2JkkFcEs8kyXUbP0F_obIzyHsygyR9k,1777
+airbyte_cdk/manifest_migrations/migrations/http_requester_url_base_to_url.py,sha256=EX1MVYVpoWypA28qoH48wA0SYZjGdlR8bcSixTDzfgo,1346
+airbyte_cdk/manifest_migrations/migrations/registry.yaml,sha256=56B09vbBlGWKF5KSB-3rashZLRiJmL8si_lmTokXWsc,960
+airbyte_cdk/manifest_migrations/migrations_registry.py,sha256=zly2fwaOxDukqC7eowzrDlvhA2v71FjW74kDzvRXhSY,2619
 airbyte_cdk/models/__init__.py,sha256=Et9wJWs5VOWynGbb-3aJRhsdAHAiLkNNLxdwqJAuqkw,2114
 airbyte_cdk/models/airbyte_protocol.py,sha256=oZdKsZ7yPjUt9hvxdWNpxCtgjSV2RWhf4R9Np03sqyY,3613
 airbyte_cdk/models/airbyte_protocol_serializers.py,sha256=s6SaFB2CMrG_7jTQGn_fhFbQ1FUxhCxf5kq2RWGHMVI,1749
@@ -116,7 +127,7 @@ airbyte_cdk/sources/declarative/interpolation/interpolated_string.py,sha256=CQkH
 airbyte_cdk/sources/declarative/interpolation/interpolation.py,sha256=9IoeuWam3L6GyN10L6U8xNWXmkt9cnahSDNkez1OmFY,982
 airbyte_cdk/sources/declarative/interpolation/jinja.py,sha256=UQeuS4Vpyp4hlOn-R3tRyeBX0e9IoV6jQ6gH-Jz8lY0,7182
 airbyte_cdk/sources/declarative/interpolation/macros.py,sha256=UYSJ5gW7TkHALYnNvUnRP3RlyGwGuRMObF3BHuNzjJM,5320
-airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=trTSVPJVAs6RN2sMeaLSuQT3jzJk8ZbdkUP6DMU29Ow,22011
+airbyte_cdk/sources/declarative/manifest_declarative_source.py,sha256=yPQ4Qsxt8rzgbj52TLVFJiSRJp4lbFF-H7dSsr88E58,22961
 airbyte_cdk/sources/declarative/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py,sha256=iemy3fKLczcU0-Aor7tx5jcT6DRedKMqyK7kCOp01hg,3924
 airbyte_cdk/sources/declarative/migrations/state_migration.py,sha256=KWPjealMLKSMtajXgkdGgKg7EmTLR-CqqD7UIh0-eDU,794
@@ -128,7 +139,7 @@ airbyte_cdk/sources/declarative/parsers/custom_exceptions.py,sha256=wnRUP0Xeru9R
 airbyte_cdk/sources/declarative/parsers/manifest_component_transformer.py,sha256=RUyFZS0zslLb7UfQrvqMC--k5CVLNSp7zHw6kbosvKE,9688
 airbyte_cdk/sources/declarative/parsers/manifest_normalizer.py,sha256=laBy7ebjA-PiNwc-50U4FHvMqS_mmHvnabxgFs4CjGw,17069
 airbyte_cdk/sources/declarative/parsers/manifest_reference_resolver.py,sha256=pJmg78vqE5VfUrF_KJnWjucQ4k9IWFULeAxHCowrHXE,6806
-airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=DGHB4nWwsm4YLuWHhuOjAdeyVDY2U3CdbAiGRdm1YF0,163771
+airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py,sha256=YXR_cKEL6U-B6eji-FxvHFw_6V-bBAVmC4-M5IawXtQ,163774
 airbyte_cdk/sources/declarative/partition_routers/__init__.py,sha256=TBC9AkGaUqHm2IKHMPN6punBIcY5tWGULowcLoAVkfw,1109
 airbyte_cdk/sources/declarative/partition_routers/async_job_partition_router.py,sha256=VelO7zKqKtzMJ35jyFeg0ypJLQC0plqqIBNXoBW1G2E,3001
 airbyte_cdk/sources/declarative/partition_routers/cartesian_product_stream_slicer.py,sha256=c5cuVFM6NFkuQqG8Z5IwkBuwDrvXZN1CunUOM_L0ezg,6892
@@ -396,9 +407,9 @@ airbyte_cdk/utils/slice_hasher.py,sha256=EDxgROHDbfG-QKQb59m7h_7crN1tRiawdf5uU7G
 airbyte_cdk/utils/spec_schema_transformations.py,sha256=-5HTuNsnDBAhj-oLeQXwpTGA0HdcjFOf2zTEMUTTg_Y,816
 airbyte_cdk/utils/stream_status_utils.py,sha256=ZmBoiy5HVbUEHAMrUONxZvxnvfV9CesmQJLDTAIWnWw,1171
 airbyte_cdk/utils/traced_exception.py,sha256=C8uIBuCL_E4WnBAOPSxBicD06JAldoN9fGsQDp463OY,6292
-airbyte_cdk-6.48.0.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
-airbyte_cdk-6.48.0.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
-airbyte_cdk-6.48.0.dist-info/METADATA,sha256=slRcZy2PXZpm5mLpefKQWolH6bgEoo1Wy285FcBhxvc,6323
-airbyte_cdk-6.48.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_cdk-6.48.0.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
-airbyte_cdk-6.48.0.dist-info/RECORD,,
+airbyte_cdk-6.48.1.dist-info/LICENSE.txt,sha256=Wfe61S4BaGPj404v8lrAbvhjYR68SHlkzeYrg3_bbuM,1051
+airbyte_cdk-6.48.1.dist-info/LICENSE_SHORT,sha256=aqF6D1NcESmpn-cqsxBtszTEnHKnlsp8L4x9wAh3Nxg,55
+airbyte_cdk-6.48.1.dist-info/METADATA,sha256=scl_OXlgL349P2PgmdntFjkKVIBv9d_dPTIF7S2iKmA,6323
+airbyte_cdk-6.48.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_cdk-6.48.1.dist-info/entry_points.txt,sha256=AKWbEkHfpzzk9nF9tqBUaw1MbvTM4mGtEzmZQm0ZWvM,139
+airbyte_cdk-6.48.1.dist-info/RECORD,,