airbyte-cdk 6.48.0__py3-none-any.whl → 6.48.2__py3-none-any.whl

airbyte_cdk/manifest_migrations/migrations_registry.py

@@ -0,0 +1,76 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+import importlib
+import inspect
+import os
+from pathlib import Path
+from types import ModuleType
+from typing import Dict, List, Type
+
+import yaml
+
+from airbyte_cdk.manifest_migrations.manifest_migration import (
+    ManifestMigration,
+)
+
+DiscoveredMigrations = Dict[str, List[Type[ManifestMigration]]]
+
+MIGRATIONS_PATH = Path(__file__).parent / "migrations"
+REGISTRY_PATH = MIGRATIONS_PATH / "registry.yaml"
+
+
+def _find_migration_module(name: str) -> str:
+    """
+    Finds the migration module by name in the migrations directory.
+    The name should match the file name of the migration module (without the .py extension).
+    Raises ImportError if the module is not found.
+    """
+
+    for migration_file in os.listdir(MIGRATIONS_PATH):
+        migration_name = name + ".py"
+        if migration_file == migration_name:
+            return migration_file.replace(".py", "")
+
+    raise ImportError(f"Migration module '{name}' not found in {MIGRATIONS_PATH}.")
+
+
+def _get_migration_class(module: ModuleType) -> Type[ManifestMigration]:
+    """
+    Returns the ManifestMigration subclass defined in the module.
+    """
+    for _, obj in inspect.getmembers(module, inspect.isclass):
+        if issubclass(obj, ManifestMigration):
+            return obj
+
+    raise ImportError(f"No ManifestMigration subclass found in module {module.__name__}.")
+
+
+def _discover_migrations() -> DiscoveredMigrations:
+    """
+    Discovers and returns a list of ManifestMigration subclasses in the order specified by registry.yaml.
+    """
+    with open(REGISTRY_PATH, "r") as f:
+        registry = yaml.safe_load(f)
+        migrations: DiscoveredMigrations = {}
+        # Iterate through the registry and import the migration classes
+        # based on the version and order specified in the registry.yaml
+        for version_entry in registry.get("manifest_migrations", []):
+            migration_version = version_entry.get("version", "0.0.0")
+            if not migration_version in migrations:
+                migrations[migration_version] = []
+
+            for migration in sorted(version_entry.get("migrations", []), key=lambda m: m["order"]):
+                module = importlib.import_module(
+                    f"airbyte_cdk.manifest_migrations.migrations.{_find_migration_module(migration['name'])}"
+                )
+                migration_class = _get_migration_class(module)
+                migrations[migration_version].append(migration_class)
+
+    return migrations
+
+
+# registered migrations
+MANIFEST_MIGRATIONS: DiscoveredMigrations = _discover_migrations()

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1911,15 +1911,16 @@ definitions:
     type: object
     required:
       - type
-      - url_base
     properties:
       type:
         type: string
         enum: [HttpRequester]
       url_base:
-        linkable: true
+        deprecated: true
+        deprecation_message: "Use `url` field instead."
         title: API Base URL
-        description: The Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        linkable: true
         type: string
         interpolation_context:
           - config
@@ -1935,9 +1936,29 @@ definitions:
           - "{{ config['base_url'] or 'https://app.posthog.com'}}/api"
           - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups"
           - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
+      url:
+        title: The URL of an API endpoint
+        description: The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        type: string
+        interpolation_context:
+          - config
+          - next_page_token
+          - stream_interval
+          - stream_partition
+          - stream_slice
+          - creation_response
+          - polling_response
+          - download_target
+        examples:
+          - "https://connect.squareup.com/v2"
+          - "{{ config['url'] or 'https://app.posthog.com'}}/api"
+          - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups"
+          - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}"
       path:
+        deprecated: true
+        deprecation_message: "Use `url` field instead."
         title: URL Path
-        description: The Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
+        description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.
         type: string
         interpolation_context:
           - config
@@ -1983,6 +2004,8 @@ definitions:
         description: Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.
         "$ref": "#/definitions/PropertiesFromEndpoint"
       request_body_data:
+        deprecated: true
+        deprecation_message: "Use `request_body` field instead."
         title: Request Body Payload (Non-JSON)
         description: Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.
         anyOf:
@@ -2001,6 +2024,8 @@ definitions:
                 [{"value": {{ stream_interval['start_time'] | int * 1000 }} }]
               }, "orderBy": 1, "columnName": "Timestamp"}]/
       request_body_json:
+        deprecated: true
+        deprecation_message: "Use `request_body` field instead."
         title: Request Body JSON Payload
         description: Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.
         anyOf:
@@ -2019,6 +2044,35 @@ definitions:
           - sort:
               field: "updated_at"
               order: "ascending"
+      request_body:
+        title: Request Body Payload to be send as a part of the API request.
+        description: Specifies how to populate the body of the request with a payload. Can contain nested objects.
+        anyOf:
+          - "$ref": "#/definitions/RequestBody"
+        interpolation_context:
+          - next_page_token
+          - stream_interval
+          - stream_partition
+          - stream_slice
+        examples:
+          - type: RequestBodyJson
+            value:
+              sort_order: "ASC"
+              sort_field: "CREATED_AT"
+          - type: RequestBodyJson
+            value:
+              key: "{{ config['value'] }}"
+          - type: RequestBodyJson
+            value:
+              sort:
+                field: "updated_at"
+                order: "ascending"
+          - type: RequestBodyData
+            value: "plain_text_body"
+          - type: RequestBodyData
+            value:
+              param1: "value1"
+              param2: "{{ config['param2_value'] }}"
       request_headers:
         title: Request Headers
         description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.
@@ -4019,6 +4073,27 @@ definitions:
       - type
       - stream_template
       - components_resolver
+  RequestBody:
+    type: object
+    description: The request body payload. Can be either URL encoded data or JSON.
+    properties:
+      type:
+        anyOf:
+          - type: string
+            enum: [RequestBodyData]
+          - type: string
+            enum: [RequestBodyJson]
+      value:
+        anyOf:
+          - type: string
+            description: The request body payload as a string.
+          - type: object
+            description: The request body payload as a Non-JSON object (url-encoded data).
+            additionalProperties:
+              type: string
+          - type: object
+            description: The request body payload as a JSON object (json-encoded data).
+            additionalProperties: true
 interpolation:
   variables:
     - title: config
@@ -4227,4 +4302,4 @@ interpolation:
         regex: The regular expression to search for. It must include a capture group.
       return_type: str
       examples:
-        - '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'
+        - '{{ "goodbye, cruel world" | regex_search("goodbye,\s(.*)$") }} -> "cruel world"'

airbyte_cdk/sources/declarative/declarative_source.py

@@ -4,8 +4,11 @@
 
 import logging
 from abc import abstractmethod
-from typing import Any, Mapping, Tuple
+from typing import Any, List, Mapping, Tuple
 
+from airbyte_cdk.connector_builder.models import (
+    LogMessage as ConnectorBuilderLogMessage,
+)
 from airbyte_cdk.sources.abstract_source import AbstractSource
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 
@@ -34,3 +37,9 @@ class DeclarativeSource(AbstractSource):
           The error object will be cast to string to display the problem to the user.
         """
         return self.connection_checker.check_connection(self, logger, config)
+
+    def deprecation_warnings(self) -> List[ConnectorBuilderLogMessage]:
+        """
+        Returns a list of deprecation warnings for the source.
+        """
+        return []

airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py

@@ -12,7 +12,7 @@ from airbyte_cdk.sources.types import Config
 NestedMappingEntry = Union[
     dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None
 ]
-NestedMapping = Union[dict[str, NestedMappingEntry], str]
+NestedMapping = Union[dict[str, NestedMappingEntry], str, dict[str, Any]]
 
 
 @dataclass

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -15,6 +15,12 @@ from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
 from packaging.version import InvalidVersion, Version
 
+from airbyte_cdk.connector_builder.models import (
+    LogMessage as ConnectorBuilderLogMessage,
+)
+from airbyte_cdk.manifest_migrations.migration_handler import (
+    ManifestMigrationHandler,
+)
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteMessage,
@@ -91,6 +97,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         debug: bool = False,
         emit_connector_builder_messages: bool = False,
         component_factory: Optional[ModelToComponentFactory] = None,
+        migrate_manifest: Optional[bool] = False,
         normalize_manifest: Optional[bool] = False,
     ) -> None:
         """
@@ -104,12 +111,11 @@ class ManifestDeclarativeSource(DeclarativeSource):
         """
         self.logger = logging.getLogger(f"airbyte.{self.name}")
         self._should_normalize = normalize_manifest
+        self._should_migrate = migrate_manifest
         self._declarative_component_schema = _get_declarative_component_schema()
         # If custom components are needed, locate and/or register them.
         self.components_module: ModuleType | None = get_registered_components_module(config=config)
-        # resolve all components in the manifest
-        self._source_config = self._preprocess_manifest(dict(source_config))
-
+        # set additional attributes
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages
         self._constructor = (
@@ -126,11 +132,12 @@ class ManifestDeclarativeSource(DeclarativeSource):
         )
         self._config = config or {}
 
+        # resolve all components in the manifest
+        self._source_config = self._pre_process_manifest(dict(source_config))
         # validate resolved manifest against the declarative component schema
         self._validate_source()
-
         # apply additional post-processing to the manifest
-        self._postprocess_manifest()
+        self._post_process_manifest()
 
     @property
     def resolved_manifest(self) -> Mapping[str, Any]:
@@ -145,7 +152,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         """
         return self._source_config
 
-    def _preprocess_manifest(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
+    def _pre_process_manifest(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
         """
         Preprocesses the provided manifest dictionary by resolving any manifest references.
 
@@ -169,12 +176,14 @@ class ManifestDeclarativeSource(DeclarativeSource):
 
         return propagated_manifest
 
-    def _postprocess_manifest(self) -> None:
+    def _post_process_manifest(self) -> None:
         """
         Post-processes the manifest after validation.
         This method is responsible for any additional modifications or transformations needed
         after the manifest has been validated and before it is used in the source.
         """
+        # apply manifest migration, if required
+        self._migrate_manifest()
         # apply manifest normalization, if required
         self._normalize_manifest()
 
@@ -190,6 +199,19 @@ class ManifestDeclarativeSource(DeclarativeSource):
             normalizer = ManifestNormalizer(self._source_config, self._declarative_component_schema)
             self._source_config = normalizer.normalize()
 
+    def _migrate_manifest(self) -> None:
+        """
+        This method is used to migrate the manifest. It should be called after the manifest has been validated.
+        The migration is done in place, so the original manifest is modified.
+
+        The original manifest is returned if any error occurs during migration.
+        """
+        if self._should_migrate:
+            manifest_migrator = ManifestMigrationHandler(self._source_config)
+            self._source_config = manifest_migrator.apply_migrations()
+            # validate migrated manifest against the declarative component schema
+            self._validate_source()
+
     def _fix_source_type(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
         """
         Fix the source type in the manifest. This is necessary because the source type is not always set in the manifest.
@@ -211,6 +233,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
             with_dynamic_stream_name=True,
         )
 
+    def deprecation_warnings(self) -> List[ConnectorBuilderLogMessage]:
+        return self._constructor.get_model_deprecations()
+
     @property
     def connection_checker(self) -> ConnectionChecker:
         check = self._source_config["check"]

airbyte_cdk/sources/declarative/models/base_model_with_deprecations.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+
+# THIS IS A STATIC CLASS MODEL USED TO DISPLAY DEPRECATION WARNINGS
+# WHEN DEPRECATED FIELDS ARE ACCESSED
+
+import warnings
+from typing import Any, List
+
+from pydantic.v1 import BaseModel
+
+from airbyte_cdk.connector_builder.models import LogMessage as ConnectorBuilderLogMessage
+
+# format the warning message
+warnings.formatwarning = (
+    lambda message, category, *args, **kwargs: f"{category.__name__}: {message}"
+)
+
+FIELDS_TAG = "__fields__"
+DEPRECATED = "deprecated"
+DEPRECATION_MESSAGE = "deprecation_message"
+DEPRECATION_LOGS_TAG = "_deprecation_logs"
+
+
+class BaseModelWithDeprecations(BaseModel):
+    """
+    Pydantic BaseModel that warns when deprecated fields are accessed.
+    The deprecation message is stored in the field's extra attributes.
+    This class is used to create models that can have deprecated fields
+    and show warnings when those fields are accessed or initialized.
+
+    The `_deprecation_logs` attribute is stored in the model itself.
+    The collected deprecation warnings are further propagated to the Airbyte log messages,
+    during the component creation process, in `model_to_component._collect_model_deprecations()`.
+
+    The component implementation is not responsible for handling the deprecation warnings,
+    since the deprecation warnings are already handled in the model itself.
+    """
+
+    class Config:
+        """
+        Allow extra fields in the model. In case the model restricts extra fields.
+        """
+
+        extra = "allow"
+
+    def __init__(self, **model_fields: Any) -> None:
+        """
+        Show warnings for deprecated fields during component initialization.
+        """
+        # call the parent constructor first to initialize Pydantic internals
+        super().__init__(**model_fields)
+        # set the placeholder for the default deprecation messages
+        self._default_deprecation_messages: List[str] = []
+        # set the placeholder for the deprecation logs
+        self._deprecation_logs: List[ConnectorBuilderLogMessage] = []
+        # process deprecated fields, if present
+        self._process_fields(model_fields)
+        # emit default deprecation messages
+        self._emit_default_deprecation_messages()
+        # set the deprecation logs attribute to the model
+        self._set_deprecation_logs_attr_to_model()
+
+    def _is_deprecated_field(self, field_name: str) -> bool:
+        return (
+            self.__fields__[field_name].field_info.extra.get(DEPRECATED, False)
+            if field_name in self.__fields__.keys()
+            else False
+        )
+
+    def _get_deprecation_message(self, field_name: str) -> str:
+        return (
+            self.__fields__[field_name].field_info.extra.get(
+                DEPRECATION_MESSAGE, "<missing_deprecation_message>"
+            )
+            if field_name in self.__fields__.keys()
+            else "<missing_deprecation_message>"
+        )
+
+    def _process_fields(self, model_fields: Any) -> None:
+        """
+        Processes the fields in the provided model data, checking for deprecated fields.
+
+        For each field in the input `model_fields`, this method checks if the field exists in the model's defined fields.
+        If the field is marked as deprecated (using the `DEPRECATED` flag in its metadata), it triggers a deprecation warning
+        by calling the `_create_warning` method with the field name and an optional deprecation message.
+
+        Args:
+            model_fields (Any): The data containing fields to be processed.
+
+        Returns:
+            None
+        """
+
+        if hasattr(self, FIELDS_TAG):
+            for field_name in model_fields.keys():
+                if self._is_deprecated_field(field_name):
+                    self._create_warning(
+                        field_name,
+                        self._get_deprecation_message(field_name),
+                    )
+
+    def _set_deprecation_logs_attr_to_model(self) -> None:
+        """
+        Sets the deprecation logs attribute on the model instance.
+
+        This method attaches the current instance's deprecation logs to the model by setting
+        an attribute named by `DEPRECATION_LOGS_TAG` to the value of `self._deprecation_logs`.
+        This is typically used to track or log deprecated features or configurations within the model.
+
+        Returns:
+            None
+        """
+        setattr(self, DEPRECATION_LOGS_TAG, self._deprecation_logs)
+
+    def _create_warning(self, field_name: str, message: str) -> None:
+        """
+        Show a warning message for deprecated fields (to stdout).
+        Args:
+            field_name (str): Name of the deprecated field.
+            message (str): Warning message to be displayed.
+        """
+
+        deprecated_message = f"Component type: `{self.__class__.__name__}`. Field '{field_name}' is deprecated. {message}"
+
+        if deprecated_message not in self._default_deprecation_messages:
+            # Avoid duplicates in the default deprecation messages
+            self._default_deprecation_messages.append(deprecated_message)
+
+        # Create an Airbyte deprecation log message
+        deprecation_log_message = ConnectorBuilderLogMessage(
+            level="WARN", message=deprecated_message
+        )
+        # Add the deprecation message to the Airbyte log messages,
+        # this logs are displayed in the Connector Builder.
+        if deprecation_log_message not in self._deprecation_logs:
+            # Avoid duplicates in the deprecation logs
+            self._deprecation_logs.append(deprecation_log_message)
+
+    def _emit_default_deprecation_messages(self) -> None:
+        """
+        Emit default deprecation messages for deprecated fields to STDOUT.
+        """
+        for message in self._default_deprecation_messages:
+            warnings.warn(message, DeprecationWarning)

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -10,6 +10,10 @@ from typing import Any, Dict, List, Literal, Optional, Union
 
 from pydantic.v1 import BaseModel, Extra, Field
 
+from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import (
+    BaseModelWithDeprecations,
+)
+
 
 class AuthFlowType(Enum):
     oauth2_0 = "oauth2.0"
@@ -1497,6 +1501,11 @@ class ConfigComponentsResolver(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class RequestBody(BaseModel):
+    type: Optional[Union[Literal["RequestBodyData"], Literal["RequestBodyJson"]]] = None
+    value: Optional[Union[str, Dict[str, str], Dict[str, Any]]] = None
+
+
 class AddedFieldDefinition(BaseModel):
     type: Literal["AddedFieldDefinition"]
     path: List[str] = Field(
@@ -2207,11 +2216,13 @@ class SessionTokenAuthenticator(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
-class HttpRequester(BaseModel):
+class HttpRequester(BaseModelWithDeprecations):
     type: Literal["HttpRequester"]
-    url_base: str = Field(
-        ...,
-        description="The Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+    url_base: Optional[str] = Field(
+        None,
+        deprecated=True,
+        deprecation_message="Use `url` field instead.",
+        description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "https://connect.squareup.com/v2",
             "{{ config['base_url'] or 'https://app.posthog.com'}}/api",
@@ -2220,9 +2231,22 @@ class HttpRequester(BaseModel):
         ],
         title="API Base URL",
     )
+    url: Optional[str] = Field(
+        None,
+        description="The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        examples=[
+            "https://connect.squareup.com/v2",
+            "{{ config['url'] or 'https://app.posthog.com'}}/api",
+            "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups",
+            "https://example.com/api/v1/resource/{{ next_page_token['id'] }}",
+        ],
+        title="The URL of an API endpoint",
+    )
     path: Optional[str] = Field(
         None,
-        description="The Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
+        deprecated=True,
+        deprecation_message="Use `url` field instead.",
+        description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.",
         examples=[
             "/products",
             "/quotes/{{ stream_partition['id'] }}/quote_line_groups",
@@ -2261,6 +2285,8 @@ class HttpRequester(BaseModel):
     )
     request_body_data: Optional[Union[Dict[str, str], str]] = Field(
         None,
+        deprecated=True,
+        deprecation_message="Use `request_body` field instead.",
         description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.",
         examples=[
             '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n    [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n  }, "orderBy": 1, "columnName": "Timestamp"}]/\n'
@@ -2269,6 +2295,8 @@ class HttpRequester(BaseModel):
     )
     request_body_json: Optional[Union[Dict[str, Any], str]] = Field(
         None,
+        deprecated=True,
+        deprecation_message="Use `request_body` field instead.",
         description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.",
         examples=[
             {"sort_order": "ASC", "sort_field": "CREATED_AT"},
@@ -2277,6 +2305,27 @@ class HttpRequester(BaseModel):
         ],
         title="Request Body JSON Payload",
     )
+    request_body: Optional[RequestBody] = Field(
+        None,
+        description="Specifies how to populate the body of the request with a payload. Can contain nested objects.",
+        examples=[
+            {
+                "type": "RequestBodyJson",
+                "value": {"sort_order": "ASC", "sort_field": "CREATED_AT"},
+            },
+            {"type": "RequestBodyJson", "value": {"key": "{{ config['value'] }}"}},
+            {
+                "type": "RequestBodyJson",
+                "value": {"sort": {"field": "updated_at", "order": "ascending"}},
+            },
+            {"type": "RequestBodyData", "value": "plain_text_body"},
+            {
+                "type": "RequestBodyData",
+                "value": {"param1": "value1", "param2": "{{ config['param2_value'] }}"},
+            },
+        ],
+        title="Request Body Payload to be send as a part of the API request.",
+    )
     request_headers: Optional[Union[Dict[str, str], str]] = Field(
         None,
         description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.",