airbyte-cdk 0.72.0__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -5,40 +5,56 @@
 import json
 import logging
 import pkgutil
-import re
 from copy import deepcopy
 from importlib import metadata
-from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union
+from typing import Any, Dict, Iterator, List, Mapping, Optional, Set
 
 import yaml
+from jsonschema.exceptions import ValidationError
+from jsonschema.validators import validate
+from packaging.version import InvalidVersion, Version
+
 from airbyte_cdk.models import (
     AirbyteConnectionStatus,
     AirbyteMessage,
     AirbyteStateMessage,
     ConfiguredAirbyteCatalog,
     ConnectorSpecification,
+    FailureType,
 )
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import CheckStream as CheckStreamModel
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import DeclarativeStream as DeclarativeStreamModel
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    CheckStream as CheckStreamModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    DeclarativeStream as DeclarativeStreamModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel
-from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer
-from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
-from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
-from airbyte_cdk.sources.declarative.types import ConnectionDefinition
+from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import (
+    ManifestComponentTransformer,
+)
+from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import (
+    ManifestReferenceResolver,
+)
+from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
+    ModelToComponentFactory,
+)
+from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.core import Stream
-from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger, DebugSliceLogger, SliceLogger
-from jsonschema.exceptions import ValidationError
-from jsonschema.validators import validate
+from airbyte_cdk.sources.types import ConnectionDefinition
+from airbyte_cdk.sources.utils.slice_logger import (
+    AlwaysLogSliceLogger,
+    DebugSliceLogger,
+    SliceLogger,
+)
+from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 
 class ManifestDeclarativeSource(DeclarativeSource):
     """Declarative source defined by a manifest of low-code components that define source connector behavior"""
 
-    VALID_TOP_LEVEL_FIELDS = {"check", "definitions", "schemas", "spec", "streams", "type", "version"}
-
     def __init__(
         self,
         source_config: ConnectionDefinition,
@@ -59,13 +75,21 @@ class ManifestDeclarativeSource(DeclarativeSource):
             manifest["type"] = "DeclarativeSource"
 
         resolved_source_config = ManifestReferenceResolver().preprocess_manifest(manifest)
-        propagated_source_config = ManifestComponentTransformer().propagate_types_and_parameters("", resolved_source_config, {})
+        propagated_source_config = ManifestComponentTransformer().propagate_types_and_parameters(
+            "", resolved_source_config, {}
+        )
         self._source_config = propagated_source_config
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages
-        self._constructor = component_factory if component_factory else ModelToComponentFactory(emit_connector_builder_messages)
+        self._constructor = (
+            component_factory
+            if component_factory
+            else ModelToComponentFactory(emit_connector_builder_messages)
+        )
         self._message_repository = self._constructor.get_message_repository()
-        self._slice_logger: SliceLogger = AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
+        self._slice_logger: SliceLogger = (
+            AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
+        )
 
         self._validate_source()
 
@@ -74,7 +98,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         return self._source_config
 
     @property
-    def message_repository(self) -> Union[None, MessageRepository]:
+    def message_repository(self) -> MessageRepository:
         return self._message_repository
 
     @property
@@ -83,20 +107,33 @@ class ManifestDeclarativeSource(DeclarativeSource):
         if "type" not in check:
             check["type"] = "CheckStream"
         check_stream = self._constructor.create_component(
-            CheckStreamModel, check, dict(), emit_connector_builder_messages=self._emit_connector_builder_messages
+            CheckStreamModel,
+            check,
+            dict(),
+            emit_connector_builder_messages=self._emit_connector_builder_messages,
         )
         if isinstance(check_stream, ConnectionChecker):
             return check_stream
         else:
-            raise ValueError(f"Expected to generate a ConnectionChecker component, but received {check_stream.__class__}")
+            raise ValueError(
+                f"Expected to generate a ConnectionChecker component, but received {check_stream.__class__}"
+            )
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        self._emit_manifest_debug_message(extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)})
-        stream_configs = self._stream_configs(self._source_config)
+        self._emit_manifest_debug_message(
+            extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)}
+        )
+
+        stream_configs = self._stream_configs(self._source_config) + self._dynamic_stream_configs(
+            self._source_config, config
+        )
 
         source_streams = [
             self._constructor.create_component(
-                DeclarativeStreamModel, stream_config, config, emit_connector_builder_messages=self._emit_connector_builder_messages
+                DeclarativeStreamModel,
+                stream_config,
+                config,
+                emit_connector_builder_messages=self._emit_connector_builder_messages,
             )
             for stream_config in self._initialize_cache_for_parent_streams(deepcopy(stream_configs))
         ]
@@ -104,7 +141,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
         return source_streams
 
     @staticmethod
-    def _initialize_cache_for_parent_streams(stream_configs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def _initialize_cache_for_parent_streams(
+        stream_configs: List[Dict[str, Any]],
+    ) -> List[Dict[str, Any]]:
         parent_streams = set()
 
         def update_with_cache_parent_configs(parent_configs: list[dict[str, Any]]) -> None:
@@ -115,12 +154,16 @@ class ManifestDeclarativeSource(DeclarativeSource):
         for stream_config in stream_configs:
             if stream_config.get("incremental_sync", {}).get("parent_stream"):
                 parent_streams.add(stream_config["incremental_sync"]["parent_stream"]["name"])
-                stream_config["incremental_sync"]["parent_stream"]["retriever"]["requester"]["use_cache"] = True
+                stream_config["incremental_sync"]["parent_stream"]["retriever"]["requester"][
+                    "use_cache"
+                ] = True
 
             elif stream_config.get("retriever", {}).get("partition_router", {}):
                 partition_router = stream_config["retriever"]["partition_router"]
 
-                if isinstance(partition_router, dict) and partition_router.get("parent_stream_configs"):
+                if isinstance(partition_router, dict) and partition_router.get(
+                    "parent_stream_configs"
+                ):
                     update_with_cache_parent_configs(partition_router["parent_stream_configs"])
                 elif isinstance(partition_router, list):
                     for router in partition_router:
@@ -141,7 +184,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
         in the project root.
         """
         self._configure_logger_level(logger)
-        self._emit_manifest_debug_message(extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)})
+        self._emit_manifest_debug_message(
+            extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)}
+        )
 
         spec = self._source_config.get("spec")
         if spec:
@@ -161,7 +206,7 @@ class ManifestDeclarativeSource(DeclarativeSource):
         logger: logging.Logger,
         config: Mapping[str, Any],
         catalog: ConfiguredAirbyteCatalog,
-        state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None,
+        state: Optional[List[AirbyteStateMessage]] = None,
     ) -> Iterator[AirbyteMessage]:
         self._configure_logger_level(logger)
         yield from super().read(logger, config, catalog, state)
@@ -178,53 +223,84 @@ class ManifestDeclarativeSource(DeclarativeSource):
         Validates the connector manifest against the declarative component schema
         """
         try:
-            raw_component_schema = pkgutil.get_data("airbyte_cdk", "sources/declarative/declarative_component_schema.yaml")
+            raw_component_schema = pkgutil.get_data(
+                "airbyte_cdk", "sources/declarative/declarative_component_schema.yaml"
+            )
             if raw_component_schema is not None:
-                declarative_component_schema = yaml.load(raw_component_schema, Loader=yaml.SafeLoader)
+                declarative_component_schema = yaml.load(
+                    raw_component_schema, Loader=yaml.SafeLoader
+                )
             else:
-                raise RuntimeError("Failed to read manifest component json schema required for validation")
+                raise RuntimeError(
+                    "Failed to read manifest component json schema required for validation"
+                )
         except FileNotFoundError as e:
-            raise FileNotFoundError(f"Failed to read manifest component json schema required for validation: {e}")
+            raise FileNotFoundError(
+                f"Failed to read manifest component json schema required for validation: {e}"
+            )
 
         streams = self._source_config.get("streams")
-        if not streams:
-            raise ValidationError(f"A valid manifest should have at least one stream defined. Got {streams}")
+        dynamic_streams = self._source_config.get("dynamic_streams")
+        if not (streams or dynamic_streams):
+            raise ValidationError(
+                f"A valid manifest should have at least one stream defined. Got {streams}"
+            )
 
         try:
             validate(self._source_config, declarative_component_schema)
         except ValidationError as e:
-            raise ValidationError("Validation against json schema defined in declarative_component_schema.yaml schema failed") from e
+            raise ValidationError(
+                "Validation against json schema defined in declarative_component_schema.yaml schema failed"
+            ) from e
 
-        cdk_version = metadata.version("airbyte_cdk")
-        cdk_major, cdk_minor, cdk_patch = self._get_version_parts(cdk_version, "airbyte-cdk")
-        manifest_version = self._source_config.get("version")
-        if manifest_version is None:
+        cdk_version_str = metadata.version("airbyte_cdk")
+        cdk_version = self._parse_version(cdk_version_str, "airbyte-cdk")
+        manifest_version_str = self._source_config.get("version")
+        if manifest_version_str is None:
             raise RuntimeError(
                 "Manifest version is not defined in the manifest. This is unexpected since it should be a required field. Please contact support."
             )
-        manifest_major, manifest_minor, manifest_patch = self._get_version_parts(manifest_version, "manifest")
-
-        if cdk_major < manifest_major or (cdk_major == manifest_major and cdk_minor < manifest_minor):
+        manifest_version = self._parse_version(manifest_version_str, "manifest")
+
+        if (cdk_version.major, cdk_version.minor, cdk_version.micro) == (0, 0, 0):
+            # Skipping version compatibility check on unreleased dev branch
+            pass
+        elif (cdk_version.major, cdk_version.minor) < (
+            manifest_version.major,
+            manifest_version.minor,
+        ):
             raise ValidationError(
-                f"The manifest version {manifest_version} is greater than the airbyte-cdk package version ({cdk_version}). Your "
+                f"The manifest version {manifest_version!s} is greater than the airbyte-cdk package version ({cdk_version!s}). Your "
                 f"manifest may contain features that are not in the current CDK version."
             )
-        elif manifest_major == 0 and manifest_minor < 29:
+        elif (manifest_version.major, manifest_version.minor) < (0, 29):
             raise ValidationError(
                 f"The low-code framework was promoted to Beta in airbyte-cdk version 0.29.0 and contains many breaking changes to the "
-                f"language. The manifest version {manifest_version} is incompatible with the airbyte-cdk package version "
-                f"{cdk_version} which contains these breaking changes."
+                f"language. The manifest version {manifest_version!s} is incompatible with the airbyte-cdk package version "
+                f"{cdk_version!s} which contains these breaking changes."
             )
 
     @staticmethod
-    def _get_version_parts(version: str, version_type: str) -> Tuple[int, int, int]:
-        """
-        Takes a semantic version represented as a string and splits it into a tuple of its major, minor, and patch versions.
+    def _parse_version(
+        version: str,
+        version_type: str,
+    ) -> Version:
+        """Takes a semantic version represented as a string and splits it into a tuple.
+
+        The fourth part (prerelease) is not returned in the tuple.
+
+        Returns:
+            Version: the parsed version object
         """
-        version_parts = re.split(r"\.", version)
-        if len(version_parts) != 3 or not all([part.isdigit() for part in version_parts]):
-            raise ValidationError(f"The {version_type} version {version} specified is not a valid version format (ex. 1.2.3)")
-        return tuple(int(part) for part in version_parts)  # type: ignore # We already verified there were 3 parts and they are all digits
+        try:
+            parsed_version = Version(version)
+        except InvalidVersion as ex:
+            raise ValidationError(
+                f"The {version_type} version '{version}' is not a valid version format."
+            ) from ex
+        else:
+            # No exception
+            return parsed_version
 
     def _stream_configs(self, manifest: Mapping[str, Any]) -> List[Dict[str, Any]]:
         # This has a warning flag for static, but after we finish part 4 we'll replace manifest with self._source_config
@@ -234,5 +310,70 @@ class ManifestDeclarativeSource(DeclarativeSource):
                 s["type"] = "DeclarativeStream"
         return stream_configs
 
+    def _dynamic_stream_configs(
+        self, manifest: Mapping[str, Any], config: Mapping[str, Any]
+    ) -> List[Dict[str, Any]]:
+        dynamic_stream_definitions: List[Dict[str, Any]] = manifest.get("dynamic_streams", [])
+        dynamic_stream_configs: List[Dict[str, Any]] = []
+        seen_dynamic_streams: Set[str] = set()
+
+        for dynamic_definition in dynamic_stream_definitions:
+            components_resolver_config = dynamic_definition["components_resolver"]
+
+            if not components_resolver_config:
+                raise ValueError(
+                    f"Missing 'components_resolver' in dynamic definition: {dynamic_definition}"
+                )
+
+            resolver_type = components_resolver_config.get("type")
+            if not resolver_type:
+                raise ValueError(
+                    f"Missing 'type' in components resolver configuration: {components_resolver_config}"
+                )
+
+            if resolver_type not in COMPONENTS_RESOLVER_TYPE_MAPPING:
+                raise ValueError(
+                    f"Invalid components resolver type '{resolver_type}'. "
+                    f"Expected one of {list(COMPONENTS_RESOLVER_TYPE_MAPPING.keys())}."
+                )
+
+            if "retriever" in components_resolver_config:
+                components_resolver_config["retriever"]["requester"]["use_cache"] = True
+
+            # Create a resolver for dynamic components based on type
+            components_resolver = self._constructor.create_component(
+                COMPONENTS_RESOLVER_TYPE_MAPPING[resolver_type], components_resolver_config, config
+            )
+
+            stream_template_config = dynamic_definition["stream_template"]
+
+            for dynamic_stream in components_resolver.resolve_components(
+                stream_template_config=stream_template_config
+            ):
+                if "type" not in dynamic_stream:
+                    dynamic_stream["type"] = "DeclarativeStream"
+
+                # Ensure that each stream is created with a unique name
+                name = dynamic_stream.get("name")
+
+                if name in seen_dynamic_streams:
+                    error_message = f"Dynamic streams list contains a duplicate name: {name}. Please contact Airbyte Support."
+                    failure_type = FailureType.system_error
+
+                    if resolver_type == "ConfigComponentsResolver":
+                        error_message = f"Dynamic streams list contains a duplicate name: {name}. Please check your configuration."
+                        failure_type = FailureType.config_error
+
+                    raise AirbyteTracedException(
+                        message=error_message,
+                        internal_message=error_message,
+                        failure_type=failure_type,
+                    )
+
+                seen_dynamic_streams.add(name)
+                dynamic_stream_configs.append(dynamic_stream)
+
+        return dynamic_stream_configs
+
     def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:
         self.logger.debug("declarative source created from manifest", extra=extra_args)

airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from typing import Any, Mapping
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
+from airbyte_cdk.sources.declarative.models import (
+    CustomIncrementalSync,
+    DatetimeBasedCursor,
+    SubstreamPartitionRouter,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import ParentStreamConfig
+
+
+def _is_already_migrated(stream_state: Mapping[str, Any]) -> bool:
+    return "states" in stream_state
+
+
+class LegacyToPerPartitionStateMigration(StateMigration):
+    """
+    Transforms the input state for per-partitioned streams from the legacy format to the low-code format.
+    The cursor field and partition ID fields are automatically extracted from the stream's DatetimebasedCursor and SubstreamPartitionRouter.
+
+    Example input state:
+    {
+    "13506132": {
+      "last_changed": "2022-12-27T08:34:39+00:00"
+    }
+    Example output state:
+    {
+      "partition": {"id": "13506132"},
+      "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}
+    }
+    """
+
+    def __init__(
+        self,
+        partition_router: SubstreamPartitionRouter,
+        cursor: CustomIncrementalSync | DatetimeBasedCursor,
+        config: Mapping[str, Any],
+        parameters: Mapping[str, Any],
+    ):
+        self._partition_router = partition_router
+        self._cursor = cursor
+        self._config = config
+        self._parameters = parameters
+        self._partition_key_field = InterpolatedString.create(
+            self._get_partition_field(self._partition_router), parameters=self._parameters
+        ).eval(self._config)
+        self._cursor_field = InterpolatedString.create(
+            self._cursor.cursor_field, parameters=self._parameters
+        ).eval(self._config)
+
+    def _get_partition_field(self, partition_router: SubstreamPartitionRouter) -> str:
+        parent_stream_config = partition_router.parent_stream_configs[0]
+
+        # Retrieve the partition field with a condition, as properties are returned as a dictionary for custom components.
+        partition_field = (
+            parent_stream_config.partition_field
+            if isinstance(parent_stream_config, ParentStreamConfig)
+            else parent_stream_config.get("partition_field")  # type: ignore # See above comment on why parent_stream_config might be a dict
+        )
+
+        return partition_field
+
+    def should_migrate(self, stream_state: Mapping[str, Any]) -> bool:
+        if _is_already_migrated(stream_state):
+            return False
+
+        # There is exactly one parent stream
+        number_of_parent_streams = len(self._partition_router.parent_stream_configs)  # type: ignore # custom partition will introduce this attribute if needed
+        if number_of_parent_streams != 1:
+            # There should be exactly one parent stream
+            return False
+        """
+        The expected state format is
+        "<parent_key_id>" : {
+          "<cursor_field>" : "<cursor_value>"
+        }
+        """
+        if stream_state:
+            for key, value in stream_state.items():
+                if isinstance(value, dict):
+                    keys = list(value.keys())
+                    if len(keys) != 1:
+                        # The input partitioned state should only have one key
+                        return False
+                    if keys[0] != self._cursor_field:
+                        # Unexpected key. Found {keys[0]}. Expected {self._cursor.cursor_field}
+                        return False
+        return True
+
+    def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
+        states = [
+            {"partition": {self._partition_key_field: key}, "cursor": value}
+            for key, value in stream_state.items()
+        ]
+        return {"states": states}

airbyte_cdk/sources/declarative/migrations/state_migration.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from abc import abstractmethod
+from typing import Any, Mapping
+
+
+class StateMigration:
+    @abstractmethod
+    def should_migrate(self, stream_state: Mapping[str, Any]) -> bool:
+        """
+        Check if the stream_state should be migrated
+
+        :param stream_state: The stream_state to potentially migrate
+        :return: true if the state is of the expected format and should be migrated. False otherwise.
+        """
+
+    @abstractmethod
+    def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
+        """
+        Migrate the stream_state. Assumes should_migrate(stream_state) returned True.
+
+        :param stream_state: The stream_state to migrate
+        :return: The migrated stream_state
+        """

airbyte_cdk/sources/declarative/models/__init__.py

@@ -1,2 +1,2 @@
-# generated by generate-component-manifest-files
+# generated by bin/generate_component_manifest_files.py
 from .declarative_component_schema import *