airbyte-cdk 6.5.3rc2__py3-none-any.whl → 6.5.5__py3-none-any.whl

airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py

@@ -38,7 +38,11 @@ class InterpolatedMapping:
         valid_value_types = additional_parameters.pop("valid_value_types", None)
         return {
             self._interpolation.eval(
-                name, config, valid_types=valid_key_types, parameters=self._parameters, **additional_parameters
+                name,
+                config,
+                valid_types=valid_key_types,
+                parameters=self._parameters,
+                **additional_parameters,
             ): self._eval(value, config, valid_types=valid_value_types, **additional_parameters)
             for name, value in self.mapping.items()
         }

airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py

@@ -9,7 +9,9 @@ from typing import Any, Mapping, Optional, Union
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
 from airbyte_cdk.sources.types import Config
 
-NestedMappingEntry = Union[dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None]
+NestedMappingEntry = Union[
+    dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None
+]
 NestedMapping = Union[dict[str, NestedMappingEntry], str]
 
 
@@ -32,12 +34,17 @@ class InterpolatedNestedMapping:
     def eval(self, config: Config, **additional_parameters: Any) -> Any:
         return self._eval(self.mapping, config, **additional_parameters)
 
-    def _eval(self, value: Union[NestedMapping, NestedMappingEntry], config: Config, **kwargs: Any) -> Any:
+    def _eval(
+        self, value: Union[NestedMapping, NestedMappingEntry], config: Config, **kwargs: Any
+    ) -> Any:
         # Recursively interpolate dictionaries and lists
         if isinstance(value, str):
             return self._interpolation.eval(value, config, parameters=self._parameters, **kwargs)
         elif isinstance(value, dict):
-            interpolated_dict = {self._eval(k, config, **kwargs): self._eval(v, config, **kwargs) for k, v in value.items()}
+            interpolated_dict = {
+                self._eval(k, config, **kwargs): self._eval(v, config, **kwargs)
+                for k, v in value.items()
+            }
             return {k: v for k, v in interpolated_dict.items() if v is not None}
         elif isinstance(value, list):
             return [self._eval(v, config, **kwargs) for v in value]

airbyte_cdk/sources/declarative/interpolation/interpolated_string.py

@@ -45,10 +45,14 @@ class InterpolatedString:
         if self._is_plain_string is None:
             # Let's check whether output from evaluation is the same as input.
             # This indicates occurrence of a plain string, not a template and we can skip Jinja in subsequent runs.
-            evaluated = self._interpolation.eval(self.string, config, self.default, parameters=self._parameters, **kwargs)
+            evaluated = self._interpolation.eval(
+                self.string, config, self.default, parameters=self._parameters, **kwargs
+            )
             self._is_plain_string = self.string == evaluated
             return evaluated
-        return self._interpolation.eval(self.string, config, self.default, parameters=self._parameters, **kwargs)
+        return self._interpolation.eval(
+            self.string, config, self.default, parameters=self._parameters, **kwargs
+        )
 
     def __eq__(self, other: Any) -> bool:
         if not isinstance(other, InterpolatedString):

airbyte_cdk/sources/declarative/interpolation/interpolation.py

@@ -14,7 +14,13 @@ class Interpolation(ABC):
     """
 
     @abstractmethod
-    def eval(self, input_str: str, config: Config, default: Optional[str] = None, **additional_options: Any) -> Any:
+    def eval(
+        self,
+        input_str: str,
+        config: Config,
+        default: Optional[str] = None,
+        **additional_options: Any,
+    ) -> Any:
         """
         Interpolates the input string using the config, and additional options passed as parameter.
 

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -61,7 +61,9 @@ class JinjaInterpolation(Interpolation):
     # By default, these Python builtin functions are available in the Jinja context.
     # We explicitely remove them because of the potential security risk.
     # Please add a unit test to test_jinja.py when adding a restriction.
-    RESTRICTED_BUILTIN_FUNCTIONS = ["range"]  # The range function can cause very expensive computations
+    RESTRICTED_BUILTIN_FUNCTIONS = [
+        "range"
+    ]  # The range function can cause very expensive computations
 
     def __init__(self) -> None:
         self._environment = StreamPartitionAccessEnvironment()
@@ -119,7 +121,9 @@ class JinjaInterpolation(Interpolation):
             undeclared = self._find_undeclared_variables(s)
             undeclared_not_in_context = {var for var in undeclared if var not in context}
             if undeclared_not_in_context:
-                raise ValueError(f"Jinja macro has undeclared variables: {undeclared_not_in_context}. Context: {context}")
+                raise ValueError(
+                    f"Jinja macro has undeclared variables: {undeclared_not_in_context}. Context: {context}"
+                )
             return self._compile(s).render(context)  # type: ignore # from_string is able to handle None
         except TypeError:
             # The string is a static value, not a jinja template

airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -104,7 +104,9 @@ def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
     :param num_days: number of days to add to current date time
     :return: datetime formatted as RFC3339
     """
-    return (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)).strftime(format)
+    return (
+        datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)
+    ).strftime(format)
 
 
 def duration(datestring: str) -> Union[datetime.timedelta, isodate.Duration]:
@@ -117,7 +119,9 @@ def duration(datestring: str) -> Union[datetime.timedelta, isodate.Duration]:
     return parse_duration(datestring)  # type: ignore # mypy thinks this returns Any for some reason
 
 
-def format_datetime(dt: Union[str, datetime.datetime], format: str, input_format: Optional[str] = None) -> str:
+def format_datetime(
+    dt: Union[str, datetime.datetime], format: str, input_format: Optional[str] = None
+) -> str:
     """
     Converts datetime to another format
 
@@ -130,11 +134,22 @@ def format_datetime(dt: Union[str, datetime.datetime], format: str, input_format
     """
     if isinstance(dt, datetime.datetime):
         return dt.strftime(format)
-    dt_datetime = datetime.datetime.strptime(dt, input_format) if input_format else _str_to_datetime(dt)
+    dt_datetime = (
+        datetime.datetime.strptime(dt, input_format) if input_format else _str_to_datetime(dt)
+    )
     if format == "%s":
         return str(int(dt_datetime.timestamp()))
     return dt_datetime.strftime(format)
 
 
-_macros_list = [now_utc, today_utc, timestamp, max, day_delta, duration, format_datetime, today_with_timezone]
+_macros_list = [
+    now_utc,
+    today_utc,
+    timestamp,
+    max,
+    day_delta,
+    duration,
+    format_datetime,
+    today_with_timezone,
+]
 macros = {f.__name__: f for f in _macros_list}

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -20,16 +20,30 @@ from airbyte_cdk.models import (
 )
 from airbyte_cdk.sources.declarative.checks.connection_checker import ConnectionChecker
 from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import CheckStream as CheckStreamModel
-from airbyte_cdk.sources.declarative.models.declarative_component_schema import DeclarativeStream as DeclarativeStreamModel
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    CheckStream as CheckStreamModel,
+)
+from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
+    DeclarativeStream as DeclarativeStreamModel,
+)
 from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel
-from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer
-from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
-from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory
+from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import (
+    ManifestComponentTransformer,
+)
+from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import (
+    ManifestReferenceResolver,
+)
+from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
+    ModelToComponentFactory,
+)
 from airbyte_cdk.sources.message import MessageRepository
 from airbyte_cdk.sources.streams.core import Stream
 from airbyte_cdk.sources.types import ConnectionDefinition
-from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger, DebugSliceLogger, SliceLogger
+from airbyte_cdk.sources.utils.slice_logger import (
+    AlwaysLogSliceLogger,
+    DebugSliceLogger,
+    SliceLogger,
+)
 from jsonschema.exceptions import ValidationError
 from jsonschema.validators import validate
 
@@ -57,13 +71,21 @@ class ManifestDeclarativeSource(DeclarativeSource):
             manifest["type"] = "DeclarativeSource"
 
         resolved_source_config = ManifestReferenceResolver().preprocess_manifest(manifest)
-        propagated_source_config = ManifestComponentTransformer().propagate_types_and_parameters("", resolved_source_config, {})
+        propagated_source_config = ManifestComponentTransformer().propagate_types_and_parameters(
+            "", resolved_source_config, {}
+        )
         self._source_config = propagated_source_config
         self._debug = debug
         self._emit_connector_builder_messages = emit_connector_builder_messages
-        self._constructor = component_factory if component_factory else ModelToComponentFactory(emit_connector_builder_messages)
+        self._constructor = (
+            component_factory
+            if component_factory
+            else ModelToComponentFactory(emit_connector_builder_messages)
+        )
         self._message_repository = self._constructor.get_message_repository()
-        self._slice_logger: SliceLogger = AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
+        self._slice_logger: SliceLogger = (
+            AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger()
+        )
 
         self._validate_source()
 
@@ -81,20 +103,30 @@ class ManifestDeclarativeSource(DeclarativeSource):
         if "type" not in check:
             check["type"] = "CheckStream"
         check_stream = self._constructor.create_component(
-            CheckStreamModel, check, dict(), emit_connector_builder_messages=self._emit_connector_builder_messages
+            CheckStreamModel,
+            check,
+            dict(),
+            emit_connector_builder_messages=self._emit_connector_builder_messages,
         )
         if isinstance(check_stream, ConnectionChecker):
             return check_stream
         else:
-            raise ValueError(f"Expected to generate a ConnectionChecker component, but received {check_stream.__class__}")
+            raise ValueError(
+                f"Expected to generate a ConnectionChecker component, but received {check_stream.__class__}"
+            )
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        self._emit_manifest_debug_message(extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)})
+        self._emit_manifest_debug_message(
+            extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)}
+        )
         stream_configs = self._stream_configs(self._source_config)
 
         source_streams = [
             self._constructor.create_component(
-                DeclarativeStreamModel, stream_config, config, emit_connector_builder_messages=self._emit_connector_builder_messages
+                DeclarativeStreamModel,
+                stream_config,
+                config,
+                emit_connector_builder_messages=self._emit_connector_builder_messages,
             )
             for stream_config in self._initialize_cache_for_parent_streams(deepcopy(stream_configs))
         ]
@@ -102,7 +134,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
         return source_streams
 
     @staticmethod
-    def _initialize_cache_for_parent_streams(stream_configs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def _initialize_cache_for_parent_streams(
+        stream_configs: List[Dict[str, Any]],
+    ) -> List[Dict[str, Any]]:
         parent_streams = set()
 
         def update_with_cache_parent_configs(parent_configs: list[dict[str, Any]]) -> None:
@@ -113,12 +147,16 @@ class ManifestDeclarativeSource(DeclarativeSource):
         for stream_config in stream_configs:
             if stream_config.get("incremental_sync", {}).get("parent_stream"):
                 parent_streams.add(stream_config["incremental_sync"]["parent_stream"]["name"])
-                stream_config["incremental_sync"]["parent_stream"]["retriever"]["requester"]["use_cache"] = True
+                stream_config["incremental_sync"]["parent_stream"]["retriever"]["requester"][
+                    "use_cache"
+                ] = True
 
             elif stream_config.get("retriever", {}).get("partition_router", {}):
                 partition_router = stream_config["retriever"]["partition_router"]
 
-                if isinstance(partition_router, dict) and partition_router.get("parent_stream_configs"):
+                if isinstance(partition_router, dict) and partition_router.get(
+                    "parent_stream_configs"
+                ):
                     update_with_cache_parent_configs(partition_router["parent_stream_configs"])
                 elif isinstance(partition_router, list):
                     for router in partition_router:
@@ -139,7 +177,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
         in the project root.
         """
         self._configure_logger_level(logger)
-        self._emit_manifest_debug_message(extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)})
+        self._emit_manifest_debug_message(
+            extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)}
+        )
 
         spec = self._source_config.get("spec")
         if spec:
@@ -176,22 +216,62 @@ class ManifestDeclarativeSource(DeclarativeSource):
         Validates the connector manifest against the declarative component schema
         """
         try:
-            raw_component_schema = pkgutil.get_data("airbyte_cdk", "sources/declarative/declarative_component_schema.yaml")
+            raw_component_schema = pkgutil.get_data(
+                "airbyte_cdk", "sources/declarative/declarative_component_schema.yaml"
+            )
             if raw_component_schema is not None:
-                declarative_component_schema = yaml.load(raw_component_schema, Loader=yaml.SafeLoader)
+                declarative_component_schema = yaml.load(
+                    raw_component_schema, Loader=yaml.SafeLoader
+                )
             else:
-                raise RuntimeError("Failed to read manifest component json schema required for validation")
+                raise RuntimeError(
+                    "Failed to read manifest component json schema required for validation"
+                )
         except FileNotFoundError as e:
-            raise FileNotFoundError(f"Failed to read manifest component json schema required for validation: {e}")
+            raise FileNotFoundError(
+                f"Failed to read manifest component json schema required for validation: {e}"
+            )
 
         streams = self._source_config.get("streams")
         if not streams:
-            raise ValidationError(f"A valid manifest should have at least one stream defined. Got {streams}")
+            raise ValidationError(
+                f"A valid manifest should have at least one stream defined. Got {streams}"
+            )
 
         try:
             validate(self._source_config, declarative_component_schema)
         except ValidationError as e:
-            raise ValidationError("Validation against json schema defined in declarative_component_schema.yaml schema failed") from e
+            raise ValidationError(
+                "Validation against json schema defined in declarative_component_schema.yaml schema failed"
+            ) from e
+
+        cdk_version = metadata.version("airbyte_cdk")
+        cdk_major, cdk_minor, cdk_patch = self._get_version_parts(cdk_version, "airbyte-cdk")
+        manifest_version = self._source_config.get("version")
+        if manifest_version is None:
+            raise RuntimeError(
+                "Manifest version is not defined in the manifest. This is unexpected since it should be a required field. Please contact support."
+            )
+        manifest_major, manifest_minor, manifest_patch = self._get_version_parts(
+            manifest_version, "manifest"
+        )
+
+        if cdk_version.startswith("0.0.0"):
+            # Skipping version compatibility check on unreleased dev branch
+            pass
+        elif cdk_major < manifest_major or (
+            cdk_major == manifest_major and cdk_minor < manifest_minor
+        ):
+            raise ValidationError(
+                f"The manifest version {manifest_version} is greater than the airbyte-cdk package version ({cdk_version}). Your "
+                f"manifest may contain features that are not in the current CDK version."
+            )
+        elif manifest_major == 0 and manifest_minor < 29:
+            raise ValidationError(
+                f"The low-code framework was promoted to Beta in airbyte-cdk version 0.29.0 and contains many breaking changes to the "
+                f"language. The manifest version {manifest_version} is incompatible with the airbyte-cdk package version "
+                f"{cdk_version} which contains these breaking changes."
+            )
 
     @staticmethod
     def _get_version_parts(version: str, version_type: str) -> Tuple[int, int, int]:
@@ -200,7 +280,9 @@ class ManifestDeclarativeSource(DeclarativeSource):
         """
         version_parts = re.split(r"\.", version)
         if len(version_parts) != 3 or not all([part.isdigit() for part in version_parts]):
-            raise ValidationError(f"The {version_type} version {version} specified is not a valid version format (ex. 1.2.3)")
+            raise ValidationError(
+                f"The {version_type} version {version} specified is not a valid version format (ex. 1.2.3)"
+            )
         return tuple(int(part) for part in version_parts)  # type: ignore # We already verified there were 3 parts and they are all digits
 
     def _stream_configs(self, manifest: Mapping[str, Any]) -> List[Dict[str, Any]]:

airbyte_cdk/sources/declarative/migrations/legacy_to_per_partition_state_migration.py

@@ -43,7 +43,9 @@ class LegacyToPerPartitionStateMigration(StateMigration):
         self._partition_key_field = InterpolatedString.create(
             self._get_partition_field(self._partition_router), parameters=self._parameters
         ).eval(self._config)
-        self._cursor_field = InterpolatedString.create(self._cursor.cursor_field, parameters=self._parameters).eval(self._config)
+        self._cursor_field = InterpolatedString.create(
+            self._cursor.cursor_field, parameters=self._parameters
+        ).eval(self._config)
 
     def _get_partition_field(self, partition_router: SubstreamPartitionRouter) -> str:
         parent_stream_config = partition_router.parent_stream_configs[0]
@@ -85,5 +87,8 @@ class LegacyToPerPartitionStateMigration(StateMigration):
         return True
 
     def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
-        states = [{"partition": {self._partition_key_field: key}, "cursor": value} for key, value in stream_state.items()]
+        states = [
+            {"partition": {self._partition_key_field: key}, "cursor": value}
+            for key, value in stream_state.items()
+        ]
         return {"states": states}