airbyte-cdk 6.45.2.dev0__tar.gz → 6.45.3__tar.gz

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-cdk
-Version: 6.45.2.dev0
+Version: 6.45.3
 Summary: A framework for writing Airbyte Connectors.
 Home-page: https://airbyte.com
 License: MIT

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -4,7 +4,7 @@
 
 
 from dataclasses import asdict, dataclass, field
-from typing import Any, Dict, List, Mapping
+from typing import Any, ClassVar, Dict, List, Mapping
 
 from airbyte_cdk.connector_builder.test_reader import TestReader
 from airbyte_cdk.models import (
@@ -37,6 +37,8 @@ MAX_STREAMS_KEY = "max_streams"
 
 @dataclass
 class TestLimits:
+    __test__: ClassVar[bool] = False  # Tell Pytest this is not a Pytest class, despite its name
+
     max_records: int = field(default=DEFAULT_MAXIMUM_RECORDS)
     max_pages_per_slice: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE)
     max_slices: int = field(default=DEFAULT_MAXIMUM_NUMBER_OF_SLICES)

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/airbyte_cdk/connector_builder/test_reader/reader.py

@@ -4,7 +4,7 @@
 
 
 import logging
-from typing import Any, Dict, Iterator, List, Mapping, Optional, Union
+from typing import Any, ClassVar, Dict, Iterator, List, Mapping, Optional, Union
 
 from airbyte_cdk.connector_builder.models import (
     AuxiliaryRequest,
@@ -66,6 +66,8 @@ class TestReader:
 
     """
 
+    __test__: ClassVar[bool] = False  # Tell Pytest this is not a Pytest class, despite its name
+
     logger = logging.getLogger("airbyte.connector-builder")
 
     def __init__(

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2307,6 +2307,27 @@ definitions:
       $parameters:
         type: object
         additionalProperties: true
+  KeyTransformation:
+    title: Transformation to apply for extracted object keys by Dpath Flatten Fields
+    type: object
+    required:
+      - type
+    properties:
+      type:
+        type: string
+        enum: [ KeyTransformation ]
+      prefix:
+        title: Key Prefix
+        description: Prefix to add for object keys. If not provided original keys remain unchanged.
+        type: string
+        examples:
+          - flattened_
+      suffix:
+        title: Key Suffix
+        description: Suffix to add for object keys. If not provided original keys remain unchanged.
+        type: string
+        examples:
+          - _flattened
   DpathFlattenFields:
     title: Dpath Flatten Fields
     description: A transformation that flatten field values to the to top of the record.
@@ -2335,6 +2356,11 @@ definitions:
         title: Replace Origin Record
         description: Whether to replace the origin record or not. Default is False.
         type: boolean
+      key_transformation:
+        title: Key transformation
+        description: Transformation for object keys. If not provided, original key will be used.
+        type: object
+        "$ref": "#/definitions/KeyTransformation"
       $parameters:
         type: object
         additionalProperties: true

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -879,6 +879,25 @@ class FlattenFields(BaseModel):
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 
+class KeyTransformation(BaseModel):
+    prefix: Optional[Union[str, None]] = Field(
+        None,
+        description="Prefix to add for object keys. If not provided original keys remain unchanged.",
+        examples=[
+            "flattened_",
+        ],
+        title="Key Prefix",
+    )
+    suffix: Optional[Union[str, None]] = Field(
+        None,
+        description="Suffix to add for object keys. If not provided original keys remain unchanged.",
+        examples=[
+            "_flattened",
+        ],
+        title="Key Suffix",
+    )
+
+
 class DpathFlattenFields(BaseModel):
     type: Literal["DpathFlattenFields"]
     field_path: List[str] = Field(
@@ -897,6 +916,11 @@ class DpathFlattenFields(BaseModel):
         description="Whether to replace the origin record or not. Default is False.",
         title="Replace Origin Record",
     )
+    key_transformation: Optional[Union[KeyTransformation, None]] = Field(
+        None,
+        description="Transformation for object keys. If not provided, original key will be used.",
+        title="Key transformation",
+    )
     parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters")
 
 

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

@@ -498,6 +498,7 @@ from airbyte_cdk.sources.declarative.transformations import (
 from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
 from airbyte_cdk.sources.declarative.transformations.dpath_flatten_fields import (
     DpathFlattenFields,
+    KeyTransformation,
 )
 from airbyte_cdk.sources.declarative.transformations.flatten_fields import (
     FlattenFields,
@@ -790,6 +791,16 @@ class ModelToComponentFactory:
         self, model: DpathFlattenFieldsModel, config: Config, **kwargs: Any
     ) -> DpathFlattenFields:
         model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path]
+        key_transformation = (
+            KeyTransformation(
+                config=config,
+                prefix=model.key_transformation.prefix,
+                suffix=model.key_transformation.suffix,
+                parameters=model.parameters or {},
+            )
+            if model.key_transformation is not None
+            else None
+        )
         return DpathFlattenFields(
             config=config,
             field_path=model_field_path,
@@ -797,6 +808,7 @@ class ModelToComponentFactory:
             if model.delete_origin_value is not None
             else False,
             replace_record=model.replace_record if model.replace_record is not None else False,
+            key_transformation=key_transformation,
             parameters=model.parameters or {},
         )
 
@@ -2054,7 +2066,6 @@ class ModelToComponentFactory:
         config: Config,
         *,
         url_base: str,
-        extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None,
         decoder: Optional[Decoder] = None,
         cursor_used_for_stop_condition: Optional[DeclarativeCursor] = None,
     ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]:
@@ -2076,10 +2087,7 @@ class ModelToComponentFactory:
             else None
         )
         pagination_strategy = self._create_component_from_model(
-            model=model.pagination_strategy,
-            config=config,
-            decoder=decoder_to_use,
-            extractor_model=extractor_model,
+            model=model.pagination_strategy, config=config, decoder=decoder_to_use
         )
         if cursor_used_for_stop_condition:
             pagination_strategy = StopConditionPaginationStrategyDecorator(
@@ -2576,12 +2584,7 @@ class ModelToComponentFactory:
         )
 
     def create_offset_increment(
-        self,
-        model: OffsetIncrementModel,
-        config: Config,
-        decoder: Decoder,
-        extractor_model: Optional[Union[CustomRecordExtractorModel, DpathExtractorModel]] = None,
-        **kwargs: Any,
+        self, model: OffsetIncrementModel, config: Config, decoder: Decoder, **kwargs: Any
     ) -> OffsetIncrement:
         if isinstance(decoder, PaginationDecoderDecorator):
             inner_decoder = decoder.decoder
@@ -2596,24 +2599,10 @@ class ModelToComponentFactory:
                 self._UNSUPPORTED_DECODER_ERROR.format(decoder_type=type(inner_decoder))
             )
 
-        # Ideally we would instantiate the runtime extractor from highest most level (in this case the SimpleRetriever)
-        # so that it can be shared by OffSetIncrement and RecordSelector. However, due to how we instantiate the
-        # decoder with various decorators here, but not in create_record_selector, it is simpler to retain existing
-        # behavior by having two separate extractors with identical behavior since they use the same extractor model.
-        # When we have more time to investigate we can look into reusing the same component.
-        extractor = (
-            self._create_component_from_model(
-                model=extractor_model, config=config, decoder=decoder_to_use
-            )
-            if extractor_model
-            else None
-        )
-
         return OffsetIncrement(
             page_size=model.page_size,
             config=config,
             decoder=decoder_to_use,
-            extractor=extractor,
             inject_on_first_request=model.inject_on_first_request or False,
             parameters=model.parameters or {},
         )
@@ -2977,7 +2966,6 @@ class ModelToComponentFactory:
                 model=model.paginator,
                 config=config,
                 url_base=url_base,
-                extractor_model=model.record_selector.extractor,
                 decoder=decoder,
                 cursor_used_for_stop_condition=cursor_used_for_stop_condition,
             )

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py

@@ -12,7 +12,6 @@ from airbyte_cdk.sources.declarative.decoders import (
     JsonDecoder,
     PaginationDecoderDecorator,
 )
-from airbyte_cdk.sources.declarative.extractors.dpath_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import (
     PaginationStrategy,
@@ -47,7 +46,6 @@ class OffsetIncrement(PaginationStrategy):
     config: Config
     page_size: Optional[Union[str, int]]
     parameters: InitVar[Mapping[str, Any]]
-    extractor: Optional[RecordExtractor]
     decoder: Decoder = field(
         default_factory=lambda: PaginationDecoderDecorator(decoder=JsonDecoder(parameters={}))
     )
@@ -77,14 +75,6 @@ class OffsetIncrement(PaginationStrategy):
     ) -> Optional[Any]:
         decoded_response = next(self.decoder.decode(response))
 
-        if self.extractor:
-            page_size_from_response = len(list(self.extractor.extract_records(response=response)))
-            # The extractor could return 0 records which is valid, but evaluates to False. Our fallback in other
-            # cases as the best effort option is to use the incoming last_page_size
-            last_page_size = (
-                page_size_from_response if page_size_from_response is not None else last_page_size
-            )
-
         # Stop paginating when there are fewer records than the page size or the current page has no records
         if (
             self._page_size

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/airbyte_cdk/sources/declarative/transformations/dpath_flatten_fields.py

@@ -8,6 +8,24 @@ from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 from airbyte_cdk.sources.types import Config, StreamSlice, StreamState
 
 
+@dataclass
+class KeyTransformation:
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    prefix: Optional[str] = None
+    suffix: Optional[str] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        if self.prefix is not None:
+            self.prefix = InterpolatedString.create(self.prefix, parameters=parameters).eval(
+                self.config
+            )
+        if self.suffix is not None:
+            self.suffix = InterpolatedString.create(self.suffix, parameters=parameters).eval(
+                self.config
+            )
+
+
 @dataclass
 class DpathFlattenFields(RecordTransformation):
     """
@@ -16,6 +34,7 @@ class DpathFlattenFields(RecordTransformation):
     field_path: List[Union[InterpolatedString, str]] path to the field to flatten.
     delete_origin_value: bool = False whether to delete origin field or keep it. Default is False.
     replace_record: bool = False whether to replace origin record or not. Default is False.
+    key_transformation: KeyTransformation = None how to transform extracted object keys
 
     """
 
@@ -24,17 +43,35 @@ class DpathFlattenFields(RecordTransformation):
     parameters: InitVar[Mapping[str, Any]]
     delete_origin_value: bool = False
     replace_record: bool = False
+    key_transformation: Optional[KeyTransformation] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._parameters = parameters
         self._field_path = [
-            InterpolatedString.create(path, parameters=parameters) for path in self.field_path
+            InterpolatedString.create(path, parameters=self._parameters) for path in self.field_path
         ]
         for path_index in range(len(self.field_path)):
             if isinstance(self.field_path[path_index], str):
                 self._field_path[path_index] = InterpolatedString.create(
-                    self.field_path[path_index], parameters=parameters
+                    self.field_path[path_index], parameters=self._parameters
                 )
 
+    def _apply_key_transformation(self, extracted: Mapping[str, Any]) -> Mapping[str, Any]:
+        if self.key_transformation:
+            if self.key_transformation.prefix:
+                extracted = {
+                    f"{self.key_transformation.prefix}{key}": value
+                    for key, value in extracted.items()
+                }
+
+            if self.key_transformation.suffix:
+                extracted = {
+                    f"{key}{self.key_transformation.suffix}": value
+                    for key, value in extracted.items()
+                }
+
+        return extracted
+
     def transform(
         self,
         record: Dict[str, Any],
@@ -50,6 +87,8 @@ class DpathFlattenFields(RecordTransformation):
             extracted = dpath.get(record, path, default=[])
 
         if isinstance(extracted, dict):
+            extracted = self._apply_key_transformation(extracted)
+
             if self.replace_record and extracted:
                 dpath.delete(record, "**")
                 record.update(extracted)

{airbyte_cdk-6.45.2.dev0 → airbyte_cdk-6.45.3}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["airbyte", "connector-development-kit", "cdk"]
 
 # Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
-version = "6.45.2dev0"  # Version will be calculated dynamically.
+version = "6.45.3"  # Version will be calculated dynamically.
 
 [tool.poetry-dynamic-versioning]
 enable = false