airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4107__py3-none-any.whl

airbyte_cdk/sources/declarative/extractors/dpath_extractor.py

@@ -2,16 +2,16 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-from dataclasses import InitVar, dataclass
-from typing import Any, List, Mapping, Union
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Iterable, List, Mapping, MutableMapping, Union
 
-import dpath.util
+import dpath
 import requests
-from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
-from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder
+
+from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -56,26 +56,31 @@ class DpathExtractor(RecordExtractor):
     field_path: List[Union[InterpolatedString, str]]
     config: Config
     parameters: InitVar[Mapping[str, Any]]
-    decoder: Decoder = JsonDecoder(parameters={})
+    decoder: Decoder = field(default_factory=lambda: JsonDecoder(parameters={}))
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._field_path = [
+            InterpolatedString.create(path, parameters=parameters) for path in self.field_path
+        ]
         for path_index in range(len(self.field_path)):
             if isinstance(self.field_path[path_index], str):
-                self.field_path[path_index] = InterpolatedString.create(self.field_path[path_index], parameters=parameters)
+                self._field_path[path_index] = InterpolatedString.create(
+                    self.field_path[path_index], parameters=parameters
+                )
 
-    def extract_records(self, response: requests.Response) -> List[Mapping[str, Any]]:
-        response_body = self.decoder.decode(response)
-        if len(self.field_path) == 0:
-            extracted = response_body
-        else:
-            path = [path.eval(self.config) for path in self.field_path]
-            if "*" in path:
-                extracted = dpath.util.values(response_body, path)
+    def extract_records(self, response: requests.Response) -> Iterable[MutableMapping[Any, Any]]:
+        for body in self.decoder.decode(response):
+            if len(self._field_path) == 0:
+                extracted = body
+            else:
+                path = [path.eval(self.config) for path in self._field_path]
+                if "*" in path:
+                    extracted = dpath.values(body, path)
+                else:
+                    extracted = dpath.get(body, path, default=[])  # type: ignore # extracted will be a MutableMapping, given input data structure
+            if isinstance(extracted, list):
+                yield from extracted
+            elif extracted:
+                yield extracted
             else:
-                extracted = dpath.util.get(response_body, path, default=[])
-        if isinstance(extracted, list):
-            return extracted
-        elif extracted:
-            return [extracted]
-        else:
-            return []
+                yield from []

airbyte_cdk/sources/declarative/extractors/http_selector.py

@@ -3,14 +3,13 @@
 #
 
 from abc import abstractmethod
-from dataclasses import dataclass
-from typing import Any, List, Mapping, Optional
+from typing import Any, Iterable, Mapping, Optional
 
 import requests
-from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState
+
+from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
 
 
-@dataclass
 class HttpSelector:
     """
     Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering
@@ -25,7 +24,7 @@ class HttpSelector:
         records_schema: Mapping[str, Any],
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> List[Record]:
+    ) -> Iterable[Record]:
         """
         Selects records from the response
         :param response: The response to select the records from

airbyte_cdk/sources/declarative/extractors/record_extractor.py

@@ -1,10 +1,9 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
 from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, List, Mapping
+from typing import Any, Iterable, Mapping
 
 import requests
 
@@ -19,7 +18,7 @@ class RecordExtractor:
     def extract_records(
         self,
         response: requests.Response,
-    ) -> List[Mapping[str, Any]]:
+    ) -> Iterable[Mapping[str, Any]]:
         """
         Selects records from the response
         :param response: The response to extract the records from

airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -1,12 +1,16 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
 from dataclasses import InitVar, dataclass
-from typing import Any, List, Mapping, Optional
+from typing import Any, Iterable, Mapping, Optional, Union
 
+from airbyte_cdk.sources.declarative.incremental import (
+    DatetimeBasedCursor,
+    GlobalSubstreamCursor,
+    PerPartitionWithGlobalCursor,
+)
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
-from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
 
 
 @dataclass
@@ -23,14 +27,67 @@ class RecordFilter:
     condition: str = ""
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
-        self._filter_interpolator = InterpolatedBoolean(condition=self.condition, parameters=parameters)
+        self._filter_interpolator = InterpolatedBoolean(
+            condition=self.condition, parameters=parameters
+        )
+
+    def filter_records(
+        self,
+        records: Iterable[Mapping[str, Any]],
+        stream_state: StreamState,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Mapping[str, Any]]:
+        kwargs = {
+            "stream_state": stream_state,
+            "stream_slice": stream_slice,
+            "next_page_token": next_page_token,
+            "stream_slice.extra_fields": stream_slice.extra_fields if stream_slice else {},
+        }
+        for record in records:
+            if self._filter_interpolator.eval(self.config, record=record, **kwargs):
+                yield record
+
+
+class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
+    """
+    Applies a filter to a list of records to exclude those that are older than the stream_state/start_date.
+
+    :param DatetimeBasedCursor date_time_based_cursor: Cursor used to extract datetime values
+    :param PerPartitionCursor per_partition_cursor: Optional Cursor used for mapping cursor value in nested stream_state
+    """
+
+    def __init__(
+        self,
+        date_time_based_cursor: DatetimeBasedCursor,
+        substream_cursor: Optional[Union[PerPartitionWithGlobalCursor, GlobalSubstreamCursor]],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self._date_time_based_cursor = date_time_based_cursor
+        self._substream_cursor = substream_cursor
 
     def filter_records(
         self,
-        records: List[Mapping[str, Any]],
+        records: Iterable[Mapping[str, Any]],
         stream_state: StreamState,
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> List[Mapping[str, Any]]:
-        kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token}
-        return [record for record in records if self._filter_interpolator.eval(self.config, record=record, **kwargs)]
+    ) -> Iterable[Mapping[str, Any]]:
+        records = (
+            record
+            for record in records
+            if (self._substream_cursor or self._date_time_based_cursor).should_be_synced(
+                # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
+                # Record stream name is empty cause it is not used durig the filtering
+                Record(data=record, associated_slice=stream_slice, stream_name="")
+            )
+        )
+        if self.condition:
+            records = super().filter_records(
+                records=records,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            )
+        yield from records

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -3,21 +3,21 @@
 #
 
 from dataclasses import InitVar, dataclass, field
-from typing import Any, List, Mapping, Optional
+from typing import Any, Iterable, List, Mapping, Optional, Union
 
 import requests
+
 from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
+from airbyte_cdk.sources.declarative.extractors.type_transformer import (
+    TypeTransformer as DeclarativeTypeTransformer,
+)
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
 from airbyte_cdk.sources.declarative.models import SchemaNormalization
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
-from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState
-from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
-
-SCHEMA_TRANSFORMER_TYPE_MAPPING = {
-    SchemaNormalization.None_: TransformConfig.NoTransform,
-    SchemaNormalization.Default: TransformConfig.DefaultSchemaNormalization,
-}
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
+from airbyte_cdk.sources.utils.transform import TypeTransformer
 
 
 @dataclass
@@ -36,12 +36,35 @@ class RecordSelector(HttpSelector):
     extractor: RecordExtractor
     config: Config
     parameters: InitVar[Mapping[str, Any]]
-    schema_normalization: TypeTransformer
+    schema_normalization: Union[TypeTransformer, DeclarativeTypeTransformer]
+    name: str
+    _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
     record_filter: Optional[RecordFilter] = None
     transformations: List[RecordTransformation] = field(default_factory=lambda: [])
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._parameters = parameters
+        self._name = (
+            InterpolatedString(self._name, parameters=parameters)
+            if isinstance(self._name, str)
+            else self._name
+        )
+
+    @property  # type: ignore
+    def name(self) -> str:
+        """
+        :return: Stream name
+        """
+        return (
+            str(self._name.eval(self.config))
+            if isinstance(self._name, InterpolatedString)
+            else self._name
+        )
+
+    @name.setter
+    def name(self, value: str) -> None:
+        if not isinstance(value, property):
+            self._name = value
 
     def select_records(
         self,
@@ -50,7 +73,7 @@ class RecordSelector(HttpSelector):
         records_schema: Mapping[str, Any],
         stream_slice: Optional[StreamSlice] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> List[Record]:
+    ) -> Iterable[Record]:
         """
         Selects records from the response
         :param response: The response to select the records from
@@ -60,38 +83,74 @@ class RecordSelector(HttpSelector):
         :param next_page_token: The paginator token
         :return: List of Records selected from the response
         """
-        all_data = self.extractor.extract_records(response)
+        all_data: Iterable[Mapping[str, Any]] = self.extractor.extract_records(response)
+        yield from self.filter_and_transform(
+            all_data, stream_state, records_schema, stream_slice, next_page_token
+        )
+
+    def filter_and_transform(
+        self,
+        all_data: Iterable[Mapping[str, Any]],
+        stream_state: StreamState,
+        records_schema: Mapping[str, Any],
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Record]:
+        """
+        There is an issue with the selector as of 2024-08-30: it does technology-agnostic processing like filtering, transformation and
+        normalization with an API that is technology-specific (as requests.Response is only for HTTP communication using the requests
+        library).
+
+        Until we decide to move this logic away from the selector, we made this method public so that users like AsyncJobRetriever could
+        share the logic of doing transformations on a set of records.
+        """
         filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token)
-        self._transform(filtered_data, stream_state, stream_slice)
-        self._normalize_by_schema(filtered_data, schema=records_schema)
-        return [Record(data, stream_slice) for data in filtered_data]
+        transformed_data = self._transform(filtered_data, stream_state, stream_slice)
+        normalized_data = self._normalize_by_schema(transformed_data, schema=records_schema)
+        for data in normalized_data:
+            yield Record(data=data, stream_name=self.name, associated_slice=stream_slice)
 
-    def _normalize_by_schema(self, records: List[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]) -> List[Mapping[str, Any]]:
+    def _normalize_by_schema(
+        self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
+    ) -> Iterable[Mapping[str, Any]]:
         if schema:
             # record has type Mapping[str, Any], but dict[str, Any] expected
-            return [self.schema_normalization.transform(record, schema) for record in records]  # type: ignore
-        return records
+            for record in records:
+                normalized_record = dict(record)
+                self.schema_normalization.transform(normalized_record, schema)
+                yield normalized_record
+        else:
+            yield from records
 
     def _filter(
         self,
-        records: List[Mapping[str, Any]],
+        records: Iterable[Mapping[str, Any]],
         stream_state: StreamState,
         stream_slice: Optional[StreamSlice],
         next_page_token: Optional[Mapping[str, Any]],
-    ) -> List[Mapping[str, Any]]:
+    ) -> Iterable[Mapping[str, Any]]:
         if self.record_filter:
-            return self.record_filter.filter_records(
-                records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token
+            yield from self.record_filter.filter_records(
+                records,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
             )
-        return records
+        else:
+            yield from records
 
     def _transform(
         self,
-        records: List[Mapping[str, Any]],
+        records: Iterable[Mapping[str, Any]],
         stream_state: StreamState,
         stream_slice: Optional[StreamSlice] = None,
-    ) -> None:
+    ) -> Iterable[Mapping[str, Any]]:
         for record in records:
             for transformation in self.transformations:
-                # record has type Mapping[str, Any], but Record expected
-                transformation.transform(record, config=self.config, stream_state=stream_state, stream_slice=stream_slice)  # type: ignore
+                transformation.transform(
+                    record,  # type: ignore  # record has type Mapping[str, Any], but Dict[str, Any] expected
+                    config=self.config,
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                )
+            yield record

airbyte_cdk/sources/declarative/extractors/response_to_file_extractor.py

@@ -0,0 +1,177 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+import logging
+import os
+import uuid
+import zlib
+from contextlib import closing
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, Iterable, Mapping, Optional, Tuple
+
+import pandas as pd
+import requests
+from numpy import nan
+
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+
+EMPTY_STR: str = ""
+DEFAULT_ENCODING: str = "utf-8"
+DOWNLOAD_CHUNK_SIZE: int = 1024 * 10
+
+
+@dataclass
+class ResponseToFileExtractor(RecordExtractor):
+    """
+    This class is used when having very big HTTP responses (usually streamed) which would require too much memory so we use disk space as
+    a tradeoff.
+
+    Eventually, we want to support multiple file type by re-using the file based CDK parsers if possible. However, the lift is too high for
+    a first iteration so we will only support CSV parsing using pandas as salesforce and sendgrid were doing.
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self.logger = logging.getLogger("airbyte")
+
+    def _get_response_encoding(self, headers: Dict[str, Any]) -> str:
+        """
+        Get the encoding of the response based on the provided headers. This method is heavily inspired by the requests library
+        implementation.
+
+        Args:
+            headers (Dict[str, Any]): The headers of the response.
+        Returns:
+            str: The encoding of the response.
+        """
+
+        content_type = headers.get("content-type")
+
+        if not content_type:
+            return DEFAULT_ENCODING
+
+        content_type, params = requests.utils.parse_header_links(content_type)
+
+        if "charset" in params:
+            return params["charset"].strip("'\"")  # type: ignore  # we assume headers are returned as str
+
+        return DEFAULT_ENCODING
+
+    def _filter_null_bytes(self, b: bytes) -> bytes:
+        """
+        Filter out null bytes from a bytes object.
+
+        Args:
+            b (bytes): The input bytes object.
+        Returns:
+            bytes: The filtered bytes object with null bytes removed.
+
+        Referenced Issue:
+            https://github.com/airbytehq/airbyte/issues/8300
+        """
+
+        res = b.replace(b"\x00", b"")
+        if len(res) < len(b):
+            self.logger.warning(
+                "Filter 'null' bytes from string, size reduced %d -> %d chars", len(b), len(res)
+            )
+        return res
+
+    def _save_to_file(self, response: requests.Response) -> Tuple[str, str]:
+        """
+        Saves the binary data from the given response to a temporary file and returns the filepath and response encoding.
+
+        Args:
+            response (Optional[requests.Response]): The response object containing the binary data. Defaults to None.
+
+        Returns:
+            Tuple[str, str]: A tuple containing the filepath of the temporary file and the response encoding.
+
+        Raises:
+            ValueError: If the temporary file does not exist after saving the binary data.
+        """
+        # set filepath for binary data from response
+        decompressor = zlib.decompressobj(zlib.MAX_WBITS | 32)
+        needs_decompression = True  # we will assume at first that the response is compressed and change the flag if not
+
+        tmp_file = str(uuid.uuid4())
+        with closing(response) as response, open(tmp_file, "wb") as data_file:
+            response_encoding = self._get_response_encoding(dict(response.headers or {}))
+            for chunk in response.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):
+                try:
+                    if needs_decompression:
+                        data_file.write(decompressor.decompress(chunk))
+                        needs_decompression = True
+                    else:
+                        data_file.write(self._filter_null_bytes(chunk))
+                except zlib.error:
+                    data_file.write(self._filter_null_bytes(chunk))
+                    needs_decompression = False
+
+        # check the file exists
+        if os.path.isfile(tmp_file):
+            return tmp_file, response_encoding
+        else:
+            raise ValueError(
+                f"The IO/Error occured while verifying binary data. Tmp file {tmp_file} doesn't exist."
+            )
+
+    def _read_with_chunks(
+        self, path: str, file_encoding: str, chunk_size: int = 100
+    ) -> Iterable[Mapping[str, Any]]:
+        """
+        Reads data from a file in chunks and yields each row as a dictionary.
+
+        Args:
+            path (str): The path to the file to be read.
+            file_encoding (str): The encoding of the file.
+            chunk_size (int, optional): The size of each chunk to be read. Defaults to 100.
+
+        Yields:
+            Mapping[str, Any]: A dictionary representing each row of data.
+
+        Raises:
+            ValueError: If an IO/Error occurs while reading the temporary data.
+        """
+
+        try:
+            with open(path, "r", encoding=file_encoding) as data:
+                chunks = pd.read_csv(
+                    data, chunksize=chunk_size, iterator=True, dialect="unix", dtype=object
+                )
+                for chunk in chunks:
+                    chunk = chunk.replace({nan: None}).to_dict(orient="records")
+                    for row in chunk:
+                        yield row
+        except pd.errors.EmptyDataError as e:
+            self.logger.info(f"Empty data received. {e}")
+            yield from []
+        except IOError as ioe:
+            raise ValueError(f"The IO/Error occured while reading tmp data. Called: {path}", ioe)
+        finally:
+            # remove binary tmp file, after data is read
+            os.remove(path)
+
+    def extract_records(
+        self, response: Optional[requests.Response] = None
+    ) -> Iterable[Mapping[str, Any]]:
+        """
+        Extracts records from the given response by:
+            1) Saving the result to a tmp file
+            2) Reading from saved file by chunks to avoid OOM
+
+        Args:
+            response (Optional[requests.Response]): The response object containing the data. Defaults to None.
+
+        Yields:
+            Iterable[Mapping[str, Any]]: An iterable of mappings representing the extracted records.
+
+        Returns:
+            None
+        """
+        if response:
+            file_path, encoding = self._save_to_file(response)
+            yield from self._read_with_chunks(file_path, encoding)
+        else:
+            yield from []

airbyte_cdk/sources/declarative/extractors/type_transformer.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, Mapping
+
+
+@dataclass
+class TypeTransformer(ABC):
+    """
+    Abstract base class for implementing type transformation logic.
+
+    This class provides a blueprint for defining custom transformations
+    on data records based on a provided schema. Implementing classes
+    must override the `transform` method to specify the transformation
+    logic.
+
+    Attributes:
+        None explicitly defined, as this is a dataclass intended to be
+        subclassed.
+
+    Methods:
+        transform(record: Dict[str, Any], schema: Mapping[str, Any]) -> None:
+            Abstract method that must be implemented by subclasses.
+            It performs a transformation on a given data record based
+            on the provided schema.
+
+    Usage:
+        To use this class, create a subclass that implements the
+        `transform` method with the desired transformation logic.
+    """
+
+    @abstractmethod
+    def transform(
+        self,
+        record: Dict[str, Any],
+        schema: Mapping[str, Any],
+    ) -> None:
+        """
+        Perform a transformation on a data record based on a given schema.
+
+        Args:
+            record (Dict[str, Any]): The data record to be transformed.
+            schema (Mapping[str, Any]): The schema that dictates how
+                the record should be transformed.
+
+        Returns:
+            None
+
+        Raises:
+            NotImplementedError: If the method is not implemented
+                by a subclass.
+        """

airbyte_cdk/sources/declarative/incremental/__init__.py

@@ -2,8 +2,30 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from airbyte_cdk.sources.declarative.incremental.cursor import Cursor
 from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor
-from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import CursorFactory, PerPartitionCursor
+from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor
+from airbyte_cdk.sources.declarative.incremental.global_substream_cursor import (
+    GlobalSubstreamCursor,
+)
+from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import (
+    CursorFactory,
+    PerPartitionCursor,
+)
+from airbyte_cdk.sources.declarative.incremental.per_partition_with_global import (
+    PerPartitionWithGlobalCursor,
+)
+from airbyte_cdk.sources.declarative.incremental.resumable_full_refresh_cursor import (
+    ChildPartitionResumableFullRefreshCursor,
+    ResumableFullRefreshCursor,
+)
 
-__all__ = ["Cursor", "CursorFactory", "DatetimeBasedCursor", "PerPartitionCursor"]
+__all__ = [
+    "CursorFactory",
+    "DatetimeBasedCursor",
+    "DeclarativeCursor",
+    "GlobalSubstreamCursor",
+    "PerPartitionCursor",
+    "PerPartitionWithGlobalCursor",
+    "ResumableFullRefreshCursor",
+    "ChildPartitionResumableFullRefreshCursor",
+]