airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/decoders/pagination_decoder_decorator.py

@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from dataclasses import dataclass
+from typing import Any, Generator, MutableMapping
+
+import requests
+
+from airbyte_cdk.sources.declarative.decoders import Decoder
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class PaginationDecoderDecorator(Decoder):
+    """
+    Decoder to wrap other decoders when instantiating a DefaultPaginator in order to bypass decoding if the response is streamed.
+    """
+
+    def __init__(self, decoder: Decoder):
+        self._decoder = decoder
+
+    @property
+    def decoder(self) -> Decoder:
+        return self._decoder
+
+    def is_stream_response(self) -> bool:
+        return self._decoder.is_stream_response()
+
+    def decode(
+        self, response: requests.Response
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        if self._decoder.is_stream_response():
+            logger.warning("Response is streamed and therefore will not be decoded for pagination.")
+            yield {}
+        else:
+            yield from self._decoder.decode(response)

airbyte_cdk/sources/declarative/decoders/xml_decoder.py

@@ -0,0 +1,98 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+from dataclasses import InitVar, dataclass
+from typing import Any, Generator, Mapping, MutableMapping
+from xml.parsers.expat import ExpatError
+
+import requests
+import xmltodict
+
+from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class XmlDecoder(Decoder):
+    """
+    XmlDecoder is a decoder strategy that parses the XML content of the resopnse, and converts it to a dict.
+
+    This class handles XML attributes by prefixing them with an '@' symbol and represents XML text content by using the '#text' key if the element has attributes or the element name/tag. It does not currently support XML namespace declarations.
+
+    Example XML Input:
+    <root>
+        <location id="123">
+            San Francisco
+        </location>
+        <item id="1" category="books">
+            <name>Book Title 1</name>
+            <price>10.99</price>
+        </item>
+        <item id="2" category="electronics">
+            <name>Gadget</name>
+            <price>299.99</price>
+            <description>A useful gadget</description>
+        </item>
+    </root>
+
+    Converted Output:
+    {
+        "root": {
+            "location: {
+                "@id": "123,
+                "#text": "San Francisco"
+            },
+            "item": [
+              {
+                "@id": "1",
+                "@category": "books",
+                "name": "Book Title 1",
+                "price": "10.99"
+              },
+              {
+                "@id": "2",
+                "@category": "electronics",
+                "name": "Gadget",
+                "price": "299.99",
+                "description": "A useful gadget"
+              }
+            ]
+        }
+    }
+
+    Notes:
+        - Attributes of an XML element are prefixed with an '@' symbol in the dictionary output.
+        - Text content of an XML element is handled in two different ways, depending on whether
+          the element has attributes.
+                - If the element has attributes, the text content will be
+                  represented by the "#text" key.
+                - If the element does not have any attributes, the text content will be
+                  represented by element name.
+        - Namespace declarations are not supported in the current implementation.
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+
+    def is_stream_response(self) -> bool:
+        return False
+
+    def decode(
+        self, response: requests.Response
+    ) -> Generator[MutableMapping[str, Any], None, None]:
+        body_xml = response.text
+        try:
+            body_json = xmltodict.parse(body_xml)
+            if not isinstance(body_json, list):
+                body_json = [body_json]
+            if len(body_json) == 0:
+                yield {}
+            else:
+                yield from body_json
+        except ExpatError as exc:
+            logger.warning(
+                f"Response cannot be parsed from XML: {response.status_code=}, {response.text=}, {exc=}"
+            )
+            yield {}

airbyte_cdk/sources/declarative/decoders/zipfile_decoder.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+#
+
+import logging
+import zipfile
+from dataclasses import dataclass
+from io import BytesIO
+
+import requests
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.declarative.decoders import Decoder
+from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import Parser
+from airbyte_cdk.sources.declarative.decoders.decoder import DECODER_OUTPUT_TYPE
+from airbyte_cdk.utils import AirbyteTracedException
+
+logger = logging.getLogger("airbyte")
+
+
+@dataclass
+class ZipfileDecoder(Decoder):
+    parser: Parser
+
+    def is_stream_response(self) -> bool:
+        return False
+
+    def decode(self, response: requests.Response) -> DECODER_OUTPUT_TYPE:
+        try:
+            with zipfile.ZipFile(BytesIO(response.content)) as zip_file:
+                for file_name in zip_file.namelist():
+                    unzipped_content = zip_file.read(file_name)
+                    buffered_content = BytesIO(unzipped_content)
+                    try:
+                        yield from self.parser.parse(
+                            buffered_content,
+                        )
+                    except Exception as e:
+                        logger.error(
+                            f"Failed to parse file: {file_name} from zip file: {response.request.url} with exception {e}."
+                        )
+                        raise AirbyteTracedException(
+                            message=f"Failed to parse file: {file_name} from zip file.",
+                            internal_message=f"Failed to parse file: {file_name} from zip file: {response.request.url}.",
+                            failure_type=FailureType.system_error,
+                        ) from e
+        except zipfile.BadZipFile as e:
+            logger.error(
+                f"Received an invalid zip file in response to URL: {response.request.url}. "
+                f"The size of the response body is: {len(response.content)}"
+            )
+            raise AirbyteTracedException(
+                message="Received an invalid zip file in response.",
+                internal_message=f"Received an invalid zip file in response to URL: {response.request.url}.",
+                failure_type=FailureType.system_error,
+            ) from e

airbyte_cdk/sources/declarative/exceptions.py

@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+class ReadException(Exception):
+    """
+    Raise when there is an error reading data from an API Source
+    """

airbyte_cdk/sources/declarative/extractors/__init__.py

@@ -0,0 +1,21 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
+from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
+from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
+from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector
+from airbyte_cdk.sources.declarative.extractors.response_to_file_extractor import (
+    ResponseToFileExtractor,
+)
+from airbyte_cdk.sources.declarative.extractors.type_transformer import TypeTransformer
+
+__all__ = [
+    "TypeTransformer",
+    "HttpSelector",
+    "DpathExtractor",
+    "RecordFilter",
+    "RecordSelector",
+    "ResponseToFileExtractor",
+]

airbyte_cdk/sources/declarative/extractors/dpath_extractor.py

@@ -0,0 +1,86 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Iterable, List, Mapping, MutableMapping, Union
+
+import dpath
+import requests
+
+from airbyte_cdk.sources.declarative.decoders import Decoder, JsonDecoder
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class DpathExtractor(RecordExtractor):
+    """
+    Record extractor that searches a decoded response over a path defined as an array of fields.
+
+    If the field path points to an array, that array is returned.
+    If the field path points to an object, that object is returned wrapped as an array.
+    If the field path points to an empty object, an empty array is returned.
+    If the field path points to a non-existing path, an empty array is returned.
+
+    Examples of instantiating this transform:
+    ```
+      extractor:
+        type: DpathExtractor
+        field_path:
+          - "root"
+          - "data"
+    ```
+
+    ```
+      extractor:
+        type: DpathExtractor
+        field_path:
+          - "root"
+          - "{{ parameters['field'] }}"
+    ```
+
+    ```
+      extractor:
+        type: DpathExtractor
+        field_path: []
+    ```
+
+    Attributes:
+        field_path (Union[InterpolatedString, str]): Path to the field that should be extracted
+        config (Config): The user-provided configuration as specified by the source's spec
+        decoder (Decoder): The decoder responsible to transfom the response in a Mapping
+    """
+
+    field_path: List[Union[InterpolatedString, str]]
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    decoder: Decoder = field(default_factory=lambda: JsonDecoder(parameters={}))
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._field_path = [
+            InterpolatedString.create(path, parameters=parameters) for path in self.field_path
+        ]
+        for path_index in range(len(self.field_path)):
+            if isinstance(self.field_path[path_index], str):
+                self._field_path[path_index] = InterpolatedString.create(
+                    self.field_path[path_index], parameters=parameters
+                )
+
+    def extract_records(self, response: requests.Response) -> Iterable[MutableMapping[Any, Any]]:
+        for body in self.decoder.decode(response):
+            if len(self._field_path) == 0:
+                extracted = body
+            else:
+                path = [path.eval(self.config) for path in self._field_path]
+                if "*" in path:
+                    extracted = dpath.values(body, path)
+                else:
+                    extracted = dpath.get(body, path, default=[])  # type: ignore # extracted will be a MutableMapping, given input data structure
+            if isinstance(extracted, list):
+                yield from extracted
+            elif extracted:
+                yield extracted
+            else:
+                yield from []

airbyte_cdk/sources/declarative/extractors/http_selector.py

@@ -0,0 +1,37 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from abc import abstractmethod
+from typing import Any, Iterable, Mapping, Optional
+
+import requests
+
+from airbyte_cdk.sources.types import Record, StreamSlice, StreamState
+
+
+class HttpSelector:
+    """
+    Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering
+    records based on a heuristic.
+    """
+
+    @abstractmethod
+    def select_records(
+        self,
+        response: requests.Response,
+        stream_state: StreamState,
+        records_schema: Mapping[str, Any],
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Record]:
+        """
+        Selects records from the response
+        :param response: The response to select the records from
+        :param stream_state: The stream state
+        :param records_schema: json schema of records to return
+        :param stream_slice: The stream slice
+        :param next_page_token: The paginator token
+        :return: List of Records selected from the response
+        """
+        pass

airbyte_cdk/sources/declarative/extractors/record_extractor.py

@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, Iterable, Mapping
+
+import requests
+
+
+@dataclass
+class RecordExtractor:
+    """
+    Responsible for translating an HTTP response into a list of records by extracting records from the response.
+    """
+
+    @abstractmethod
+    def extract_records(
+        self,
+        response: requests.Response,
+    ) -> Iterable[Mapping[str, Any]]:
+        """
+        Selects records from the response
+        :param response: The response to extract the records from
+        :return: List of Records extracted from the response
+        """
+        pass

airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -0,0 +1,91 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+from dataclasses import InitVar, dataclass
+from typing import Any, Iterable, Mapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.incremental import (
+    DatetimeBasedCursor,
+    GlobalSubstreamCursor,
+    PerPartitionWithGlobalCursor,
+)
+from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
+
+
+@dataclass
+class RecordFilter:
+    """
+    Filter applied on a list of Records
+
+    config (Config): The user-provided configuration as specified by the source's spec
+    condition (str): The string representing the predicate to filter a record. Records will be removed if evaluated to False
+    """
+
+    parameters: InitVar[Mapping[str, Any]]
+    config: Config
+    condition: str = ""
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._filter_interpolator = InterpolatedBoolean(
+            condition=self.condition, parameters=parameters
+        )
+
+    def filter_records(
+        self,
+        records: Iterable[Mapping[str, Any]],
+        stream_state: StreamState,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Mapping[str, Any]]:
+        kwargs = {
+            "stream_state": stream_state,
+            "stream_slice": stream_slice,
+            "next_page_token": next_page_token,
+            "stream_slice.extra_fields": stream_slice.extra_fields if stream_slice else {},
+        }
+        for record in records:
+            if self._filter_interpolator.eval(self.config, record=record, **kwargs):
+                yield record
+
+
+class ClientSideIncrementalRecordFilterDecorator(RecordFilter):
+    """
+    Applies a filter to a list of records to exclude those that are older than the stream_state/start_date.
+
+    :param DatetimeBasedCursor date_time_based_cursor: Cursor used to extract datetime values
+    :param PerPartitionCursor per_partition_cursor: Optional Cursor used for mapping cursor value in nested stream_state
+    """
+
+    def __init__(
+        self,
+        cursor: Union[DatetimeBasedCursor, PerPartitionWithGlobalCursor, GlobalSubstreamCursor],
+        **kwargs: Any,
+    ):
+        super().__init__(**kwargs)
+        self._cursor = cursor
+
+    def filter_records(
+        self,
+        records: Iterable[Mapping[str, Any]],
+        stream_state: StreamState,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Mapping[str, Any]]:
+        records = (
+            record
+            for record in records
+            if self._cursor.should_be_synced(
+                # Record is created on the fly to align with cursors interface; stream name is ignored as we don't need it here
+                # Record stream name is empty cause it is not used durig the filtering
+                Record(data=record, associated_slice=stream_slice, stream_name="")
+            )
+        )
+        if self.condition:
+            records = super().filter_records(
+                records=records,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            )
+        yield from records

airbyte_cdk/sources/declarative/extractors/record_selector.py

@@ -0,0 +1,170 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass, field
+from typing import Any, Iterable, List, Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
+from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
+from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
+from airbyte_cdk.sources.declarative.extractors.type_transformer import (
+    TypeTransformer as DeclarativeTypeTransformer,
+)
+from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
+from airbyte_cdk.sources.declarative.models import SchemaNormalization
+from airbyte_cdk.sources.declarative.retrievers.file_uploader import FileUploader
+from airbyte_cdk.sources.declarative.transformations import RecordTransformation
+from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState
+from airbyte_cdk.sources.utils.transform import TypeTransformer
+
+
+@dataclass
+class RecordSelector(HttpSelector):
+    """
+    Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering
+    records based on a heuristic.
+
+    Attributes:
+        extractor (RecordExtractor): The record extractor responsible for extracting records from a response
+        schema_normalization (TypeTransformer): The record normalizer responsible for casting record values to stream schema types
+        record_filter (RecordFilter): The record filter responsible for filtering extracted records
+        transformations (List[RecordTransformation]): The transformations to be done on the records
+    """
+
+    extractor: RecordExtractor
+    config: Config
+    parameters: InitVar[Mapping[str, Any]]
+    schema_normalization: Union[TypeTransformer, DeclarativeTypeTransformer]
+    name: str
+    _name: Union[InterpolatedString, str] = field(init=False, repr=False, default="")
+    record_filter: Optional[RecordFilter] = None
+    transformations: List[RecordTransformation] = field(default_factory=lambda: [])
+    transform_before_filtering: bool = False
+    file_uploader: Optional[FileUploader] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._parameters = parameters
+        self._name = (
+            InterpolatedString(self._name, parameters=parameters)
+            if isinstance(self._name, str)
+            else self._name
+        )
+
+    @property  # type: ignore
+    def name(self) -> str:
+        """
+        :return: Stream name
+        """
+        return (
+            str(self._name.eval(self.config))
+            if isinstance(self._name, InterpolatedString)
+            else self._name
+        )
+
+    @name.setter
+    def name(self, value: str) -> None:
+        if not isinstance(value, property):
+            self._name = value
+
+    def select_records(
+        self,
+        response: requests.Response,
+        stream_state: StreamState,
+        records_schema: Mapping[str, Any],
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Record]:
+        """
+        Selects records from the response
+        :param response: The response to select the records from
+        :param stream_state: The stream state
+        :param records_schema: json schema of records to return
+        :param stream_slice: The stream slice
+        :param next_page_token: The paginator token
+        :return: List of Records selected from the response
+        """
+        all_data: Iterable[Mapping[str, Any]] = self.extractor.extract_records(response)
+        yield from self.filter_and_transform(
+            all_data, stream_state, records_schema, stream_slice, next_page_token
+        )
+
+    def filter_and_transform(
+        self,
+        all_data: Iterable[Mapping[str, Any]],
+        stream_state: StreamState,
+        records_schema: Mapping[str, Any],
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Iterable[Record]:
+        """
+        There is an issue with the selector as of 2024-08-30: it does technology-agnostic processing like filtering, transformation and
+        normalization with an API that is technology-specific (as requests.Response is only for HTTP communication using the requests
+        library).
+
+        Until we decide to move this logic away from the selector, we made this method public so that users like AsyncJobRetriever could
+        share the logic of doing transformations on a set of records.
+        """
+        if self.transform_before_filtering:
+            transformed_data = self._transform(all_data, stream_state, stream_slice)
+            transformed_filtered_data = self._filter(
+                transformed_data, stream_state, stream_slice, next_page_token
+            )
+        else:
+            filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token)
+            transformed_filtered_data = self._transform(filtered_data, stream_state, stream_slice)
+        normalized_data = self._normalize_by_schema(
+            transformed_filtered_data, schema=records_schema
+        )
+        for data in normalized_data:
+            record = Record(data=data, stream_name=self.name, associated_slice=stream_slice)
+            if self.file_uploader:
+                self.file_uploader.upload(record)
+            yield record
+
+    def _normalize_by_schema(
+        self, records: Iterable[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]
+    ) -> Iterable[Mapping[str, Any]]:
+        if schema:
+            # record has type Mapping[str, Any], but dict[str, Any] expected
+            for record in records:
+                normalized_record = dict(record)
+                self.schema_normalization.transform(normalized_record, schema)
+                yield normalized_record
+        else:
+            yield from records
+
+    def _filter(
+        self,
+        records: Iterable[Mapping[str, Any]],
+        stream_state: StreamState,
+        stream_slice: Optional[StreamSlice],
+        next_page_token: Optional[Mapping[str, Any]],
+    ) -> Iterable[Mapping[str, Any]]:
+        if self.record_filter:
+            yield from self.record_filter.filter_records(
+                records,
+                stream_state=stream_state,
+                stream_slice=stream_slice,
+                next_page_token=next_page_token,
+            )
+        else:
+            yield from records
+
+    def _transform(
+        self,
+        records: Iterable[Mapping[str, Any]],
+        stream_state: StreamState,
+        stream_slice: Optional[StreamSlice] = None,
+    ) -> Iterable[Mapping[str, Any]]:
+        for record in records:
+            for transformation in self.transformations:
+                transformation.transform(
+                    record,  # type: ignore  # record has type Mapping[str, Any], but Dict[str, Any] expected
+                    config=self.config,
+                    stream_state=stream_state,
+                    stream_slice=stream_slice,
+                )
+            yield record