airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Iterable, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.incremental import DeclarativeCursor
+from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState
+from airbyte_cdk.sources.streams.checkpoint.checkpoint_reader import FULL_REFRESH_COMPLETE_STATE
+
+
+@dataclass
+class ResumableFullRefreshCursor(DeclarativeCursor):
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._cursor: StreamState = {}
+
+    def get_stream_state(self) -> StreamState:
+        return self._cursor
+
+    def set_initial_state(self, stream_state: StreamState) -> None:
+        self._cursor = stream_state
+
+    def observe(self, stream_slice: StreamSlice, record: Record) -> None:
+        """
+        Resumable full refresh manages state using a page number so it does not need to update state by observing incoming records.
+        """
+        pass
+
+    def close_slice(self, stream_slice: StreamSlice, *args: Any) -> None:
+        # The ResumableFullRefreshCursor doesn't support nested streams yet so receiving a partition is unexpected
+        if stream_slice.partition:
+            raise ValueError(
+                f"Stream slice {stream_slice} should not have a partition. Got {stream_slice.partition}."
+            )
+        self._cursor = stream_slice.cursor_slice
+
+    def should_be_synced(self, record: Record) -> bool:
+        """
+        Unlike date-based cursors which filter out records outside slice boundaries, resumable full refresh records exist within pages
+        that don't have filterable bounds. We should always return them.
+        """
+        return True
+
+    def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
+        """
+        RFR record don't have ordering to be compared between one another.
+        """
+        return False
+
+    def select_state(self, stream_slice: Optional[StreamSlice] = None) -> Optional[StreamState]:
+        # A top-level RFR cursor only manages the state of a single partition
+        return self._cursor
+
+    def stream_slices(self) -> Iterable[StreamSlice]:
+        """
+        Resumable full refresh cursors only return a single slice and can't perform partitioning because iteration is done per-page
+        along an unbounded set.
+        """
+        yield from [StreamSlice(cursor_slice=self._cursor, partition={})]
+
+    # This is an interesting pattern that might not seem obvious at first glance. This cursor itself has no functional need to
+    # inject any request values into the outbound response because the up-to-date pagination state is already loaded and
+    # maintained by the paginator component
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+
+@dataclass
+class ChildPartitionResumableFullRefreshCursor(ResumableFullRefreshCursor):
+    """
+    The Sub-stream Resumable Cursor for Full-Refresh substreams.
+    Follows the parent type `ResumableFullRefreshCursor` with a small override,
+    to provide the ability to close the substream's slice once it has finished processing.
+
+    Check the `close_slice` method overide for more info about the actual behaviour of this cursor.
+    """
+
+    def close_slice(self, stream_slice: StreamSlice, *args: Any) -> None:
+        """
+        Once the current slice has finished syncing:
+         - paginator returns None
+         - no more slices to process
+
+        we assume that the records are processed and emitted already,
+        thus we have to set the cursor to ` __ab_full_refresh_sync_complete: true `,
+        otherwise there is a risk of Inf. Loop processing the same slice.
+        """
+        self._cursor = FULL_REFRESH_COMPLETE_STATE

airbyte_cdk/sources/declarative/interpolation/__init__.py

@@ -0,0 +1,9 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
+from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+
+__all__ = ["InterpolatedBoolean", "InterpolatedMapping", "InterpolatedString"]

airbyte_cdk/sources/declarative/interpolation/filters.py

@@ -0,0 +1,139 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import base64
+import hashlib
+import json
+import re
+from typing import Any, Optional
+
+
+def hash(value: Any, hash_type: str = "md5", salt: Optional[str] = None) -> str:
+    """
+      Implementation of a custom Jinja2 hash filter
+      Hash type defaults to 'md5' if one is not specified.
+
+      If you are using this has function for GDPR compliance, then
+      you should probably also pass in a salt as discussed in:
+      https://security.stackexchange.com/questions/202022/hashing-email-addresses-for-gdpr-compliance
+
+      This can be used in a low code connector definition under the AddFields transformation.
+      For example:
+
+    rates_stream:
+      $ref: "#/definitions/base_stream"
+      $parameters:
+        name: "rates"
+        primary_key: "date"
+        path: "/exchangerates_data/latest"
+      transformations:
+        - type: AddFields
+          fields:
+            - path: ["some_new_path"]
+              value: "{{ record['rates']['CAD'] | hash('md5', 'mysalt')  }}"
+
+
+
+      :param value: value to be hashed
+      :param hash_type: valid hash type
+      :param salt: a salt that will be combined with the value to ensure that the hash created for a given value on this system
+                   is different from the hash created for that value on other systems.
+      :return: computed hash as a hexadecimal string
+    """
+    hash_func = getattr(hashlib, hash_type, None)
+
+    if hash_func:
+        hash_obj = hash_func()
+        hash_obj.update(str(value).encode("utf-8"))
+        if salt:
+            hash_obj.update(str(salt).encode("utf-8"))
+        computed_hash: str = hash_obj.hexdigest()
+    else:
+        raise AttributeError("No hashing function named {hname}".format(hname=hash_type))
+
+    return computed_hash
+
+
+def base64encode(value: str) -> str:
+    """
+    Implementation of a custom Jinja2 base64encode filter
+
+    For example:
+
+      OAuthAuthenticator:
+        $ref: "#/definitions/OAuthAuthenticator"
+        $parameters:
+          name: "client_id"
+          value: "{{ config['client_id'] | base64encode }}"
+
+    :param value: value to be encoded in base64
+    :return: base64 encoded string
+    """
+
+    return base64.b64encode(value.encode("utf-8")).decode()
+
+
+def base64decode(value: str) -> str:
+    """
+    Implementation of a custom Jinja2 base64decode filter
+
+    For example:
+
+      OAuthAuthenticator:
+        $ref: "#/definitions/OAuthAuthenticator"
+        $parameters:
+          name: "client_id"
+          value: "{{ config['client_id'] | base64decode }}"
+
+    :param value: value to be decoded from base64
+    :return: base64 decoded string
+    """
+
+    return base64.b64decode(value.encode("utf-8")).decode()
+
+
+def base64binascii_decode(value: str) -> str:
+    """
+    Implementation of a custom Jinja2 filter to decode base64 strings using ASCII encoding
+
+    For example:
+
+      OAuthAuthenticator:
+        $ref: "#/definitions/OAuthAuthenticator"
+        $parameters:
+          name: "client_id"
+          value: "{{ config['client_id'] | base64binascii_decode }}"
+
+    :param value: value to be decoded from base64 using ascii
+    :return: base64 decoded string ascii
+    """
+    return base64.standard_b64encode(value.encode("ascii")).decode("ascii")
+
+
+def string(value: Any) -> str:
+    """
+    Converts the input value to a string.
+    If the value is already a string, it is returned as is.
+    Otherwise, the value is interpreted as a json object and wrapped in triple-quotes so it's evalued as a string by the JinjaInterpolation
+    :param value: the value to convert to a string
+    :return: string representation of the input value
+    """
+    if isinstance(value, str):
+        return value
+    ret = f'"""{json.dumps(value)}"""'
+    return ret
+
+
+def regex_search(value: str, regex: str) -> str:
+    """
+    Match a regular expression against a string and return the first match group if it exists.
+    """
+    match = re.search(regex, value)
+    if match and len(match.groups()) > 0:
+        return match.group(1)
+    return ""
+
+
+_filters_list = [hash, base64encode, base64decode, base64binascii_decode, string, regex_search]
+filters = {f.__name__: f for f in _filters_list}

airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py

@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Final, List, Mapping
+
+from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
+from airbyte_cdk.sources.types import Config
+
+FALSE_VALUES: Final[List[Any]] = [
+    "False",
+    "false",
+    "{}",
+    "[]",
+    "()",
+    "",
+    "0",
+    "0.0",
+    {},
+    False,
+    [],
+    (),
+    set(),
+]
+
+
+@dataclass
+class InterpolatedBoolean:
+    f"""
+    Wrapper around a string to be evaluated to a boolean value.
+    The string will be evaluated as False if it interpolates to a value in {FALSE_VALUES}
+
+    Attributes:
+        condition (str): The string representing the condition to evaluate to a boolean
+    """
+    condition: str
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._default = "False"
+        self._interpolation = JinjaInterpolation()
+        self._parameters = parameters
+
+    def eval(self, config: Config, **additional_parameters: Any) -> bool:
+        """
+        Interpolates the predicate condition string using the config and other optional arguments passed as parameter.
+
+        :param config: The user-provided configuration as specified by the source's spec
+        :param additional_parameters: Optional parameters used for interpolation
+        :return: The interpolated string
+        """
+        if isinstance(self.condition, bool):
+            return self.condition
+        else:
+            evaluated = self._interpolation.eval(
+                self.condition,
+                config,
+                self._default,
+                parameters=self._parameters,
+                **additional_parameters,
+            )
+            if evaluated in FALSE_VALUES:
+                return False
+            # The presence of a value is generally regarded as truthy, so we treat it as such
+            return True

airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Dict, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class InterpolatedMapping:
+    """
+    Wrapper around a Mapping[str, str] where both the keys and values are to be interpolated.
+
+    Attributes:
+        mapping (Mapping[str, str]): to be evaluated
+    """
+
+    mapping: Mapping[str, str]
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Optional[Mapping[str, Any]]) -> None:
+        self._interpolation = JinjaInterpolation()
+        self._parameters = parameters
+
+    def eval(self, config: Config, **additional_parameters: Any) -> Dict[str, Any]:
+        """
+        Wrapper around a Mapping[str, str] that allows for both keys and values to be interpolated.
+
+        :param config: The user-provided configuration as specified by the source's spec
+        :param additional_parameters: Optional parameters used for interpolation
+        :return: The interpolated mapping
+        """
+        valid_key_types = additional_parameters.pop("valid_key_types", (str,))
+        valid_value_types = additional_parameters.pop("valid_value_types", None)
+        return {
+            self._interpolation.eval(
+                name,
+                config,
+                valid_types=valid_key_types,
+                parameters=self._parameters,
+                **additional_parameters,
+            ): self._eval(value, config, valid_types=valid_value_types, **additional_parameters)
+            for name, value in self.mapping.items()
+        }
+
+    def _eval(self, value: str, config: Config, **kwargs: Any) -> Any:
+        # The values in self._mapping can be of Any type
+        # We only want to interpolate them if they are strings
+        if isinstance(value, str):
+            return self._interpolation.eval(value, config, parameters=self._parameters, **kwargs)
+        else:
+            return value

airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
+from airbyte_cdk.sources.types import Config
+
+NestedMappingEntry = Union[
+    dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None
+]
+NestedMapping = Union[dict[str, NestedMappingEntry], str]
+
+
+@dataclass
+class InterpolatedNestedMapping:
+    """
+    Wrapper around a nested dict which can contain lists and primitive values where both the keys and values are interpolated recursively.
+
+    Attributes:
+        mapping (NestedMapping): to be evaluated
+    """
+
+    mapping: NestedMapping
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Optional[Mapping[str, Any]]) -> None:
+        self._interpolation = JinjaInterpolation()
+        self._parameters = parameters
+
+    def eval(self, config: Config, **additional_parameters: Any) -> Any:
+        return self._eval(self.mapping, config, **additional_parameters)
+
+    def _eval(
+        self, value: Union[NestedMapping, NestedMappingEntry], config: Config, **kwargs: Any
+    ) -> Any:
+        # Recursively interpolate dictionaries and lists
+        if isinstance(value, str):
+            return self._interpolation.eval(value, config, parameters=self._parameters, **kwargs)
+        elif isinstance(value, dict):
+            interpolated_dict = {
+                self._eval(k, config, **kwargs): self._eval(v, config, **kwargs)
+                for k, v in value.items()
+            }
+            return {k: v for k, v in interpolated_dict.items() if v is not None}
+        elif isinstance(value, list):
+            return [self._eval(v, config, **kwargs) for v in value]
+        else:
+            return value

airbyte_cdk/sources/declarative/interpolation/interpolated_string.py

@@ -0,0 +1,79 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
+from airbyte_cdk.sources.types import Config
+
+
+@dataclass
+class InterpolatedString:
+    """
+    Wrapper around a raw string to be interpolated with the Jinja2 templating engine
+
+    Attributes:
+        string (str): The string to evalute
+        default (Optional[str]): The default value to return if the evaluation returns an empty string
+        parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation
+    """
+
+    string: str
+    parameters: InitVar[Mapping[str, Any]]
+    default: Optional[str] = None
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self.default = self.default or self.string
+        self._interpolation = JinjaInterpolation()
+        self._parameters = parameters
+        # indicates whether passed string is just a plain string, not Jinja template
+        # This allows for optimization, but we do not know it yet at this stage
+        self._is_plain_string = None
+
+    def eval(self, config: Config, **kwargs: Any) -> Any:
+        """
+        Interpolates the input string using the config and other optional arguments passed as parameter.
+
+        :param config: The user-provided configuration as specified by the source's spec
+        :param kwargs: Optional parameters used for interpolation
+        :return: The interpolated string
+        """
+        if self._is_plain_string:
+            return self.string
+        if self._is_plain_string is None:
+            # Let's check whether output from evaluation is the same as input.
+            # This indicates occurrence of a plain string, not a template and we can skip Jinja in subsequent runs.
+            evaluated = self._interpolation.eval(
+                self.string, config, self.default, parameters=self._parameters, **kwargs
+            )
+            self._is_plain_string = self.string == evaluated
+            return evaluated
+        return self._interpolation.eval(
+            self.string, config, self.default, parameters=self._parameters, **kwargs
+        )
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, InterpolatedString):
+            return False
+        return self.string == other.string and self.default == other.default
+
+    @classmethod
+    def create(
+        cls,
+        string_or_interpolated: Union["InterpolatedString", str],
+        *,
+        parameters: Mapping[str, Any],
+    ) -> "InterpolatedString":
+        """
+        Helper function to obtain an InterpolatedString from either a raw string or an InterpolatedString.
+
+        :param string_or_interpolated: either a raw string or an InterpolatedString.
+        :param parameters: parameters propagated from parent component
+        :return: InterpolatedString representing the input string.
+        """
+        if isinstance(string_or_interpolated, str):
+            return InterpolatedString(string=string_or_interpolated, parameters=parameters)
+        else:
+            return string_or_interpolated

airbyte_cdk/sources/declarative/interpolation/interpolation.py

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+from airbyte_cdk.sources.types import Config
+
+
+class Interpolation(ABC):
+    """
+    Strategy for evaluating the interpolated value of a string at runtime using Jinja.
+    """
+
+    @abstractmethod
+    def eval(
+        self,
+        input_str: str,
+        config: Config,
+        default: Optional[str] = None,
+        **additional_options: Any,
+    ) -> Any:
+        """
+        Interpolates the input string using the config, and additional options passed as parameter.
+
+        :param input_str: The string to interpolate
+        :param config: The user-provided configuration as specified by the source's spec
+        :param default: Default value to return if the evaluation returns an empty string
+        :param additional_options: Optional parameters used for interpolation
+        :return: The interpolated string
+        """
+        pass