airbyte-cdk 0.72.0__py3-none-any.whl → 6.17.1.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/incremental/resumable_full_refresh_cursor.py

@@ -0,0 +1,122 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from dataclasses import InitVar, dataclass
+from typing import Any, Iterable, Mapping, Optional
+
+from airbyte_cdk.sources.declarative.incremental import DeclarativeCursor
+from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState
+from airbyte_cdk.sources.streams.checkpoint.checkpoint_reader import FULL_REFRESH_COMPLETE_STATE
+
+
+@dataclass
+class ResumableFullRefreshCursor(DeclarativeCursor):
+    parameters: InitVar[Mapping[str, Any]]
+
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        self._cursor: StreamState = {}
+
+    def get_stream_state(self) -> StreamState:
+        return self._cursor
+
+    def set_initial_state(self, stream_state: StreamState) -> None:
+        self._cursor = stream_state
+
+    def observe(self, stream_slice: StreamSlice, record: Record) -> None:
+        """
+        Resumable full refresh manages state using a page number so it does not need to update state by observing incoming records.
+        """
+        pass
+
+    def close_slice(self, stream_slice: StreamSlice, *args: Any) -> None:
+        # The ResumableFullRefreshCursor doesn't support nested streams yet so receiving a partition is unexpected
+        if stream_slice.partition:
+            raise ValueError(
+                f"Stream slice {stream_slice} should not have a partition. Got {stream_slice.partition}."
+            )
+        self._cursor = stream_slice.cursor_slice
+
+    def should_be_synced(self, record: Record) -> bool:
+        """
+        Unlike date-based cursors which filter out records outside slice boundaries, resumable full refresh records exist within pages
+        that don't have filterable bounds. We should always return them.
+        """
+        return True
+
+    def is_greater_than_or_equal(self, first: Record, second: Record) -> bool:
+        """
+        RFR record don't have ordering to be compared between one another.
+        """
+        return False
+
+    def select_state(self, stream_slice: Optional[StreamSlice] = None) -> Optional[StreamState]:
+        # A top-level RFR cursor only manages the state of a single partition
+        return self._cursor
+
+    def stream_slices(self) -> Iterable[StreamSlice]:
+        """
+        Resumable full refresh cursors only return a single slice and can't perform partitioning because iteration is done per-page
+        along an unbounded set.
+        """
+        yield from [StreamSlice(cursor_slice=self._cursor, partition={})]
+
+    # This is an interesting pattern that might not seem obvious at first glance. This cursor itself has no functional need to
+    # inject any request values into the outbound response because the up-to-date pagination state is already loaded and
+    # maintained by the paginator component
+    def get_request_params(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_headers(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_body_data(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+    def get_request_body_json(
+        self,
+        *,
+        stream_state: Optional[StreamState] = None,
+        stream_slice: Optional[StreamSlice] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Mapping[str, Any]:
+        return {}
+
+
+@dataclass
+class ChildPartitionResumableFullRefreshCursor(ResumableFullRefreshCursor):
+    """
+    The Sub-stream Resumable Cursor for Full-Refresh substreams.
+    Follows the parent type `ResumableFullRefreshCursor` with a small override,
+    to provide the ability to close the substream's slice once it has finished processing.
+
+    Check the `close_slice` method overide for more info about the actual behaviour of this cursor.
+    """
+
+    def close_slice(self, stream_slice: StreamSlice, *args: Any) -> None:
+        """
+        Once the current slice has finished syncing:
+         - paginator returns None
+         - no more slices to process
+
+        we assume that the records are processed and emitted already,
+        thus we have to set the cursor to ` __ab_full_refresh_sync_complete: true `,
+        otherwise there is a risk of Inf. Loop processing the same slice.
+        """
+        self._cursor = FULL_REFRESH_COMPLETE_STATE

airbyte_cdk/sources/declarative/interpolation/filters.py

@@ -3,6 +3,8 @@
 #
 import base64
 import hashlib
+import json
+import re
 from typing import Any, Optional
 
 
@@ -90,5 +92,29 @@ def base64decode(value: str) -> str:
     return base64.b64decode(value.encode("utf-8")).decode()
 
 
-_filters_list = [hash, base64encode, base64decode]
+def string(value: Any) -> str:
+    """
+    Converts the input value to a string.
+    If the value is already a string, it is returned as is.
+    Otherwise, the value is interpreted as a json object and wrapped in triple-quotes so it's evalued as a string by the JinjaInterpolation
+    :param value: the value to convert to a string
+    :return: string representation of the input value
+    """
+    if isinstance(value, str):
+        return value
+    ret = f'"""{json.dumps(value)}"""'
+    return ret
+
+
+def regex_search(value: str, regex: str) -> str:
+    """
+    Match a regular expression against a string and return the first match group if it exists.
+    """
+    match = re.search(regex, value)
+    if match and len(match.groups()) > 0:
+        return match.group(1)
+    return ""
+
+
+_filters_list = [hash, base64encode, base64decode, string, regex_search]
 filters = {f.__name__: f for f in _filters_list}

airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py

@@ -6,9 +6,23 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Final, List, Mapping
 
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
-FALSE_VALUES: Final[List[Any]] = ["False", "false", "{}", "[]", "()", "", "0", "0.0", {}, False, [], (), set()]
+FALSE_VALUES: Final[List[Any]] = [
+    "False",
+    "false",
+    "{}",
+    "[]",
+    "()",
+    "",
+    "0",
+    "0.0",
+    {},
+    False,
+    [],
+    (),
+    set(),
+]
 
 
 @dataclass
@@ -23,12 +37,12 @@ class InterpolatedBoolean:
     condition: str
     parameters: InitVar[Mapping[str, Any]]
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._default = "False"
         self._interpolation = JinjaInterpolation()
         self._parameters = parameters
 
-    def eval(self, config: Config, **additional_parameters):
+    def eval(self, config: Config, **additional_parameters: Any) -> bool:
         """
         Interpolates the predicate condition string using the config and other optional arguments passed as parameter.
 
@@ -40,7 +54,11 @@ class InterpolatedBoolean:
             return self.condition
         else:
             evaluated = self._interpolation.eval(
-                self.condition, config, self._default, parameters=self._parameters, **additional_parameters
+                self.condition,
+                config,
+                self._default,
+                parameters=self._parameters,
+                **additional_parameters,
             )
             if evaluated in FALSE_VALUES:
                 return False

airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py

@@ -4,10 +4,10 @@
 
 
 from dataclasses import InitVar, dataclass
-from typing import Any, Mapping, Optional
+from typing import Any, Dict, Mapping, Optional
 
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -22,31 +22,35 @@ class InterpolatedMapping:
     mapping: Mapping[str, str]
     parameters: InitVar[Mapping[str, Any]]
 
-    def __post_init__(self, parameters: Optional[Mapping[str, Any]]):
+    def __post_init__(self, parameters: Optional[Mapping[str, Any]]) -> None:
         self._interpolation = JinjaInterpolation()
         self._parameters = parameters
 
-    def eval(self, config: Config, **additional_parameters):
+    def eval(self, config: Config, **additional_parameters: Any) -> Dict[str, Any]:
         """
         Wrapper around a Mapping[str, str] that allows for both keys and values to be interpolated.
 
         :param config: The user-provided configuration as specified by the source's spec
         :param additional_parameters: Optional parameters used for interpolation
-        :return: The interpolated string
+        :return: The interpolated mapping
         """
         valid_key_types = additional_parameters.pop("valid_key_types", (str,))
         valid_value_types = additional_parameters.pop("valid_value_types", None)
         return {
             self._interpolation.eval(
-                name, config, valid_types=valid_key_types, parameters=self._parameters, **additional_parameters
+                name,
+                config,
+                valid_types=valid_key_types,
+                parameters=self._parameters,
+                **additional_parameters,
             ): self._eval(value, config, valid_types=valid_value_types, **additional_parameters)
             for name, value in self.mapping.items()
         }
 
-    def _eval(self, value, config, **kwargs):
+    def _eval(self, value: str, config: Config, **kwargs: Any) -> Any:
         # The values in self._mapping can be of Any type
         # We only want to interpolate them if they are strings
-        if type(value) == str:
+        if isinstance(value, str):
             return self._interpolation.eval(value, config, parameters=self._parameters, **kwargs)
         else:
             return value

airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py

@@ -7,9 +7,11 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
-NestedMappingEntry = Union[dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None]
+NestedMappingEntry = Union[
+    dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None
+]
 NestedMapping = Union[dict[str, NestedMappingEntry], str]
 
 
@@ -25,19 +27,24 @@ class InterpolatedNestedMapping:
     mapping: NestedMapping
     parameters: InitVar[Mapping[str, Any]]
 
-    def __post_init__(self, parameters: Optional[Mapping[str, Any]]):
+    def __post_init__(self, parameters: Optional[Mapping[str, Any]]) -> None:
         self._interpolation = JinjaInterpolation()
         self._parameters = parameters
 
-    def eval(self, config: Config, **additional_parameters):
+    def eval(self, config: Config, **additional_parameters: Any) -> Any:
         return self._eval(self.mapping, config, **additional_parameters)
 
-    def _eval(self, value, config, **kwargs):
+    def _eval(
+        self, value: Union[NestedMapping, NestedMappingEntry], config: Config, **kwargs: Any
+    ) -> Any:
         # Recursively interpolate dictionaries and lists
         if isinstance(value, str):
             return self._interpolation.eval(value, config, parameters=self._parameters, **kwargs)
         elif isinstance(value, dict):
-            interpolated_dict = {self._eval(k, config, **kwargs): self._eval(v, config, **kwargs) for k, v in value.items()}
+            interpolated_dict = {
+                self._eval(k, config, **kwargs): self._eval(v, config, **kwargs)
+                for k, v in value.items()
+            }
             return {k: v for k, v in interpolated_dict.items() if v is not None}
         elif isinstance(value, list):
             return [self._eval(v, config, **kwargs) for v in value]

airbyte_cdk/sources/declarative/interpolation/interpolated_string.py

@@ -6,7 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -24,12 +24,15 @@ class InterpolatedString:
     parameters: InitVar[Mapping[str, Any]]
     default: Optional[str] = None
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.default = self.default or self.string
         self._interpolation = JinjaInterpolation()
         self._parameters = parameters
+        # indicates whether passed string is just a plain string, not Jinja template
+        # This allows for optimization, but we do not know it yet at this stage
+        self._is_plain_string = None
 
-    def eval(self, config: Config, **kwargs):
+    def eval(self, config: Config, **kwargs: Any) -> Any:
         """
         Interpolates the input string using the config and other optional arguments passed as parameter.
 
@@ -37,9 +40,21 @@ class InterpolatedString:
         :param kwargs: Optional parameters used for interpolation
         :return: The interpolated string
         """
-        return self._interpolation.eval(self.string, config, self.default, parameters=self._parameters, **kwargs)
+        if self._is_plain_string:
+            return self.string
+        if self._is_plain_string is None:
+            # Let's check whether output from evaluation is the same as input.
+            # This indicates occurrence of a plain string, not a template and we can skip Jinja in subsequent runs.
+            evaluated = self._interpolation.eval(
+                self.string, config, self.default, parameters=self._parameters, **kwargs
+            )
+            self._is_plain_string = self.string == evaluated
+            return evaluated
+        return self._interpolation.eval(
+            self.string, config, self.default, parameters=self._parameters, **kwargs
+        )
 
-    def __eq__(self, other):
+    def __eq__(self, other: Any) -> bool:
         if not isinstance(other, InterpolatedString):
             return False
         return self.string == other.string and self.default == other.default
@@ -50,7 +65,7 @@ class InterpolatedString:
         string_or_interpolated: Union["InterpolatedString", str],
         *,
         parameters: Mapping[str, Any],
-    ):
+    ) -> "InterpolatedString":
         """
         Helper function to obtain an InterpolatedString from either a raw string or an InterpolatedString.
 

airbyte_cdk/sources/declarative/interpolation/interpolation.py

@@ -3,9 +3,9 @@
 #
 
 from abc import ABC, abstractmethod
-from typing import Optional
+from typing import Any, Optional
 
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
 
 class Interpolation(ABC):
@@ -14,7 +14,13 @@ class Interpolation(ABC):
     """
 
     @abstractmethod
-    def eval(self, input_str: str, config: Config, default: Optional[str] = None, **additional_options):
+    def eval(
+        self,
+        input_str: str,
+        config: Config,
+        default: Optional[str] = None,
+        **additional_options: Any,
+    ) -> Any:
         """
         Interpolates the input string using the config, and additional options passed as parameter.
 

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -3,15 +3,59 @@
 #
 
 import ast
-from typing import Any, Mapping, Optional, Tuple, Type
+from functools import cache
+from typing import Any, Mapping, Optional, Set, Tuple, Type
+
+from jinja2 import meta
+from jinja2.environment import Template
+from jinja2.exceptions import UndefinedError
+from jinja2.sandbox import SandboxedEnvironment
 
 from airbyte_cdk.sources.declarative.interpolation.filters import filters
 from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
 from airbyte_cdk.sources.declarative.interpolation.macros import macros
-from airbyte_cdk.sources.declarative.types import Config
-from jinja2 import meta
-from jinja2.exceptions import UndefinedError
-from jinja2.sandbox import Environment
+from airbyte_cdk.sources.types import Config
+
+
+class StreamPartitionAccessEnvironment(SandboxedEnvironment):
+    """
+    Currently, source-jira is setting an attribute to StreamSlice specific to its use case which because of the PerPartitionCursor is set to
+    StreamSlice._partition but not exposed through StreamSlice.partition. This is a patch to still allow source-jira to have access to this
+    parameter
+    """
+
+    def is_safe_attribute(self, obj: Any, attr: str, value: Any) -> bool:
+        if attr in ["_partition"]:
+            return True
+        return super().is_safe_attribute(obj, attr, value)  # type: ignore  # for some reason, mypy says 'Returning Any from function declared to return "bool"'
+
+
+# These aliases are used to deprecate existing keywords without breaking all existing connectors.
+_ALIASES = {
+    "stream_interval": "stream_slice",  # Use stream_interval to access incremental_sync values
+    "stream_partition": "stream_slice",  # Use stream_partition to access partition router's values
+}
+
+# These extensions are not installed so they're not currently a problem,
+# but we're still explicitly removing them from the jinja context.
+# At worst, this is documentation that we do NOT want to include these extensions because of the potential security risks
+_RESTRICTED_EXTENSIONS = ["jinja2.ext.loopcontrols"]  # Adds support for break continue in loops
+
+# By default, these Python builtin functions are available in the Jinja context.
+# We explicitly remove them because of the potential security risk.
+# Please add a unit test to test_jinja.py when adding a restriction.
+_RESTRICTED_BUILTIN_FUNCTIONS = [
+    "range"
+]  # The range function can cause very expensive computations
+
+_ENVIRONMENT = StreamPartitionAccessEnvironment()
+_ENVIRONMENT.filters.update(**filters)
+_ENVIRONMENT.globals.update(**macros)
+
+for extension in _RESTRICTED_EXTENSIONS:
+    _ENVIRONMENT.extensions.pop(extension, None)
+for builtin in _RESTRICTED_BUILTIN_FUNCTIONS:
+    _ENVIRONMENT.globals.pop(builtin, None)
 
 
 class JinjaInterpolation(Interpolation):
@@ -32,32 +76,6 @@ class JinjaInterpolation(Interpolation):
     Additional information on jinja templating can be found at https://jinja.palletsprojects.com/en/3.1.x/templates/#
     """
 
-    # These aliases are used to deprecate existing keywords without breaking all existing connectors.
-    ALIASES = {
-        "stream_interval": "stream_slice",  # Use stream_interval to access incremental_sync values
-        "stream_partition": "stream_slice",  # Use stream_partition to access partition router's values
-    }
-
-    # These extensions are not installed so they're not currently a problem,
-    # but we're still explicitely removing them from the jinja context.
-    # At worst, this is documentation that we do NOT want to include these extensions because of the potential security risks
-    RESTRICTED_EXTENSIONS = ["jinja2.ext.loopcontrols"]  # Adds support for break continue in loops
-
-    # By default, these Python builtin functions are available in the Jinja context.
-    # We explicitely remove them because of the potential security risk.
-    # Please add a unit test to test_jinja.py when adding a restriction.
-    RESTRICTED_BUILTIN_FUNCTIONS = ["range"]  # The range function can cause very expensive computations
-
-    def __init__(self) -> None:
-        self._environment = Environment()
-        self._environment.filters.update(**filters)
-        self._environment.globals.update(**macros)
-
-        for extension in self.RESTRICTED_EXTENSIONS:
-            self._environment.extensions.pop(extension, None)
-        for builtin in self.RESTRICTED_BUILTIN_FUNCTIONS:
-            self._environment.globals.pop(builtin, None)
-
     def eval(
         self,
         input_str: str,
@@ -68,7 +86,7 @@ class JinjaInterpolation(Interpolation):
     ) -> Any:
         context = {"config": config, **additional_parameters}
 
-        for alias, equivalent in self.ALIASES.items():
+        for alias, equivalent in _ALIASES.items():
             if alias in context:
                 # This is unexpected. We could ignore or log a warning, but failing loudly should result in fewer surprises
                 raise ValueError(
@@ -84,9 +102,10 @@ class JinjaInterpolation(Interpolation):
                     return self._literal_eval(result, valid_types)
             else:
                 # If input is not a string, return it as is
-                raise Exception(f"Expected a string. got {input_str}")
+                raise Exception(f"Expected a string, got {input_str}")
         except UndefinedError:
             pass
+
         # If result is empty or resulted in an undefined error, evaluate and return the default string
         return self._literal_eval(self._eval(default, context), valid_types)
 
@@ -101,13 +120,29 @@ class JinjaInterpolation(Interpolation):
 
     def _eval(self, s: Optional[str], context: Mapping[str, Any]) -> Optional[str]:
         try:
-            ast = self._environment.parse(s)  # type: ignore # parse is able to handle None
-            undeclared = meta.find_undeclared_variables(ast)
+            undeclared = self._find_undeclared_variables(s)
             undeclared_not_in_context = {var for var in undeclared if var not in context}
             if undeclared_not_in_context:
-                raise ValueError(f"Jinja macro has undeclared variables: {undeclared_not_in_context}. Context: {context}")
-            return self._environment.from_string(s).render(context)  # type: ignore # from_string is able to handle None
+                raise ValueError(
+                    f"Jinja macro has undeclared variables: {undeclared_not_in_context}. Context: {context}"
+                )
+            return self._compile(s).render(context)  # type: ignore # from_string is able to handle None
         except TypeError:
             # The string is a static value, not a jinja template
             # It can be returned as is
             return s
+
+    @cache
+    def _find_undeclared_variables(self, s: Optional[str]) -> Set[str]:
+        """
+        Find undeclared variables and cache them
+        """
+        ast = _ENVIRONMENT.parse(s)  # type: ignore # parse is able to handle None
+        return meta.find_undeclared_variables(ast)
+
+    @cache
+    def _compile(self, s: str) -> Template:
+        """
+        We must cache the Jinja Template ourselves because we're using `from_string` instead of a template loader
+        """
+        return _ENVIRONMENT.from_string(s)