airbyte-cdk 0.72.1__py3-none-any.whl → 6.13.1.dev4106__py3-none-any.whl

airbyte_cdk/sources/declarative/interpolation/filters.py

@@ -3,6 +3,8 @@
 #
 import base64
 import hashlib
+import json
+import re
 from typing import Any, Optional
 
 
@@ -90,5 +92,29 @@ def base64decode(value: str) -> str:
     return base64.b64decode(value.encode("utf-8")).decode()
 
 
-_filters_list = [hash, base64encode, base64decode]
+def string(value: Any) -> str:
+    """
+    Converts the input value to a string.
+    If the value is already a string, it is returned as is.
+    Otherwise, the value is interpreted as a json object and wrapped in triple-quotes so it's evalued as a string by the JinjaInterpolation
+    :param value: the value to convert to a string
+    :return: string representation of the input value
+    """
+    if isinstance(value, str):
+        return value
+    ret = f'"""{json.dumps(value)}"""'
+    return ret
+
+
+def regex_search(value: str, regex: str) -> str:
+    """
+    Match a regular expression against a string and return the first match group if it exists.
+    """
+    match = re.search(regex, value)
+    if match and len(match.groups()) > 0:
+        return match.group(1)
+    return ""
+
+
+_filters_list = [hash, base64encode, base64decode, string, regex_search]
 filters = {f.__name__: f for f in _filters_list}

airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py

@@ -6,9 +6,23 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Final, List, Mapping
 
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
-FALSE_VALUES: Final[List[Any]] = ["False", "false", "{}", "[]", "()", "", "0", "0.0", {}, False, [], (), set()]
+FALSE_VALUES: Final[List[Any]] = [
+    "False",
+    "false",
+    "{}",
+    "[]",
+    "()",
+    "",
+    "0",
+    "0.0",
+    {},
+    False,
+    [],
+    (),
+    set(),
+]
 
 
 @dataclass
@@ -23,12 +37,12 @@ class InterpolatedBoolean:
     condition: str
     parameters: InitVar[Mapping[str, Any]]
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._default = "False"
         self._interpolation = JinjaInterpolation()
         self._parameters = parameters
 
-    def eval(self, config: Config, **additional_parameters):
+    def eval(self, config: Config, **additional_parameters: Any) -> bool:
         """
         Interpolates the predicate condition string using the config and other optional arguments passed as parameter.
 
@@ -40,7 +54,11 @@ class InterpolatedBoolean:
             return self.condition
         else:
             evaluated = self._interpolation.eval(
-                self.condition, config, self._default, parameters=self._parameters, **additional_parameters
+                self.condition,
+                config,
+                self._default,
+                parameters=self._parameters,
+                **additional_parameters,
             )
             if evaluated in FALSE_VALUES:
                 return False

airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py

@@ -4,10 +4,10 @@
 
 
 from dataclasses import InitVar, dataclass
-from typing import Any, Mapping, Optional
+from typing import Any, Dict, Mapping, Optional
 
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -22,31 +22,35 @@ class InterpolatedMapping:
     mapping: Mapping[str, str]
     parameters: InitVar[Mapping[str, Any]]
 
-    def __post_init__(self, parameters: Optional[Mapping[str, Any]]):
+    def __post_init__(self, parameters: Optional[Mapping[str, Any]]) -> None:
         self._interpolation = JinjaInterpolation()
         self._parameters = parameters
 
-    def eval(self, config: Config, **additional_parameters):
+    def eval(self, config: Config, **additional_parameters: Any) -> Dict[str, Any]:
         """
         Wrapper around a Mapping[str, str] that allows for both keys and values to be interpolated.
 
         :param config: The user-provided configuration as specified by the source's spec
         :param additional_parameters: Optional parameters used for interpolation
-        :return: The interpolated string
+        :return: The interpolated mapping
         """
         valid_key_types = additional_parameters.pop("valid_key_types", (str,))
         valid_value_types = additional_parameters.pop("valid_value_types", None)
         return {
             self._interpolation.eval(
-                name, config, valid_types=valid_key_types, parameters=self._parameters, **additional_parameters
+                name,
+                config,
+                valid_types=valid_key_types,
+                parameters=self._parameters,
+                **additional_parameters,
             ): self._eval(value, config, valid_types=valid_value_types, **additional_parameters)
             for name, value in self.mapping.items()
         }
 
-    def _eval(self, value, config, **kwargs):
+    def _eval(self, value: str, config: Config, **kwargs: Any) -> Any:
         # The values in self._mapping can be of Any type
         # We only want to interpolate them if they are strings
-        if type(value) == str:
+        if isinstance(value, str):
             return self._interpolation.eval(value, config, parameters=self._parameters, **kwargs)
         else:
             return value

airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py

@@ -7,9 +7,11 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
-NestedMappingEntry = Union[dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None]
+NestedMappingEntry = Union[
+    dict[str, "NestedMapping"], list["NestedMapping"], str, int, float, bool, None
+]
 NestedMapping = Union[dict[str, NestedMappingEntry], str]
 
 
@@ -25,19 +27,24 @@ class InterpolatedNestedMapping:
     mapping: NestedMapping
     parameters: InitVar[Mapping[str, Any]]
 
-    def __post_init__(self, parameters: Optional[Mapping[str, Any]]):
+    def __post_init__(self, parameters: Optional[Mapping[str, Any]]) -> None:
         self._interpolation = JinjaInterpolation()
         self._parameters = parameters
 
-    def eval(self, config: Config, **additional_parameters):
+    def eval(self, config: Config, **additional_parameters: Any) -> Any:
         return self._eval(self.mapping, config, **additional_parameters)
 
-    def _eval(self, value, config, **kwargs):
+    def _eval(
+        self, value: Union[NestedMapping, NestedMappingEntry], config: Config, **kwargs: Any
+    ) -> Any:
         # Recursively interpolate dictionaries and lists
         if isinstance(value, str):
             return self._interpolation.eval(value, config, parameters=self._parameters, **kwargs)
         elif isinstance(value, dict):
-            interpolated_dict = {self._eval(k, config, **kwargs): self._eval(v, config, **kwargs) for k, v in value.items()}
+            interpolated_dict = {
+                self._eval(k, config, **kwargs): self._eval(v, config, **kwargs)
+                for k, v in value.items()
+            }
             return {k: v for k, v in interpolated_dict.items() if v is not None}
         elif isinstance(value, list):
             return [self._eval(v, config, **kwargs) for v in value]

airbyte_cdk/sources/declarative/interpolation/interpolated_string.py

@@ -6,7 +6,7 @@ from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Union
 
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
 
 @dataclass
@@ -24,12 +24,15 @@ class InterpolatedString:
     parameters: InitVar[Mapping[str, Any]]
     default: Optional[str] = None
 
-    def __post_init__(self, parameters: Mapping[str, Any]):
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.default = self.default or self.string
         self._interpolation = JinjaInterpolation()
         self._parameters = parameters
+        # indicates whether passed string is just a plain string, not Jinja template
+        # This allows for optimization, but we do not know it yet at this stage
+        self._is_plain_string = None
 
-    def eval(self, config: Config, **kwargs):
+    def eval(self, config: Config, **kwargs: Any) -> Any:
         """
         Interpolates the input string using the config and other optional arguments passed as parameter.
 
@@ -37,9 +40,21 @@ class InterpolatedString:
         :param kwargs: Optional parameters used for interpolation
         :return: The interpolated string
         """
-        return self._interpolation.eval(self.string, config, self.default, parameters=self._parameters, **kwargs)
+        if self._is_plain_string:
+            return self.string
+        if self._is_plain_string is None:
+            # Let's check whether output from evaluation is the same as input.
+            # This indicates occurrence of a plain string, not a template and we can skip Jinja in subsequent runs.
+            evaluated = self._interpolation.eval(
+                self.string, config, self.default, parameters=self._parameters, **kwargs
+            )
+            self._is_plain_string = self.string == evaluated
+            return evaluated
+        return self._interpolation.eval(
+            self.string, config, self.default, parameters=self._parameters, **kwargs
+        )
 
-    def __eq__(self, other):
+    def __eq__(self, other: Any) -> bool:
         if not isinstance(other, InterpolatedString):
             return False
         return self.string == other.string and self.default == other.default
@@ -50,7 +65,7 @@ class InterpolatedString:
         string_or_interpolated: Union["InterpolatedString", str],
         *,
         parameters: Mapping[str, Any],
-    ):
+    ) -> "InterpolatedString":
         """
         Helper function to obtain an InterpolatedString from either a raw string or an InterpolatedString.
 

airbyte_cdk/sources/declarative/interpolation/interpolation.py

@@ -3,9 +3,9 @@
 #
 
 from abc import ABC, abstractmethod
-from typing import Optional
+from typing import Any, Optional
 
-from airbyte_cdk.sources.declarative.types import Config
+from airbyte_cdk.sources.types import Config
 
 
 class Interpolation(ABC):
@@ -14,7 +14,13 @@ class Interpolation(ABC):
     """
 
     @abstractmethod
-    def eval(self, input_str: str, config: Config, default: Optional[str] = None, **additional_options):
+    def eval(
+        self,
+        input_str: str,
+        config: Config,
+        default: Optional[str] = None,
+        **additional_options: Any,
+    ) -> Any:
         """
         Interpolates the input string using the config, and additional options passed as parameter.
 

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -3,15 +3,59 @@
 #
 
 import ast
-from typing import Any, Mapping, Optional, Tuple, Type
+from functools import cache
+from typing import Any, Mapping, Optional, Set, Tuple, Type
+
+from jinja2 import meta
+from jinja2.environment import Template
+from jinja2.exceptions import UndefinedError
+from jinja2.sandbox import SandboxedEnvironment
 
 from airbyte_cdk.sources.declarative.interpolation.filters import filters
 from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
 from airbyte_cdk.sources.declarative.interpolation.macros import macros
-from airbyte_cdk.sources.declarative.types import Config
-from jinja2 import meta
-from jinja2.exceptions import UndefinedError
-from jinja2.sandbox import Environment
+from airbyte_cdk.sources.types import Config
+
+
+class StreamPartitionAccessEnvironment(SandboxedEnvironment):
+    """
+    Currently, source-jira is setting an attribute to StreamSlice specific to its use case which because of the PerPartitionCursor is set to
+    StreamSlice._partition but not exposed through StreamSlice.partition. This is a patch to still allow source-jira to have access to this
+    parameter
+    """
+
+    def is_safe_attribute(self, obj: Any, attr: str, value: Any) -> bool:
+        if attr in ["_partition"]:
+            return True
+        return super().is_safe_attribute(obj, attr, value)  # type: ignore  # for some reason, mypy says 'Returning Any from function declared to return "bool"'
+
+
+# These aliases are used to deprecate existing keywords without breaking all existing connectors.
+_ALIASES = {
+    "stream_interval": "stream_slice",  # Use stream_interval to access incremental_sync values
+    "stream_partition": "stream_slice",  # Use stream_partition to access partition router's values
+}
+
+# These extensions are not installed so they're not currently a problem,
+# but we're still explicitly removing them from the jinja context.
+# At worst, this is documentation that we do NOT want to include these extensions because of the potential security risks
+_RESTRICTED_EXTENSIONS = ["jinja2.ext.loopcontrols"]  # Adds support for break continue in loops
+
+# By default, these Python builtin functions are available in the Jinja context.
+# We explicitly remove them because of the potential security risk.
+# Please add a unit test to test_jinja.py when adding a restriction.
+_RESTRICTED_BUILTIN_FUNCTIONS = [
+    "range"
+]  # The range function can cause very expensive computations
+
+_ENVIRONMENT = StreamPartitionAccessEnvironment()
+_ENVIRONMENT.filters.update(**filters)
+_ENVIRONMENT.globals.update(**macros)
+
+for extension in _RESTRICTED_EXTENSIONS:
+    _ENVIRONMENT.extensions.pop(extension, None)
+for builtin in _RESTRICTED_BUILTIN_FUNCTIONS:
+    _ENVIRONMENT.globals.pop(builtin, None)
 
 
 class JinjaInterpolation(Interpolation):
@@ -32,32 +76,6 @@ class JinjaInterpolation(Interpolation):
     Additional information on jinja templating can be found at https://jinja.palletsprojects.com/en/3.1.x/templates/#
     """
 
-    # These aliases are used to deprecate existing keywords without breaking all existing connectors.
-    ALIASES = {
-        "stream_interval": "stream_slice",  # Use stream_interval to access incremental_sync values
-        "stream_partition": "stream_slice",  # Use stream_partition to access partition router's values
-    }
-
-    # These extensions are not installed so they're not currently a problem,
-    # but we're still explicitely removing them from the jinja context.
-    # At worst, this is documentation that we do NOT want to include these extensions because of the potential security risks
-    RESTRICTED_EXTENSIONS = ["jinja2.ext.loopcontrols"]  # Adds support for break continue in loops
-
-    # By default, these Python builtin functions are available in the Jinja context.
-    # We explicitely remove them because of the potential security risk.
-    # Please add a unit test to test_jinja.py when adding a restriction.
-    RESTRICTED_BUILTIN_FUNCTIONS = ["range"]  # The range function can cause very expensive computations
-
-    def __init__(self) -> None:
-        self._environment = Environment()
-        self._environment.filters.update(**filters)
-        self._environment.globals.update(**macros)
-
-        for extension in self.RESTRICTED_EXTENSIONS:
-            self._environment.extensions.pop(extension, None)
-        for builtin in self.RESTRICTED_BUILTIN_FUNCTIONS:
-            self._environment.globals.pop(builtin, None)
-
     def eval(
         self,
         input_str: str,
@@ -68,7 +86,7 @@ class JinjaInterpolation(Interpolation):
     ) -> Any:
         context = {"config": config, **additional_parameters}
 
-        for alias, equivalent in self.ALIASES.items():
+        for alias, equivalent in _ALIASES.items():
             if alias in context:
                 # This is unexpected. We could ignore or log a warning, but failing loudly should result in fewer surprises
                 raise ValueError(
@@ -84,9 +102,10 @@ class JinjaInterpolation(Interpolation):
                     return self._literal_eval(result, valid_types)
             else:
                 # If input is not a string, return it as is
-                raise Exception(f"Expected a string. got {input_str}")
+                raise Exception(f"Expected a string, got {input_str}")
         except UndefinedError:
             pass
+
         # If result is empty or resulted in an undefined error, evaluate and return the default string
         return self._literal_eval(self._eval(default, context), valid_types)
 
@@ -101,13 +120,29 @@ class JinjaInterpolation(Interpolation):
 
     def _eval(self, s: Optional[str], context: Mapping[str, Any]) -> Optional[str]:
         try:
-            ast = self._environment.parse(s)  # type: ignore # parse is able to handle None
-            undeclared = meta.find_undeclared_variables(ast)
+            undeclared = self._find_undeclared_variables(s)
             undeclared_not_in_context = {var for var in undeclared if var not in context}
             if undeclared_not_in_context:
-                raise ValueError(f"Jinja macro has undeclared variables: {undeclared_not_in_context}. Context: {context}")
-            return self._environment.from_string(s).render(context)  # type: ignore # from_string is able to handle None
+                raise ValueError(
+                    f"Jinja macro has undeclared variables: {undeclared_not_in_context}. Context: {context}"
+                )
+            return self._compile(s).render(context)  # type: ignore # from_string is able to handle None
         except TypeError:
             # The string is a static value, not a jinja template
             # It can be returned as is
             return s
+
+    @cache
+    def _find_undeclared_variables(self, s: Optional[str]) -> Set[str]:
+        """
+        Find undeclared variables and cache them
+        """
+        ast = _ENVIRONMENT.parse(s)  # type: ignore # parse is able to handle None
+        return meta.find_undeclared_variables(ast)
+
+    @cache
+    def _compile(self, s: str) -> Template:
+        """
+        We must cache the Jinja Template ourselves because we're using `from_string` instead of a template loader
+        """
+        return _ENVIRONMENT.from_string(s)

airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -4,9 +4,11 @@
 
 import builtins
 import datetime
-import numbers
-from typing import Union
+import typing
+from typing import Optional, Union
 
+import isodate
+import pytz
 from dateutil import parser
 from isodate import parse_duration
 
@@ -15,7 +17,7 @@ This file contains macros that can be evaluated by a `JinjaInterpolation` object
 """
 
 
-def now_utc():
+def now_utc() -> datetime.datetime:
     """
     Current local date and time in UTC timezone
 
@@ -25,7 +27,7 @@ def now_utc():
     return datetime.datetime.now(datetime.timezone.utc)
 
 
-def today_utc():
+def today_utc() -> datetime.date:
     """
     Current date in UTC timezone
 
@@ -35,7 +37,17 @@ def today_utc():
     return datetime.datetime.now(datetime.timezone.utc).date()
 
 
-def timestamp(dt: Union[numbers.Number, str]):
+def today_with_timezone(timezone: str) -> datetime.date:
+    """
+    Current date in custom timezone
+
+    :param timezone: timezone expressed as IANA keys format. Example: "Pacific/Tarawa"
+    :return:
+    """
+    return datetime.datetime.now(tz=pytz.timezone(timezone)).date()
+
+
+def timestamp(dt: Union[float, str]) -> Union[int, float]:
     """
     Converts a number or a string to a timestamp
 
@@ -48,21 +60,21 @@ def timestamp(dt: Union[numbers.Number, str]):
     :param dt: datetime to convert to timestamp
     :return: unix timestamp
     """
-    if isinstance(dt, numbers.Number):
+    if isinstance(dt, (int, float)):
         return int(dt)
     else:
-        return _str_to_datetime(dt).astimezone(datetime.timezone.utc).timestamp()
+        return _str_to_datetime(dt).astimezone(pytz.utc).timestamp()
 
 
 def _str_to_datetime(s: str) -> datetime.datetime:
     parsed_date = parser.isoparse(s)
     if not parsed_date.tzinfo:
         # Assume UTC if the input does not contain a timezone
-        parsed_date = parsed_date.replace(tzinfo=datetime.timezone.utc)
-    return parsed_date.astimezone(datetime.timezone.utc)
+        parsed_date = parsed_date.replace(tzinfo=pytz.utc)
+    return parsed_date.astimezone(pytz.utc)
 
 
-def max(*args):
+def max(*args: typing.Any) -> typing.Any:
     """
     Returns biggest object of an iterable, or two or more arguments.
 
@@ -82,6 +94,26 @@ def max(*args):
     return builtins.max(*args)
 
 
+def min(*args: typing.Any) -> typing.Any:
+    """
+    Returns smallest object of an iterable, or two or more arguments.
+
+    min(iterable, *[, default=obj, key=func]) -> value
+    min(arg1, arg2, *args, *[, key=func]) -> value
+
+    Usage:
+    `"{{ min(2,3) }}"
+
+    With a single iterable argument, return its smallest item. The
+    default keyword-only argument specifies an object to return if
+    the provided iterable is empty.
+    With two or more arguments, return the smallest argument.
+    :param args: args to compare
+    :return: smallest argument
+    """
+    return builtins.min(*args)
+
+
 def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
     """
     Returns datetime of now() + num_days
@@ -92,10 +124,12 @@ def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
     :param num_days: number of days to add to current date time
     :return: datetime formatted as RFC3339
     """
-    return (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)).strftime(format)
+    return (
+        datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)
+    ).strftime(format)
 
 
-def duration(datestring: str) -> datetime.timedelta:
+def duration(datestring: str) -> Union[datetime.timedelta, isodate.Duration]:
     """
     Converts ISO8601 duration to datetime.timedelta
 
@@ -105,17 +139,38 @@ def duration(datestring: str) -> datetime.timedelta:
     return parse_duration(datestring)
 
 
-def format_datetime(dt: Union[str, datetime.datetime], format: str) -> str:
+def format_datetime(
+    dt: Union[str, datetime.datetime], format: str, input_format: Optional[str] = None
+) -> str:
     """
     Converts datetime to another format
 
     Usage:
     `"{{ format_datetime(config.start_date, '%Y-%m-%d') }}"`
+
+    CPython Datetime package has known bug with `stfrtime` method: '%s' formatting uses locale timezone
+    https://github.com/python/cpython/issues/77169
+    https://github.com/python/cpython/issues/56959
     """
     if isinstance(dt, datetime.datetime):
         return dt.strftime(format)
-    return _str_to_datetime(dt).strftime(format)
-
-
-_macros_list = [now_utc, today_utc, timestamp, max, day_delta, duration, format_datetime]
+    dt_datetime = (
+        datetime.datetime.strptime(dt, input_format) if input_format else _str_to_datetime(dt)
+    )
+    if format == "%s":
+        return str(int(dt_datetime.timestamp()))
+    return dt_datetime.strftime(format)
+
+
+_macros_list = [
+    now_utc,
+    today_utc,
+    timestamp,
+    max,
+    min,
+    day_delta,
+    duration,
+    format_datetime,
+    today_with_timezone,
+]
 macros = {f.__name__: f for f in _macros_list}