airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/sources/declarative/interpolation/jinja.py

@@ -0,0 +1,161 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import ast
+from functools import cache
+from typing import Any, Mapping, Optional, Set, Tuple, Type
+
+from jinja2 import meta
+from jinja2.environment import Template
+from jinja2.exceptions import UndefinedError
+from jinja2.sandbox import SandboxedEnvironment
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.declarative.interpolation.filters import filters
+from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation
+from airbyte_cdk.sources.declarative.interpolation.macros import macros
+from airbyte_cdk.sources.types import Config
+from airbyte_cdk.utils import AirbyteTracedException
+
+
+class StreamPartitionAccessEnvironment(SandboxedEnvironment):
+    """
+    Currently, source-jira is setting an attribute to StreamSlice specific to its use case which because of the PerPartitionCursor is set to
+    StreamSlice._partition but not exposed through StreamSlice.partition. This is a patch to still allow source-jira to have access to this
+    parameter
+    """
+
+    def is_safe_attribute(self, obj: Any, attr: str, value: Any) -> bool:
+        if attr in ["_partition"]:
+            return True
+        return super().is_safe_attribute(obj, attr, value)  # type: ignore  # for some reason, mypy says 'Returning Any from function declared to return "bool"'
+
+
+# These aliases are used to deprecate existing keywords without breaking all existing connectors.
+_ALIASES = {
+    "stream_interval": "stream_slice",  # Use stream_interval to access incremental_sync values
+    "stream_partition": "stream_slice",  # Use stream_partition to access partition router's values
+}
+
+_UNSUPPORTED_INTERPOLATION_VARIABLES: Mapping[str, str] = {
+    "stream_state": "`stream_state` is no longer supported for interpolation. We recommend using `stream_interval` instead. Please reference the CDK Migration Guide for more information.",
+}
+
+# These extensions are not installed so they're not currently a problem,
+# but we're still explicitly removing them from the jinja context.
+# At worst, this is documentation that we do NOT want to include these extensions because of the potential security risks
+_RESTRICTED_EXTENSIONS = ["jinja2.ext.loopcontrols"]  # Adds support for break continue in loops
+
+# By default, these Python builtin functions are available in the Jinja context.
+# We explicitly remove them because of the potential security risk.
+# Please add a unit test to test_jinja.py when adding a restriction.
+_RESTRICTED_BUILTIN_FUNCTIONS = [
+    "range"
+]  # The range function can cause very expensive computations
+
+_ENVIRONMENT = StreamPartitionAccessEnvironment()
+_ENVIRONMENT.filters.update(**filters)
+_ENVIRONMENT.globals.update(**macros)
+
+for extension in _RESTRICTED_EXTENSIONS:
+    _ENVIRONMENT.extensions.pop(extension, None)
+for builtin in _RESTRICTED_BUILTIN_FUNCTIONS:
+    _ENVIRONMENT.globals.pop(builtin, None)
+
+
+class JinjaInterpolation(Interpolation):
+    """
+    Interpolation strategy using the Jinja2 template engine.
+
+    If the input string is a raw string, the interpolated string will be the same.
+    `eval("hello world") -> "hello world"`
+
+    The engine will evaluate the content passed within {{}}, interpolating the keys from the config and context-specific arguments.
+    `eval("hello {{ name }}", name="airbyte") -> "hello airbyte")`
+    `eval("hello {{ config.name }}", config={"name": "airbyte"}) -> "hello airbyte")`
+
+    In additional to passing additional values through the kwargs argument, macros can be called from within the string interpolation.
+    For example,
+    "{{ max(2, 3) }}" will return 3
+
+    Additional information on jinja templating can be found at https://jinja.palletsprojects.com/en/3.1.x/templates/#
+    """
+
+    def eval(
+        self,
+        input_str: str,
+        config: Config,
+        default: Optional[str] = None,
+        valid_types: Optional[Tuple[Type[Any]]] = None,
+        **additional_parameters: Any,
+    ) -> Any:
+        context = {"config": config, **additional_parameters}
+
+        for alias, equivalent in _ALIASES.items():
+            if alias in context:
+                # This is unexpected. We could ignore or log a warning, but failing loudly should result in fewer surprises
+                raise ValueError(
+                    f"Found reserved keyword {alias} in interpolation context. This is unexpected and indicative of a bug in the CDK."
+                )
+            elif equivalent in context:
+                context[alias] = context[equivalent]
+
+        for variable_name in _UNSUPPORTED_INTERPOLATION_VARIABLES:
+            if variable_name in input_str:
+                raise AirbyteTracedException(
+                    message=_UNSUPPORTED_INTERPOLATION_VARIABLES[variable_name],
+                    internal_message=_UNSUPPORTED_INTERPOLATION_VARIABLES[variable_name],
+                    failure_type=FailureType.config_error,
+                )
+        try:
+            if isinstance(input_str, str):
+                result = self._eval(input_str, context)
+                if result:
+                    return self._literal_eval(result, valid_types)
+            else:
+                # If input is not a string, return it as is
+                raise Exception(f"Expected a string, got {input_str}")
+        except UndefinedError:
+            pass
+
+        # If result is empty or resulted in an undefined error, evaluate and return the default string
+        return self._literal_eval(self._eval(default, context), valid_types)
+
+    def _literal_eval(self, result: Optional[str], valid_types: Optional[Tuple[Type[Any]]]) -> Any:
+        try:
+            evaluated = ast.literal_eval(result)  # type: ignore # literal_eval is able to handle None
+        except (ValueError, SyntaxError):
+            return result
+        if not valid_types or (valid_types and isinstance(evaluated, valid_types)):
+            return evaluated
+        return result
+
+    def _eval(self, s: Optional[str], context: Mapping[str, Any]) -> Optional[str]:
+        try:
+            undeclared = self._find_undeclared_variables(s)
+            undeclared_not_in_context = {var for var in undeclared if var not in context}
+            if undeclared_not_in_context:
+                raise ValueError(
+                    f"Jinja macro has undeclared variables: {undeclared_not_in_context}. Context: {context}"
+                )
+            return self._compile(s).render(context)  # type: ignore # from_string is able to handle None
+        except TypeError:
+            # The string is a static value, not a jinja template
+            # It can be returned as is
+            return s
+
+    @cache
+    def _find_undeclared_variables(self, s: Optional[str]) -> Set[str]:
+        """
+        Find undeclared variables and cache them
+        """
+        ast = _ENVIRONMENT.parse(s)  # type: ignore # parse is able to handle None
+        return meta.find_undeclared_variables(ast)
+
+    @cache
+    def _compile(self, s: str) -> Template:
+        """
+        We must cache the Jinja Template ourselves because we're using `from_string` instead of a template loader
+        """
+        return _ENVIRONMENT.from_string(s)

airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -0,0 +1,191 @@
+#
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+#
+
+import builtins
+import datetime
+import typing
+from typing import Optional, Union
+
+import isodate
+import pytz
+from dateutil import parser
+from isodate import parse_duration
+
+from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
+
+"""
+This file contains macros that can be evaluated by a `JinjaInterpolation` object
+"""
+
+
+def now_utc() -> datetime.datetime:
+    """
+    Current local date and time in UTC timezone
+
+    Usage:
+    `"{{ now_utc() }}"`
+    """
+    return datetime.datetime.now(datetime.timezone.utc)
+
+
+def today_utc() -> datetime.date:
+    """
+    Current date in UTC timezone
+
+    Usage:
+    `"{{ today_utc() }}"`
+    """
+    return datetime.datetime.now(datetime.timezone.utc).date()
+
+
+def today_with_timezone(timezone: str) -> datetime.date:
+    """
+    Current date in custom timezone
+
+    :param timezone: timezone expressed as IANA keys format. Example: "Pacific/Tarawa"
+    :return:
+    """
+    return datetime.datetime.now(tz=pytz.timezone(timezone)).date()
+
+
+def timestamp(dt: Union[float, str]) -> Union[int, float]:
+    """
+    Converts a number or a string to a timestamp
+
+    If dt is a number, then convert to an int
+    If dt is a string, then parse it using dateutil.parser
+
+    Usage:
+    `"{{ timestamp(1658505815.223235) }}"
+
+    :param dt: datetime to convert to timestamp
+    :return: unix timestamp
+    """
+    if isinstance(dt, (int, float)):
+        return int(dt)
+    else:
+        return str_to_datetime(dt).astimezone(pytz.utc).timestamp()
+
+
+def str_to_datetime(s: str) -> datetime.datetime:
+    """
+    Converts a string to a datetime object with UTC timezone
+
+    If the input string does not contain timezone information, UTC is assumed.
+    Supports both basic date strings like "2022-01-14" and datetime strings with optional timezone
+    like "2022-01-01T13:45:30+00:00".
+
+    Usage:
+    `"{{ str_to_datetime('2022-01-14') }}"`
+
+    :param s: string to parse as datetime
+    :return: datetime object in UTC timezone
+    """
+
+    parsed_date = parser.isoparse(s)
+    if not parsed_date.tzinfo:
+        # Assume UTC if the input does not contain a timezone
+        parsed_date = parsed_date.replace(tzinfo=pytz.utc)
+    return parsed_date.astimezone(pytz.utc)
+
+
+def max(*args: typing.Any) -> typing.Any:
+    """
+    Returns biggest object of an iterable, or two or more arguments.
+
+    max(iterable, *[, default=obj, key=func]) -> value
+    max(arg1, arg2, *args, *[, key=func]) -> value
+
+    Usage:
+    `"{{ max(2,3) }}"
+
+    With a single iterable argument, return its biggest item. The
+    default keyword-only argument specifies an object to return if
+    the provided iterable is empty.
+    With two or more arguments, return the largest argument.
+    :param args: args to compare
+    :return: largest argument
+    """
+    return builtins.max(*args)
+
+
+def min(*args: typing.Any) -> typing.Any:
+    """
+    Returns smallest object of an iterable, or two or more arguments.
+
+    min(iterable, *[, default=obj, key=func]) -> value
+    min(arg1, arg2, *args, *[, key=func]) -> value
+
+    Usage:
+    `"{{ min(2,3) }}"
+
+    With a single iterable argument, return its smallest item. The
+    default keyword-only argument specifies an object to return if
+    the provided iterable is empty.
+    With two or more arguments, return the smallest argument.
+    :param args: args to compare
+    :return: smallest argument
+    """
+    return builtins.min(*args)
+
+
+def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
+    """
+    Returns datetime of now() + num_days
+
+    Usage:
+    `"{{ day_delta(25) }}"`
+
+    :param num_days: number of days to add to current date time
+    :return: datetime formatted as RFC3339
+    """
+    return (
+        datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)
+    ).strftime(format)
+
+
+def duration(datestring: str) -> Union[datetime.timedelta, isodate.Duration]:
+    """
+    Converts ISO8601 duration to datetime.timedelta
+
+    Usage:
+    `"{{ now_utc() - duration('P1D') }}"`
+    """
+    return parse_duration(datestring)
+
+
+def format_datetime(
+    dt: Union[str, datetime.datetime], format: str, input_format: Optional[str] = None
+) -> str:
+    """
+    Converts datetime to another format
+
+    Usage:
+    `"{{ format_datetime(config.start_date, '%Y-%m-%d') }}"`
+
+    CPython Datetime package has known bug with `stfrtime` method: '%s' formatting uses locale timezone
+    https://github.com/python/cpython/issues/77169
+    https://github.com/python/cpython/issues/56959
+    """
+    if isinstance(dt, datetime.datetime):
+        return dt.strftime(format)
+    dt_datetime = (
+        datetime.datetime.strptime(dt, input_format) if input_format else str_to_datetime(dt)
+    )
+    return DatetimeParser().format(dt=dt_datetime, format=format)
+
+
+_macros_list = [
+    now_utc,
+    today_utc,
+    timestamp,
+    max,
+    min,
+    day_delta,
+    duration,
+    format_datetime,
+    today_with_timezone,
+    str_to_datetime,
+]
+macros = {f.__name__: f for f in _macros_list}