airbyte-cdk 0.0.0.dev0__py3-none-any.whl

airbyte_cdk/test/utils/reading.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+from typing import Any, List, Mapping, Optional
+
+from airbyte_cdk import AbstractSource
+from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode
+from airbyte_cdk.test.catalog_builder import CatalogBuilder
+from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read
+
+
+def catalog(stream_name: str, sync_mode: SyncMode) -> ConfiguredAirbyteCatalog:
+    """Create a catalog with a single stream."""
+    return CatalogBuilder().with_stream(stream_name, sync_mode).build()
+
+
+def read_records(
+    source: AbstractSource,
+    config: Mapping[str, Any],
+    stream_name: str,
+    sync_mode: SyncMode,
+    state: Optional[List[AirbyteStateMessage]] = None,
+    expecting_exception: bool = False,
+) -> EntrypointOutput:
+    """Read records from a stream."""
+    _catalog = catalog(stream_name, sync_mode)
+    return read(source, config, _catalog, state, expecting_exception)

airbyte_cdk/utils/__init__.py

@@ -0,0 +1,10 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from .is_cloud_environment import is_cloud_environment
+from .print_buffer import PrintBuffer
+from .schema_inferrer import SchemaInferrer
+from .traced_exception import AirbyteTracedException
+
+__all__ = ["AirbyteTracedException", "SchemaInferrer", "is_cloud_environment", "PrintBuffer"]

airbyte_cdk/utils/airbyte_secrets_utils.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, List, Mapping
+
+import dpath
+
+
+def get_secret_paths(spec: Mapping[str, Any]) -> List[List[str]]:
+    paths = []
+
+    def traverse_schema(schema_item: Any, path: List[str]) -> None:
+        """
+        schema_item can be any property or value in the originally input jsonschema, depending on how far down the recursion stack we go
+        path is the path to that schema item in the original input
+        for example if we have the input {'password': {'type': 'string', 'airbyte_secret': True}} then the arguments will evolve
+        as follows:
+        schema_item=<whole_object>, path=[]
+        schema_item={'type': 'string', 'airbyte_secret': True}, path=['password']
+        schema_item='string', path=['password', 'type']
+        schema_item=True, path=['password', 'airbyte_secret']
+        """
+        if isinstance(schema_item, dict):
+            for k, v in schema_item.items():
+                traverse_schema(v, [*path, k])
+        elif isinstance(schema_item, list):
+            for i in schema_item:
+                traverse_schema(i, path)
+        else:
+            if path[-1] == "airbyte_secret" and schema_item is True:
+                filtered_path = [p for p in path[:-1] if p not in ["properties", "oneOf"]]
+                paths.append(filtered_path)
+
+    traverse_schema(spec, [])
+    return paths
+
+
+def get_secrets(
+    connection_specification: Mapping[str, Any], config: Mapping[str, Any]
+) -> List[Any]:
+    """
+    Get a list of secret values from the source config based on the source specification
+    :type connection_specification: the connection_specification field of an AirbyteSpecification i.e the JSONSchema definition
+    """
+    secret_paths = get_secret_paths(connection_specification.get("properties", {}))
+    result = []
+    for path in secret_paths:
+        try:
+            result.append(dpath.get(config, path))  # type: ignore  # dpath expect MutableMapping but doesn't need it
+        except KeyError:
+            # Since we try to get paths to all known secrets in the spec, in the case of oneOfs, some secret fields may not be present
+            # In that case, a KeyError is thrown. This is expected behavior.
+            pass
+    return result
+
+
+__SECRETS_FROM_CONFIG: List[str] = []
+
+
+def update_secrets(secrets: List[str]) -> None:
+    """Update the list of secrets to be replaced"""
+    global __SECRETS_FROM_CONFIG
+    __SECRETS_FROM_CONFIG = secrets
+
+
+def add_to_secrets(secret: str) -> None:
+    """Add to the list of secrets to be replaced"""
+    global __SECRETS_FROM_CONFIG
+    __SECRETS_FROM_CONFIG.append(secret)
+
+
+def filter_secrets(string: str) -> str:
+    """Filter secrets from a string by replacing them with ****"""
+    # TODO this should perform a maximal match for each secret. if "x" and "xk" are both secret values, and this method is called twice on
+    #  the input "xk", then depending on call order it might only obfuscate "*k". This is a bug.
+    for secret in __SECRETS_FROM_CONFIG:
+        if secret:
+            string = string.replace(str(secret), "****")
+    return string

airbyte_cdk/utils/analytics_message.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import time
+from typing import Any, Optional
+
+from airbyte_cdk.models import (
+    AirbyteAnalyticsTraceMessage,
+    AirbyteMessage,
+    AirbyteTraceMessage,
+    TraceType,
+    Type,
+)
+
+
+def create_analytics_message(type: str, value: Optional[Any]) -> AirbyteMessage:
+    return AirbyteMessage(
+        type=Type.TRACE,
+        trace=AirbyteTraceMessage(
+            type=TraceType.ANALYTICS,
+            emitted_at=time.time() * 1000,
+            analytics=AirbyteAnalyticsTraceMessage(
+                type=type, value=str(value) if value is not None else None
+            ),
+        ),
+    )

airbyte_cdk/utils/constants.py

@@ -0,0 +1,5 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH"

airbyte_cdk/utils/datetime_format_inferrer.py

@@ -0,0 +1,94 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Dict, Optional
+
+from airbyte_cdk.models import AirbyteRecordMessage
+from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser
+
+
+class DatetimeFormatInferrer:
+    """
+    This class is used to detect toplevel fields in records that might be datetime values, along with the used format.
+    """
+
+    def __init__(self) -> None:
+        self._parser = DatetimeParser()
+        self._datetime_candidates: Optional[Dict[str, str]] = None
+        self._formats = [
+            "%Y-%m-%d",
+            "%Y-%m-%d %H:%M:%S",
+            "%Y-%m-%dT%H:%M:%SZ",
+            "%Y-%m-%dT%H:%M:%S.%fZ",
+            "%Y-%m-%d %H:%M:%S.%f%z",
+            "%Y-%m-%dT%H:%M:%S.%f%z",
+            "%s",
+            "%ms",
+            "%d/%m/%Y %H:%M",
+            "%Y-%m",
+            "%d-%m-%Y",
+        ]
+        self._timestamp_heuristic_ranges = [
+            range(1_000_000_000, 2_000_000_000),
+            range(1_000_000_000_000, 2_000_000_000_000),
+        ]
+
+    def _can_be_datetime(self, value: Any) -> bool:
+        """Checks if the value can be a datetime.
+        This is the case if the value is a string or an integer between 1_000_000_000 and 2_000_000_000 for seconds
+        or between 1_000_000_000_000 and 2_000_000_000_000 for milliseconds.
+        This is separate from the format check for performance reasons"""
+        if isinstance(value, (str, int)):
+            try:
+                value_as_int = int(value)
+                for timestamp_range in self._timestamp_heuristic_ranges:
+                    if value_as_int in timestamp_range:
+                        return True
+            except ValueError:
+                # given that it's not parsable as an int, it can represent a datetime with one of the self._formats
+                return True
+        return False
+
+    def _matches_format(self, value: Any, format: str) -> bool:
+        """Checks if the value matches the format"""
+        try:
+            self._parser.parse(value, format)
+            return True
+        except ValueError:
+            return False
+
+    def _initialize(self, record: AirbyteRecordMessage) -> None:
+        """Initializes the internal state of the class"""
+        self._datetime_candidates = {}
+        for field_name, field_value in record.data.items():
+            if not self._can_be_datetime(field_value):
+                continue
+            for format in self._formats:
+                if self._matches_format(field_value, format):
+                    self._datetime_candidates[field_name] = format
+                    break
+
+    def _validate(self, record: AirbyteRecordMessage) -> None:
+        """Validates that the record is consistent with the inferred datetime formats"""
+        if self._datetime_candidates:
+            for candidate_field_name in list(self._datetime_candidates.keys()):
+                candidate_field_format = self._datetime_candidates[candidate_field_name]
+                current_value = record.data.get(candidate_field_name, None)
+                if (
+                    current_value is None
+                    or not self._can_be_datetime(current_value)
+                    or not self._matches_format(current_value, candidate_field_format)
+                ):
+                    self._datetime_candidates.pop(candidate_field_name)
+
+    def accumulate(self, record: AirbyteRecordMessage) -> None:
+        """Analyzes the record and updates the internal state of candidate datetime fields"""
+        self._initialize(record) if self._datetime_candidates is None else self._validate(record)
+
+    def get_inferred_datetime_formats(self) -> Dict[str, str]:
+        """
+        Returns the list of candidate datetime fields - the keys are the field names and the values are the inferred datetime formats.
+        For these fields the format was consistent across all visited records.
+        """
+        return self._datetime_candidates or {}