hotglue-singer-sdk 1.0.2__py3-none-any.whl

hotglue_singer_sdk/helpers/_typing.py

@@ -0,0 +1,231 @@
+"""General helper functions for json typing."""
+
+import copy
+import datetime
+import logging
+from enum import Enum
+from functools import lru_cache
+from typing import Any, Dict, List, Optional, Tuple, cast
+
+import pendulum
+
+_MAX_TIMESTAMP = "9999-12-31 23:59:59.999999"
+_MAX_TIME = "23:59:59.999999"
+
+
+class DatetimeErrorTreatmentEnum(Enum):
+    """Enum for treatment options for date parsing error."""
+
+    ERROR = "error"
+    MAX = "max"
+    NULL = "null"
+
+
+def to_json_compatible(val: Any) -> Any:
+    """Return as string if datetime. JSON does not support proper datetime types.
+
+    If given a naive datetime object, pendulum automatically makes it utc
+    """
+    if isinstance(val, (datetime.datetime, pendulum.DateTime)):
+        val = pendulum.instance(val).isoformat()
+    return val
+
+
+def append_type(type_dict: dict, new_type: str) -> dict:
+    """Return a combined type definition using the 'anyOf' JSON Schema construct."""
+    result = copy.deepcopy(type_dict)
+    if "anyOf" in result:
+        if isinstance(result["anyOf"], list) and new_type not in result["anyOf"]:
+            result["anyOf"].append(new_type)
+        elif new_type != result["anyOf"]:
+            result["anyOf"] = [result["anyOf"], new_type]
+        return result
+
+    elif "type" in result:
+        if isinstance(result["type"], list) and new_type not in result["type"]:
+            result["type"].append(new_type)
+        elif new_type != result["type"]:
+            result["type"] = [result["type"], new_type]
+        return result
+
+    raise ValueError(
+        "Could not append type because the JSON schema for the dictionary "
+        f"`{type_dict}` appears to be invalid."
+    )
+
+
+def is_object_type(property_schema: dict) -> Optional[bool]:
+    """Return true if the JSON Schema type is an object or None if detection fails."""
+    if "anyOf" not in property_schema and "type" not in property_schema:
+        return None  # Could not detect data type
+    for property_type in property_schema.get("anyOf", [property_schema.get("type")]):
+        if "object" in property_type or property_type == "object":
+            return True
+    return False
+
+
+def is_datetime_type(type_dict: dict) -> bool:
+    """Return True if JSON Schema type definition is a 'date-time' type.
+
+    Also returns True if 'date-time' is nested within an 'anyOf' type Array.
+    """
+    if not type_dict:
+        raise ValueError(
+            "Could not detect type from empty type_dict. "
+            "Did you forget to define a property in the stream schema?"
+        )
+    if "anyOf" in type_dict:
+        for type_dict in type_dict["anyOf"]:
+            if is_datetime_type(type_dict):
+                return True
+        return False
+    elif "type" in type_dict:
+        return type_dict.get("format") == "date-time"
+    raise ValueError(
+        f"Could not detect type of replication key using schema '{type_dict}'"
+    )
+
+
+def get_datelike_property_type(property_schema: Dict) -> Optional[str]:
+    """Return one of 'date-time', 'time', or 'date' if property is date-like.
+
+    Otherwise return None.
+    """
+    if _is_string_with_format(property_schema):
+        return cast(str, property_schema["format"])
+    elif "anyOf" in property_schema:
+        for type_dict in property_schema["anyOf"]:
+            if _is_string_with_format(type_dict):
+                return cast(str, type_dict["format"])
+    return None
+
+
+def _is_string_with_format(type_dict):
+    if "string" in type_dict.get("type", []) and type_dict.get("format") in {
+        "date-time",
+        "time",
+        "date",
+    }:
+        return True
+
+
+def handle_invalid_timestamp_in_record(
+    record,
+    key_breadcrumb: List[str],
+    invalid_value: str,
+    datelike_typename: str,
+    ex: Exception,
+    treatment: Optional[DatetimeErrorTreatmentEnum],
+    logger: logging.Logger,
+) -> Any:
+    """Apply treatment or raise an error for invalid time values."""
+    treatment = treatment or DatetimeErrorTreatmentEnum.ERROR
+    msg = (
+        f"Could not parse value '{invalid_value}' for "
+        f"field '{':'.join(key_breadcrumb)}'."
+    )
+    if treatment == DatetimeErrorTreatmentEnum.MAX:
+        logger.warning(f"{msg}. Replacing with MAX value.\n{ex}\n")
+        return _MAX_TIMESTAMP if datelike_typename != "time" else _MAX_TIME
+
+    if treatment == DatetimeErrorTreatmentEnum.NULL:
+        logger.warning(f"{msg}. Replacing with NULL.\n{ex}\n")
+        return None
+
+    raise ValueError(msg)
+
+
+def is_string_array_type(type_dict: dict) -> bool:
+    """Return True if JSON Schema type definition is a string array."""
+    if not type_dict:
+        raise ValueError(
+            "Could not detect type from empty type_dict. "
+            "Did you forget to define a property in the stream schema?"
+        )
+
+    if "anyOf" in type_dict:
+        return any([is_string_array_type(t) for t in type_dict["anyOf"]])
+
+    if "type" not in type_dict:
+        raise ValueError(f"Could not detect type from schema '{type_dict}'")
+
+    return "array" in type_dict["type"] and bool(is_string_type(type_dict["items"]))
+
+
+def is_boolean_type(property_schema: dict) -> Optional[bool]:
+    """Return true if the JSON Schema type is a boolean or None if detection fails."""
+    if "anyOf" not in property_schema and "type" not in property_schema:
+        return None  # Could not detect data type
+    for property_type in property_schema.get("anyOf", [property_schema.get("type")]):
+        if "boolean" in property_type or property_type == "boolean":
+            return True
+    return False
+
+
+def is_string_type(property_schema: dict) -> Optional[bool]:
+    """Return true if the JSON Schema type is a boolean or None if detection fails."""
+    if "anyOf" not in property_schema and "type" not in property_schema:
+        return None  # Could not detect data type
+    for property_type in property_schema.get("anyOf", [property_schema.get("type")]):
+        if "string" in property_type or property_type == "string":
+            return True
+    return False
+
+
+@lru_cache()
+def _warn_unmapped_properties(
+    stream_name: str, property_names: Tuple[str], logger: logging.Logger
+):
+    logger.info(
+        f"Properties {property_names} were present in the '{stream_name}' stream but "
+        "not found in catalog schema. Ignoring."
+    )
+
+
+def conform_record_data_types(  # noqa: C901
+    stream_name: str, row: Dict[str, Any], schema: dict, logger: logging.Logger
+) -> Dict[str, Any]:
+    """Translate values in record dictionary to singer-compatible data types.
+
+    Any property names not found in the schema catalog will be removed, and a
+    warning will be logged exactly once per unmapped property name.
+    """
+    rec: Dict[str, Any] = {}
+    unmapped_properties: List[str] = []
+    for property_name, elem in row.items():
+        if property_name not in schema["properties"]:
+            unmapped_properties.append(property_name)
+            continue
+
+        property_schema = schema["properties"][property_name]
+        if isinstance(elem, (datetime.datetime, pendulum.DateTime)):
+            rec[property_name] = to_json_compatible(elem)
+        elif isinstance(elem, datetime.date):
+            rec[property_name] = elem.isoformat() + "T00:00:00+00:00"
+        elif isinstance(elem, datetime.timedelta):
+            epoch = datetime.datetime.utcfromtimestamp(0)
+            timedelta_from_epoch = epoch + elem
+            rec[property_name] = timedelta_from_epoch.isoformat() + "+00:00"
+        elif isinstance(elem, datetime.time):
+            rec[property_name] = str(elem)
+        elif isinstance(elem, bytes):
+            # for BIT value, treat 0 as False and anything else as True
+            bit_representation: bool
+            if is_boolean_type(property_schema):
+                bit_representation = elem != b"\x00"
+                rec[property_name] = bit_representation
+            else:
+                rec[property_name] = elem.hex()
+        elif is_boolean_type(property_schema):
+            boolean_representation: Optional[bool]
+            if elem is None:
+                boolean_representation = None
+            elif elem == 0:
+                boolean_representation = False
+            else:
+                boolean_representation = True
+            rec[property_name] = boolean_representation
+        else:
+            rec[property_name] = elem
+    _warn_unmapped_properties(stream_name, tuple(unmapped_properties), logger)
+    return rec

hotglue_singer_sdk/helpers/_util.py

@@ -0,0 +1,27 @@
+"""General helper functions, helper classes, and decorators."""
+
+import json
+from pathlib import Path, PurePath
+from typing import Any, Dict, Union, cast
+
+import pendulum
+
+
+def read_json_file(path: Union[PurePath, str]) -> Dict[str, Any]:
+    """Read json file, thowing an error if missing."""
+    if not path:
+        raise RuntimeError("Could not open file. Filepath not provided.")
+
+    if not Path(path).exists():
+        msg = f"File at '{path}' was not found."
+        for template in [f"{path}.template"]:
+            if Path(template).exists():
+                msg += f"\nFor more info, please see the sample template at: {template}"
+        raise FileExistsError(msg)
+
+    return cast(dict, json.loads(Path(path).read_text()))
+
+
+def utc_now() -> pendulum.DateTime:
+    """Return current time in UTC."""
+    return pendulum.now(tz="UTC")

hotglue_singer_sdk/helpers/capabilities.py

@@ -0,0 +1,240 @@
+"""Module with helpers to declare capabilities and plugin behavior."""
+
+from __future__ import annotations
+
+from enum import Enum, EnumMeta
+from typing import Any, TypeVar
+from warnings import warn
+
+from hotglue_singer_sdk.typing import (
+    BooleanType,
+    IntegerType,
+    ObjectType,
+    PropertiesList,
+    Property,
+)
+
+_EnumMemberT = TypeVar("_EnumMemberT")
+
+# Default JSON Schema to support config for built-in capabilities:
+
+STREAM_MAPS_CONFIG = PropertiesList(
+    Property(
+        "stream_maps",
+        ObjectType(),
+        description="Config object for stream maps capability. "
+        + "For more information check out "
+        + "[Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html).",
+    ),
+    Property(
+        "stream_map_config",
+        ObjectType(),
+        description="User-defined config values to be used within map expressions.",
+    ),
+).to_dict()
+FLATTENING_CONFIG = PropertiesList(
+    Property(
+        "flattening_enabled",
+        BooleanType(),
+        description=(
+            "'True' to enable schema flattening and automatically expand nested "
+            "properties."
+        ),
+    ),
+    Property(
+        "flattening_max_depth",
+        IntegerType(),
+        description="The max depth to flatten schemas.",
+    ),
+).to_dict()
+
+
+class DeprecatedEnum(Enum):
+    """Base class for capabilities enumeration."""
+
+    def __new__(
+        cls,
+        value: _EnumMemberT,
+        deprecation: str | None = None,
+    ) -> DeprecatedEnum:
+        """Create a new enum member.
+
+        Args:
+            value: Enum member value.
+            deprecation: Deprecation message.
+
+        Returns:
+            An enum member value.
+        """
+        member: DeprecatedEnum = object.__new__(cls)
+        member._value_ = value
+        member._deprecation = deprecation
+        return member
+
+    @property
+    def deprecation_message(self) -> str | None:
+        """Get deprecation message.
+
+        Returns:
+            Deprecation message.
+        """
+        self._deprecation: str | None
+        return self._deprecation
+
+    def emit_warning(self) -> None:
+        """Emit deprecation warning."""
+        warn(
+            f"{self.name} is deprecated. {self.deprecation_message}",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+
+
+class DeprecatedEnumMeta(EnumMeta):
+    """Metaclass for enumeration with deprecation support."""
+
+    def __getitem__(self, name: str) -> Any:  # noqa: ANN401
+        """Retrieve mapping item.
+
+        Args:
+            name: Item name.
+
+        Returns:
+            Enum member.
+        """
+        obj: Enum = super().__getitem__(name)
+        if isinstance(obj, DeprecatedEnum) and obj.deprecation_message:
+            obj.emit_warning()
+        return obj
+
+    def __getattribute__(cls, name: str) -> Any:  # noqa: ANN401
+        """Retrieve enum attribute.
+
+        Args:
+            name: Attribute name.
+
+        Returns:
+            Attribute.
+        """
+        obj = super().__getattribute__(name)
+        if isinstance(obj, DeprecatedEnum) and obj.deprecation_message:
+            obj.emit_warning()
+        return obj
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:  # noqa: ANN401
+        """Call enum member.
+
+        Args:
+            args: Positional arguments.
+            kwargs: Keyword arguments.
+
+        Returns:
+            Enum member.
+        """
+        obj = super().__call__(*args, **kwargs)
+        if isinstance(obj, DeprecatedEnum) and obj.deprecation_message:
+            obj.emit_warning()
+        return obj
+
+
+class CapabilitiesEnum(DeprecatedEnum, metaclass=DeprecatedEnumMeta):
+    """Base capabilities enumeration."""
+
+    def __str__(self) -> str:
+        """String representation.
+
+        Returns:
+            Stringified enum value.
+        """
+        return str(self.value)
+
+    def __repr__(self) -> str:
+        """String representation.
+
+        Returns:
+            Stringified enum value.
+        """
+        return str(self.value)
+
+
+class PluginCapabilities(CapabilitiesEnum):
+    """Core capabilities which can be supported by taps and targets."""
+
+    #: Support plugin capability and setting discovery.
+    ABOUT = "about"
+
+    #: Support :doc:`inline stream map transforms</stream_maps>`.
+    STREAM_MAPS = "stream-maps"
+
+    #: Support schema flattening, aka denesting of complex properties.
+    FLATTENING = "schema-flattening"
+
+    #: Support the
+    #: `ACTIVATE_VERSION <https://hub.meltano.com/singer/docs#activate-version>`_
+    #: extension.
+    ACTIVATE_VERSION = "activate-version"
+
+    #: Input and output from
+    #: `batched files <https://hub.meltano.com/singer/docs#batch>`_.
+    #: A.K.A ``FAST_SYNC``.
+    BATCH = "batch"
+
+    # Supports raising hotglue exception classes
+    HOTGLUE_EXCEPTIONS_CLASSES = "hotglue-exceptions-classes"
+
+
+class TapCapabilities(CapabilitiesEnum):
+    """Tap-specific capabilities."""
+
+    #: Generate a catalog with `--discover`.
+    DISCOVER = "discover"
+
+    #: Accept input catalog, apply metadata and selection rules.
+    CATALOG = "catalog"
+
+    #: Incremental refresh by means of state tracking.
+    STATE = "state"
+
+    #: Automatic connectivity and stream init test via :ref:`--test<Test connectivity>`.
+    TEST = "test"
+
+    #: Support for ``replication_method: LOG_BASED``. You can read more about this
+    #: feature in `MeltanoHub <https://hub.meltano.com/singer/docs#log-based>`_.
+    LOG_BASED = "log-based"
+
+    #: Deprecated. Please use :attr:`~TapCapabilities.CATALOG` instead.
+    PROPERTIES = "properties", "Please use CATALOG instead."
+
+
+class TargetCapabilities(CapabilitiesEnum):
+    """Target-specific capabilities."""
+
+    #: Allows a ``soft_delete=True`` config option.
+    #: Requires a tap stream supporting :attr:`PluginCapabilities.ACTIVATE_VERSION`
+    #: and/or :attr:`TapCapabilities.LOG_BASED`.
+    SOFT_DELETE = "soft-delete"
+
+    #: Allows a ``hard_delete=True`` config option.
+    #: Requires a tap stream supporting :attr:`PluginCapabilities.ACTIVATE_VERSION`
+    #: and/or :attr:`TapCapabilities.LOG_BASED`.
+    HARD_DELETE = "hard-delete"
+
+    #: Fail safe for unknown JSON Schema types.
+    DATATYPE_FAILSAFE = "datatype-failsafe"
+
+    #: Allow denesting complex properties.
+    RECORD_FLATTENING = "record-flattening"
+
+    #: Allow setting the target schema.
+    TARGET_SCHEMA = "target-schema"
+
+class AlertingLevel(Enum):
+    """
+    The Alerting level to be used when this connector fails on an unexpected error.
+
+    This can be used to prevent too much noise from this connector's failures.
+    """
+
+    WARNING = "warning"
+    ERROR = "error"
+    NONE = "none"

hotglue_singer_sdk/helpers/jsonpath.py

@@ -0,0 +1,39 @@
+"""JSONPath helpers."""
+
+from typing import Any, Generator, Union
+
+import jsonpath_ng
+import memoization
+from jsonpath_ng.ext import parse
+
+
+def extract_jsonpath(
+    expression: str, input: Union[dict, list]
+) -> Generator[Any, None, None]:
+    """Extract records from an input based on a JSONPath expression.
+
+    Args:
+        expression: JSONPath expression to match against the input.
+        input: JSON object or array to extract records from.
+
+    Yields:
+        Records matched with JSONPath expression.
+    """
+    compiled_jsonpath = _compile_jsonpath(expression)
+
+    match: jsonpath_ng.DatumInContext
+    for match in compiled_jsonpath.find(input):
+        yield match.value
+
+
+@memoization.cached
+def _compile_jsonpath(expression: str) -> jsonpath_ng.JSONPath:
+    """Parse a JSONPath expression and cache the result.
+
+    Args:
+        expression: A string representing a JSONPath expression.
+
+    Returns:
+        A compiled JSONPath object.
+    """
+    return parse(expression)

hotglue_singer_sdk/io_base.py

@@ -0,0 +1,134 @@
+"""Abstract base classes for all Singer messages IO operations."""
+
+from __future__ import annotations
+
+import abc
+import enum
+import json
+import logging
+import sys
+from collections import Counter, defaultdict
+from typing import IO
+from typing import Counter as CounterType
+
+from hotglue_singer_sdk.helpers._compat import final
+
+logger = logging.getLogger(__name__)
+
+
+class SingerMessageType(str, enum.Enum):
+    """Singer specification message types."""
+
+    RECORD = "RECORD"
+    SCHEMA = "SCHEMA"
+    STATE = "STATE"
+    ACTIVATE_VERSION = "ACTIVATE_VERSION"
+
+
+class SingerReader(metaclass=abc.ABCMeta):
+    """Interface for all plugins reading Singer messages from stdin."""
+
+    @final
+    def listen(self, file_input: IO[str] | None = None) -> None:
+        """Read from input until all messages are processed.
+
+        Args:
+            file_input: Readable stream of messages. Defaults to standard in.
+
+        This method is internal to the SDK and should not need to be overridden.
+        """
+        if not file_input:
+            file_input = sys.stdin
+
+        self._process_lines(file_input)
+        self._process_endofpipe()
+
+    @staticmethod
+    def _assert_line_requires(line_dict: dict, requires: set[str]) -> None:
+        """Check if dictionary .
+
+        Args:
+            line_dict: TODO
+            requires: TODO
+
+        Raises:
+            Exception: TODO
+        """
+        if not requires.issubset(line_dict):
+            missing = requires - set(line_dict)
+            raise Exception(
+                f"Line is missing required {', '.join(missing)} key(s): {line_dict}"
+            )
+
+    def _process_lines(self, file_input: IO[str]) -> CounterType[str]:
+        """Internal method to process jsonl lines from a Singer tap.
+
+        Args:
+            file_input: Readable stream of messages, each on a separate line.
+
+        Returns:
+            A counter object for the processed lines.
+
+        Raises:
+            json.decoder.JSONDecodeError: raised if any lines are not valid json
+        """
+        stats: dict[str, int] = defaultdict(int)
+        for line in file_input:
+            try:
+                line_dict = json.loads(line)
+            except json.decoder.JSONDecodeError as exc:
+                logger.error("Unable to parse:\n%s", line, exc_info=exc)
+                raise
+
+            self._assert_line_requires(line_dict, requires={"type"})
+
+            record_type: SingerMessageType = line_dict["type"]
+            if record_type == SingerMessageType.SCHEMA:
+                self._process_schema_message(line_dict)
+
+            elif record_type == SingerMessageType.RECORD:
+                self._process_record_message(line_dict)
+
+            elif record_type == SingerMessageType.ACTIVATE_VERSION:
+                self._process_activate_version_message(line_dict)
+
+            elif record_type == SingerMessageType.STATE:
+                self._process_state_message(line_dict)
+
+            else:
+                self._process_unknown_message(line_dict)
+
+            stats[record_type] += 1
+
+        return Counter(**stats)
+
+    @abc.abstractmethod
+    def _process_schema_message(self, message_dict: dict) -> None:
+        ...
+
+    @abc.abstractmethod
+    def _process_record_message(self, message_dict: dict) -> None:
+        ...
+
+    @abc.abstractmethod
+    def _process_state_message(self, message_dict: dict) -> None:
+        ...
+
+    @abc.abstractmethod
+    def _process_activate_version_message(self, message_dict: dict) -> None:
+        ...
+
+    def _process_unknown_message(self, message_dict: dict) -> None:
+        """Internal method to process unknown message types from a Singer tap.
+
+        Args:
+            message_dict: Dictionary representation of the Singer message.
+
+        Raises:
+            ValueError: raised if a message type is not recognized
+        """
+        record_type = message_dict["type"]
+        raise ValueError(f"Unknown message type '{record_type}' in message.")
+
+    def _process_endofpipe(self) -> None:
+        pass