hotglue-singer-sdk 1.0.2__py3-none-any.whl

hotglue_singer_sdk/configuration/_dict_config.py

@@ -0,0 +1,101 @@
+"""Helpers for parsing and wrangling configuration dictionaries."""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Any, Iterable
+
+from dotenv import find_dotenv
+from dotenv.main import DotEnv
+
+from hotglue_singer_sdk.helpers._typing import is_string_array_type
+from hotglue_singer_sdk.helpers._util import read_json_file
+
+logger = logging.getLogger(__name__)
+
+
+def parse_environment_config(
+    config_schema: dict[str, Any],
+    prefix: str,
+    dotenv_path: str | None = None,
+) -> dict[str, Any]:
+    """Parse configuration from environment variables.
+
+    Args:
+        config_schema: A JSON Schema dictionary for the configuration.
+        prefix: Prefix for environment variables.
+        dotenv_path: Path to a .env file. If None, will try to find one in increasingly
+            higher folders.
+
+    Raises:
+        ValueError: If an un-parsable setting is found.
+
+    Returns:
+        A configuration dictionary.
+    """
+    result: dict[str, Any] = {}
+
+    if not dotenv_path:
+        dotenv_path = find_dotenv()
+
+    logger.debug("Loading configuration from %s", dotenv_path)
+    DotEnv(dotenv_path).set_as_environment_variables()
+
+    for config_key in config_schema["properties"].keys():
+        env_var_name = prefix + config_key.upper().replace("-", "_")
+        if env_var_name in os.environ:
+            env_var_value = os.environ[env_var_name]
+            logger.info(
+                "Parsing '%s' config from env variable '%s'.",
+                config_key,
+                env_var_name,
+            )
+            if is_string_array_type(config_schema["properties"][config_key]):
+                if env_var_value[0] == "[" and env_var_value[-1] == "]":
+                    raise ValueError(
+                        "A bracketed list was detected in the environment variable "
+                        f"'{env_var_name}'. This syntax is no longer supported. "
+                        "Please remove the brackets and try again."
+                    )
+                result[config_key] = env_var_value.split(",")
+            else:
+                result[config_key] = env_var_value
+    return result
+
+
+def merge_config_sources(
+    inputs: Iterable[str],
+    config_schema: dict[str, Any],
+    env_prefix: str,
+) -> dict[str, Any]:
+    """Merge configuration from multiple sources into a single dictionary.
+
+    Args:
+        inputs: A sequence of configuration sources (file paths or ENV).
+        config_schema: A JSON Schema dictionary for the configuration.
+        env_prefix: Prefix for environment variables.
+
+    Raises:
+        FileNotFoundError: If any of config files does not exist.
+
+    Returns:
+        A single configuration dictionary.
+    """
+    config: dict[str, Any] = {}
+    for config_path in inputs:
+        if config_path == "ENV":
+            env_config = parse_environment_config(config_schema, prefix=env_prefix)
+            config.update(env_config)
+            continue
+
+        if not Path(config_path).is_file():
+            raise FileNotFoundError(
+                f"Could not locate config file at '{config_path}'."
+                "Please check that the file exists."
+            )
+
+        config.update(read_json_file(config_path))
+
+    return config

hotglue_singer_sdk/exceptions.py

@@ -0,0 +1,52 @@
+"""Defines a common set of exceptions which developers can raise and/or catch."""
+import requests
+
+
+class ConfigValidationError(Exception):
+    """Raised when a user's config settings fail validation."""
+
+
+class FatalAPIError(Exception):
+    """Exception raised when a failed request should not be considered retriable."""
+
+
+class InvalidStreamSortException(Exception):
+    """Exception to raise if sorting errors are found while syncing the records."""
+
+
+class MapExpressionError(Exception):
+    """Failed map expression evaluation."""
+
+
+class MaxRecordsLimitException(Exception):
+    """Exception to raise if the maximum number of allowable records is exceeded."""
+
+
+class RecordsWitoutSchemaException(Exception):
+    """Raised if a target receives RECORD messages prior to a SCHEMA message."""
+
+
+class RetriableAPIError(Exception):
+    """Exception raised when a failed request can be safely retried."""
+
+    def __init__(self, message: str, response: requests.Response = None) -> None:
+        """Extends the default with the failed response as an attribute.
+
+        Args:
+            message (str): The Error Message
+            response (requests.Response): The response object.
+        """
+        super().__init__(message)
+        self.response = response
+
+
+class StreamMapConfigError(Exception):
+    """Raised when a stream map has an invalid configuration."""
+
+
+class TapStreamConnectionFailure(Exception):
+    """Exception to raise when stream connection fails or stream is disconnected."""
+
+
+class TooManyRecordsException(Exception):
+    """Exception to raise when query returns more records than max_records."""

hotglue_singer_sdk/helpers/__init__.py

@@ -0,0 +1 @@
+"""Helper library for the SDK."""

hotglue_singer_sdk/helpers/_catalog.py

@@ -0,0 +1,122 @@
+"""Private helper functions for catalog and selection logic."""
+
+from copy import deepcopy
+from logging import Logger
+from typing import Any, Dict, Optional, Tuple
+
+from memoization import cached
+
+from hotglue_singer_sdk.helpers._singer import Catalog, SelectionMask
+from hotglue_singer_sdk.helpers._typing import is_object_type
+
+_MAX_LRU_CACHE = 500
+
+
+@cached(max_size=_MAX_LRU_CACHE)
+def get_selected_schema(
+    stream_name: str, schema: dict, mask: SelectionMask, logger: Logger
+) -> dict:
+    """Return a copy of the provided JSON schema, dropping any fields not selected."""
+    new_schema = deepcopy(schema)
+    _pop_deselected_schema(new_schema, mask, stream_name, (), logger)
+    return new_schema
+
+
+def _pop_deselected_schema(
+    schema: dict,
+    mask: SelectionMask,
+    stream_name: str,
+    breadcrumb: Tuple[str, ...],
+    logger: Logger,
+) -> None:
+    """Remove anything from schema that is not selected.
+
+    Walk through schema, starting at the index in breadcrumb, recursively updating in
+    place.
+    """
+    schema_at_breadcrumb = schema
+    for crumb in breadcrumb:
+        schema_at_breadcrumb = schema_at_breadcrumb.get(crumb, {})
+
+    if not isinstance(schema_at_breadcrumb, dict):
+        raise ValueError(
+            f"Expected dictionary type instead of "
+            f"'{type(schema_at_breadcrumb).__name__}' '{schema_at_breadcrumb}' "
+            f"for '{stream_name}' bookmark '{str(breadcrumb)}' in '{schema}'"
+        )
+
+    if "properties" not in schema_at_breadcrumb:
+        return
+
+    for property_name, property_def in list(schema_at_breadcrumb["properties"].items()):
+        property_breadcrumb: Tuple[str, ...] = tuple(
+            list(breadcrumb) + ["properties", property_name]
+        )
+        selected = mask[property_breadcrumb]
+        if not selected:
+            schema_at_breadcrumb["properties"].pop(property_name, None)
+            continue
+
+        if is_object_type(property_def):
+            # call recursively in case any subproperties are deselected.
+            _pop_deselected_schema(
+                schema, mask, stream_name, property_breadcrumb, logger
+            )
+
+
+def pop_deselected_record_properties(
+    record: Dict[str, Any],
+    schema: dict,
+    mask: SelectionMask,
+    logger: Logger,
+    breadcrumb: Tuple[str, ...] = (),
+) -> None:
+    """Remove anything from record properties that is not selected.
+
+    Walk through properties, starting at the index in breadcrumb, recursively
+    updating in place.
+    """
+    for property_name, val in list(record.items()):
+        property_breadcrumb = breadcrumb + ("properties", property_name)
+        selected = mask[property_breadcrumb]
+        if not selected:
+            record.pop(property_name)
+            continue
+
+        if isinstance(val, dict):
+            # call recursively in case any subproperties are deselected.
+            pop_deselected_record_properties(
+                val, schema, mask, logger, property_breadcrumb
+            )
+
+
+def deselect_all_streams(catalog: Catalog) -> None:
+    """Deselect all streams in catalog dictionary."""
+    for entry in catalog.streams:
+        set_catalog_stream_selected(catalog, entry.tap_stream_id, selected=False)
+
+
+def set_catalog_stream_selected(
+    catalog: Catalog,
+    stream_name: str,
+    selected: bool,
+    breadcrumb: Optional[Tuple[str, ...]] = None,
+) -> None:
+    """Return True if the property is selected for extract.
+
+    Breadcrumb of `[]` or `None` indicates the stream itself. Otherwise, the
+    breadcrumb is the path to a property within the stream.
+    """
+    breadcrumb = breadcrumb or ()
+    if not isinstance(breadcrumb, tuple):
+        raise ValueError(
+            f"Expected tuple value for breadcrumb '{breadcrumb}'. "
+            f"Got {type(breadcrumb).__name__}"
+        )
+
+    catalog_entry = catalog.get_stream(stream_name)
+    if not catalog_entry:
+        raise ValueError(f"Catalog entry missing for '{stream_name}'. Skipping.")
+
+    md_entry = catalog_entry.metadata[breadcrumb]
+    md_entry.selected = selected

hotglue_singer_sdk/helpers/_classproperty.py

@@ -0,0 +1,18 @@
+# flake8: noqa
+
+"""Defines the `classproperty` decorator."""
+
+# noqa
+
+
+class classproperty(property):
+    """Class property decorator."""
+
+    def __get__(self, obj, objtype=None):
+        return super().__get__(objtype)
+
+    def __set__(self, obj, value):
+        super().__set__(type(obj), value)
+
+    def __delete__(self, obj):
+        super().__delete__(type(obj))

hotglue_singer_sdk/helpers/_compat.py

@@ -0,0 +1,15 @@
+"""Compatibility helpers."""
+
+try:
+    from typing import final
+except ImportError:
+    # Final not available until Python3.8
+    final = lambda f: f  # noqa: E731
+
+try:
+    from importlib import metadata
+except ImportError:
+    # Running on pre-3.8 Python; use importlib-metadata package
+    import importlib_metadata as metadata  # type: ignore
+
+__all__ = ["metadata", "final"]

hotglue_singer_sdk/helpers/_flattening.py

@@ -0,0 +1,374 @@
+"""Internal helper library for record flatteting functions."""
+
+import collections
+import itertools
+import json
+import re
+from copy import deepcopy
+from typing import Any, List, Mapping, MutableMapping, NamedTuple, Optional, Tuple
+
+import inflection
+
+DEFAULT_FLATTENING_SEPARATOR = "__"
+
+
+class FlatteningOptions(NamedTuple):
+    """A stream map which performs the flattening role."""
+
+    max_level: int
+    flattening_enabled: bool = True
+    separator: str = DEFAULT_FLATTENING_SEPARATOR
+
+
+def get_flattening_options(
+    plugin_config: Mapping,
+) -> Optional[FlatteningOptions]:
+    """Get flattening options, if flattening is enabled.
+
+    Args:
+        plugin_config: The tap or target config dictionary.
+
+    Returns:
+        A new FlatteningOptions object or None if flattening is disabled.
+    """
+    if "flattening_enabled" in plugin_config and plugin_config["flattening_enabled"]:
+        return FlatteningOptions(max_level=int(plugin_config["flattening_max_depth"]))
+
+    return None
+
+
+def flatten_key(key_name: str, parent_keys: List[str], separator: str = "__") -> str:
+    """Concatenate `key_name` with its `parent_keys` using `separator`.
+
+    Args:
+        key_name: The node's key.
+        parent_keys: A list of parent keys which are ancestors to this node.
+        separator: The separator used during concatenation. Defaults to "__".
+
+    Returns:
+        The flattened key name as a string.
+
+    >>> flatten_key("foo", ["bar", "baz"])
+    'bar__baz__foo'
+
+    >>> flatten_key("foo", ["bar", "baz"], separator=".")
+    'bar.baz.foo'
+    """
+    full_key = parent_keys + [key_name]
+    inflected_key = full_key.copy()
+    reducer_index = 0
+    while len(separator.join(inflected_key)) >= 255 and reducer_index < len(
+        inflected_key
+    ):
+        reduced_key = re.sub(
+            r"[a-z]", "", inflection.camelize(inflected_key[reducer_index])
+        )
+        inflected_key[reducer_index] = (
+            reduced_key if len(reduced_key) > 1 else inflected_key[reducer_index][0:3]
+        ).lower()
+        reducer_index += 1
+
+    return separator.join(inflected_key)
+
+
+def flatten_schema(
+    schema: dict,
+    max_level: int,
+    separator: str = "__",
+) -> dict:
+    """Flatten the provided schema up to a depth of max_level.
+
+    Args:
+        schema: The schema definition to flatten.
+        separator: The string to use when concatenating key names.
+        max_level: The max recursion level (zero-based, exclusive).
+
+    Returns:
+        A flattened version of the provided schema definition.
+
+    >>> import json
+    >>> schema = {
+    ...     "type": "object",
+    ...     "properties": {
+    ...         "id": {
+    ...             "type": "string"
+    ...         },
+    ...         "foo": {
+    ...             "type": "object",
+    ...             "properties": {
+    ...                 "bar": {
+    ...                     "type": "object",
+    ...                     "properties": {
+    ...                         "baz": {
+    ...                             "type": "object",
+    ...                             "properties": {
+    ...                                 "qux": {
+    ...                                     "type": "string"
+    ...                                 }
+    ...                             }
+    ...                         }
+    ...                     }
+    ...                 }
+    ...             }
+    ...         }
+    ...     }
+    ... }
+    >>> print(json.dumps(flatten_schema(schema, 0), indent=2))
+    {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string"
+        },
+        "foo": {
+          "type": "object",
+          "properties": {
+            "bar": {
+              "type": "object",
+              "properties": {
+                "baz": {
+                  "type": "object",
+                  "properties": {
+                    "qux": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    >>> print(json.dumps(flatten_schema(schema, 1), indent=2))
+    {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string"
+        },
+        "foo__bar": {
+          "type": "object",
+          "properties": {
+            "baz": {
+              "type": "object",
+              "properties": {
+                "qux": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    >>> print(json.dumps(flatten_schema(schema, 2), indent=2))
+    {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string"
+        },
+        "foo__bar__baz": {
+          "type": "object",
+          "properties": {
+            "qux": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    }
+
+    >>> print(json.dumps(flatten_schema(schema, 3), indent=2))
+    {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string"
+        },
+        "foo__bar__baz__qux": {
+          "type": "string"
+        }
+      }
+    }
+    """
+    new_schema = deepcopy(schema)
+    new_schema["properties"] = _flatten_schema(
+        schema_node=new_schema,
+        max_level=max_level,
+        separator=separator,
+    )
+    return new_schema
+
+
+def _flatten_schema(
+    schema_node: dict,
+    parent_keys: List[str] = None,
+    separator: str = "__",
+    level: int = 0,
+    max_level: int = 0,
+) -> dict:
+    """Flatten the provided schema node, recursively up to depth of `max_level`.
+
+    Args:
+        schema_node: The schema node to flatten.
+        parent_key: The parent's key, provided as a list of node names.
+        separator: The string to use when concatenating key names.
+        level: The current recursion level (zero-based).
+        max_level: The max recursion level (zero-based, exclusive).
+
+    Returns:
+        A flattened version of the provided node.
+    """
+    if parent_keys is None:
+        parent_keys = []
+
+    items: List[Tuple[str, dict]] = []
+    if "properties" not in schema_node:
+        return {}
+
+    for k, v in schema_node["properties"].items():
+        new_key = flatten_key(k, parent_keys, separator)
+        if "type" in v.keys():
+            if "object" in v["type"] and "properties" in v and level < max_level:
+                items.extend(
+                    _flatten_schema(
+                        v,
+                        parent_keys + [k],
+                        separator=separator,
+                        level=level + 1,
+                        max_level=max_level,
+                    ).items()
+                )
+            else:
+                items.append((new_key, v))
+        else:
+            if len(v.values()) > 0:
+                if list(v.values())[0][0]["type"] == "string":
+                    list(v.values())[0][0]["type"] = ["null", "string"]
+                    items.append((new_key, list(v.values())[0][0]))
+                elif list(v.values())[0][0]["type"] == "array":
+                    list(v.values())[0][0]["type"] = ["null", "array"]
+                    items.append((new_key, list(v.values())[0][0]))
+                elif list(v.values())[0][0]["type"] == "object":
+                    list(v.values())[0][0]["type"] = ["null", "object"]
+                    items.append((new_key, list(v.values())[0][0]))
+
+    # Sort and check for duplicates
+    def _key_func(item):
+        return item[0]  # first item is tuple is the key name.
+
+    sorted_items = sorted(items, key=_key_func)
+    for k, g in itertools.groupby(sorted_items, key=_key_func):
+        if len(list(g)) > 1:
+            raise ValueError(f"Duplicate column name produced in schema: {k}")
+
+    # Return the (unsorted) result as a dict.
+    return dict(items)
+
+
+def flatten_record(
+    record: dict,
+    flattened_schema: dict,
+    max_level: int,
+    separator: str = "__",
+) -> dict:
+    """Flatten a record up to max_level.
+
+    Args:
+        record: The record to flatten.
+        flattened_schema: The already flattened schema.
+        separator: The string used to separate concatenated key names. Defaults to "__".
+        max_level: The maximum depth of keys to flatten recursively.
+
+    Returns:
+        A flattened version of the record.
+    """
+    return _flatten_record(
+        record_node=record,
+        flattened_schema=flattened_schema,
+        separator=separator,
+        max_level=max_level,
+    )
+
+
+def _flatten_record(
+    record_node: MutableMapping[Any, Any],
+    flattened_schema: dict = None,
+    parent_key: List[str] = None,
+    separator: str = "__",
+    level: int = 0,
+    max_level: int = 0,
+) -> dict:
+    """This recursive function flattens the record node.
+
+    The current invocation is expected to be at `level` and will continue recursively
+    until the provided `max_level` is reached.
+
+    Args:
+        record_node: The record node to flatten.
+        flattened_schema: The already flattened full schema for the record.
+        parent_key: The parent's key, provided as a list of node names.
+        separator: The string to use when concatenating key names.
+        level: The current recursion level (zero-based).
+        max_level: The max recursion level (zero-based, exclusive).
+
+    Returns:
+        A flattened version of the provided node.
+    """
+    if parent_key is None:
+        parent_key = []
+
+    items: List[Tuple[str, Any]] = []
+    for k, v in record_node.items():
+        new_key = flatten_key(k, parent_key, separator)
+        if isinstance(v, collections.abc.MutableMapping) and level < max_level:
+            items.extend(
+                _flatten_record(
+                    v,
+                    flattened_schema,
+                    parent_key + [k],
+                    separator=separator,
+                    level=level + 1,
+                    max_level=max_level,
+                ).items()
+            )
+        else:
+            items.append(
+                (
+                    new_key,
+                    json.dumps(v)
+                    if _should_jsondump_value(k, v, flattened_schema)
+                    else v,
+                )
+            )
+
+    return dict(items)
+
+
+def _should_jsondump_value(key: str, value: Any, flattened_schema=None) -> bool:
+    """Return True if json.dump() should be used to serialize the value.
+
+    Args:
+        key: [description]
+        value: [description]
+        schema: [description]. Defaults to None.
+
+    Returns:
+        [description]
+    """
+    if isinstance(value, (dict, list)):
+        return True
+
+    if (
+        flattened_schema
+        and key in flattened_schema
+        and "type" in flattened_schema[key]
+        and set(flattened_schema[key]["type"]) == {"null", "object", "array"}
+    ):
+        return True
+
+    return False