hotglue-singer-sdk 1.0.2__py3-none-any.whl

hotglue_singer_sdk/helpers/_singer.py

@@ -0,0 +1,280 @@
+import logging
+from dataclasses import dataclass, fields
+from enum import Enum
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union, cast
+
+from singer.catalog import Catalog as BaseCatalog
+from singer.catalog import CatalogEntry as BaseCatalogEntry
+
+from hotglue_singer_sdk.helpers._schema import SchemaPlus
+
+Breadcrumb = Tuple[str, ...]
+
+logger = logging.getLogger(__name__)
+
+
+class SelectionMask(Dict[Breadcrumb, bool]):
+    """Boolean mask for property selection in schemas and records."""
+
+    def __missing__(self, breadcrumb: Breadcrumb) -> bool:
+        """Handle missing breadcrumbs.
+
+        - Properties default to parent value if available.
+        - Root (stream) defaults to True.
+        """
+        if len(breadcrumb) >= 2:
+            parent = breadcrumb[:-2]
+            return self[parent]
+        else:
+            return True
+
+
+@dataclass
+class Metadata:
+    """Base stream or property metadata."""
+
+    class InclusionType(str, Enum):
+        """Catalog inclusion types."""
+
+        AVAILABLE = "available"
+        AUTOMATIC = "automatic"
+        UNSUPPORTED = "unsupported"
+
+    inclusion: Optional[InclusionType] = None
+    selected: Optional[bool] = None
+    selected_by_default: Optional[bool] = None
+
+    @classmethod
+    def from_dict(cls, value: Dict[str, Any]):
+        """Parse metadata dictionary."""
+        return cls(
+            **{
+                field.name: value.get(field.name.replace("_", "-"))
+                for field in fields(cls)
+            }
+        )
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert metadata to a JSON-encodeable dictionary."""
+        result = {}
+
+        for field in fields(self):
+            value = getattr(self, field.name)
+            if value is not None:
+                result[field.name.replace("_", "-")] = value
+
+        return result
+
+
+@dataclass
+class StreamMetadata(Metadata):
+    """Stream metadata."""
+
+    table_key_properties: Optional[List[str]] = None
+    forced_replication_method: Optional[str] = None
+    valid_replication_keys: Optional[List[str]] = None
+    schema_name: Optional[str] = None
+
+
+class MetadataMapping(Dict[Breadcrumb, Union[Metadata, StreamMetadata]]):
+    """Stream metadata mapping."""
+
+    @classmethod
+    def from_iterable(cls, iterable: Iterable[Dict[str, Any]]):
+        """Create a metadata mapping from an iterable of metadata dictionaries."""
+        mapping = cls()
+        for d in iterable:
+            breadcrumb = tuple(d["breadcrumb"])
+            metadata = d["metadata"]
+            if breadcrumb:
+                mapping[breadcrumb] = Metadata.from_dict(metadata)
+            else:
+                mapping[breadcrumb] = StreamMetadata.from_dict(metadata)
+
+        return mapping
+
+    def to_list(self) -> List[Dict[str, Any]]:
+        """Convert mapping to a JSON-encodable list."""
+        return [
+            {"breadcrumb": list(k), "metadata": v.to_dict()} for k, v in self.items()
+        ]
+
+    def __missing__(self, breadcrumb: Breadcrumb):
+        """Handle missing metadata entries."""
+        self[breadcrumb] = Metadata() if breadcrumb else StreamMetadata()
+        return self[breadcrumb]
+
+    @property
+    def root(self):
+        """Get stream (root) metadata from this mapping."""
+        meta: StreamMetadata = self[()]
+        return meta
+
+    @classmethod
+    def get_standard_metadata(
+        cls,
+        schema: Optional[Dict[str, Any]] = None,
+        schema_name: Optional[str] = None,
+        key_properties: Optional[List[str]] = None,
+        valid_replication_keys: Optional[List[str]] = None,
+        replication_method: Optional[str] = None,
+    ):
+        """Get default metadata for a stream."""
+        mapping = cls()
+        root = StreamMetadata(
+            table_key_properties=key_properties,
+            forced_replication_method=replication_method,
+            valid_replication_keys=valid_replication_keys,
+        )
+
+        if schema:
+            root.inclusion = Metadata.InclusionType.AVAILABLE
+
+            if schema_name:
+                root.schema_name = schema_name
+
+            for field_name in schema.get("properties", {}).keys():
+                if key_properties and field_name in key_properties:
+                    entry = Metadata(inclusion=Metadata.InclusionType.AUTOMATIC)
+                else:
+                    entry = Metadata(inclusion=Metadata.InclusionType.AVAILABLE)
+
+                mapping[("properties", field_name)] = entry
+
+        mapping[()] = root
+
+        return mapping
+
+    def resolve_selection(self) -> SelectionMask:
+        """Resolve selection for metadata breadcrumbs and store them in a mapping."""
+        return SelectionMask(
+            (breadcrumb, self._breadcrumb_is_selected(breadcrumb))
+            for breadcrumb in self
+        )
+
+    def _breadcrumb_is_selected(self, breadcrumb: Breadcrumb) -> bool:
+        """Determine if a property breadcrumb is selected based on existing metadata.
+
+        An empty breadcrumb (empty tuple) indicates the stream itself. Otherwise, the
+        breadcrumb is the path to a property within the stream.
+        """
+        if not self:
+            # Default to true if no metadata to say otherwise
+            return True
+
+        md_entry = self.get(breadcrumb, Metadata())
+        parent_value = None
+
+        if len(breadcrumb) > 0:
+            parent_breadcrumb = breadcrumb[:-2]
+            parent_value = self._breadcrumb_is_selected(parent_breadcrumb)
+
+        if parent_value is False:
+            return parent_value
+
+        if md_entry.inclusion == Metadata.InclusionType.UNSUPPORTED:
+            if md_entry.selected is True:
+                logger.debug(
+                    "Property '%s' was selected but is not supported. "
+                    "Ignoring selected==True input.",
+                    ":".join(breadcrumb),
+                )
+            return False
+
+        if md_entry.inclusion == Metadata.InclusionType.AUTOMATIC:
+            if md_entry.selected is False:
+                logger.debug(
+                    "Property '%s' was deselected while also set "
+                    "for automatic inclusion. Ignoring selected==False input.",
+                    ":".join(breadcrumb),
+                )
+            return True
+
+        if md_entry.selected is not None:
+            return md_entry.selected
+
+        if md_entry.selected_by_default is not None:
+            return md_entry.selected_by_default
+
+        logger.debug(
+            "Selection metadata omitted for '%s'. "
+            "Using parent value of selected=%s.",
+            breadcrumb,
+            parent_value,
+        )
+        return parent_value or False
+
+
+@dataclass
+class CatalogEntry(BaseCatalogEntry):
+    """Singer catalog entry."""
+
+    tap_stream_id: str
+    metadata: MetadataMapping
+    schema: SchemaPlus
+    stream: Optional[str] = None
+    key_properties: Optional[List[str]] = None
+    replication_key: Optional[str] = None
+    is_view: Optional[bool] = None
+    database: Optional[str] = None
+    table: Optional[str] = None
+    row_count: Optional[int] = None
+    stream_alias: Optional[str] = None
+    replication_method: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, stream: Dict[str, Any]):
+        """Create a catalog entry from a dictionary."""
+        return cls(
+            tap_stream_id=stream["tap_stream_id"],
+            stream=stream.get("stream"),
+            replication_key=stream.get("replication_key"),
+            key_properties=stream.get("key_properties"),
+            database=stream.get("database_name"),
+            table=stream.get("table_name"),
+            schema=SchemaPlus.from_dict(stream.get("schema", {})),
+            is_view=stream.get("is_view"),
+            stream_alias=stream.get("stream_alias"),
+            metadata=MetadataMapping.from_iterable(stream.get("metadata", [])),
+            replication_method=stream.get("replication_method"),
+        )
+
+    def to_dict(self):
+        """Convert entry to a dictionary."""
+        d = super().to_dict()
+        d["metadata"] = self.metadata.to_list()
+        return d
+
+
+class Catalog(Dict[str, CatalogEntry], BaseCatalog):
+    """Singer catalog mapping of stream entries."""
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, List[Dict[str, Any]]]) -> "Catalog":
+        """Create a catalog from a dictionary."""
+        instance = cls()
+        for stream in data.get("streams", []):
+            entry = CatalogEntry.from_dict(stream)
+            instance[entry.tap_stream_id] = entry
+        return instance
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a dictionary representation of the catalog.
+
+        Returns:
+            A dictionary with the defined catalog streams.
+        """
+        return cast(Dict[str, Any], super().to_dict())
+
+    @property
+    def streams(self) -> List[CatalogEntry]:
+        """Get catalog entries."""
+        return list(self.values())
+
+    def add_stream(self, entry: CatalogEntry) -> None:
+        """Add a stream entry to the catalog."""
+        self[entry.tap_stream_id] = entry
+
+    def get_stream(self, stream_id: str) -> Optional[CatalogEntry]:
+        """Retrieve a stream entry from the catalog."""
+        return self.get(stream_id)

hotglue_singer_sdk/helpers/_state.py

@@ -0,0 +1,282 @@
+"""Helper functions for state and bookmark management."""
+
+import datetime
+from typing import Any, Callable, List, Optional, Union, cast
+
+from hotglue_singer_sdk.exceptions import InvalidStreamSortException
+from hotglue_singer_sdk.helpers._typing import to_json_compatible
+
+PROGRESS_MARKERS = "progress_markers"
+PROGRESS_MARKER_NOTE = "Note"
+SIGNPOST_MARKER = "replication_key_signpost"
+STARTING_MARKER = "starting_replication_value"
+
+
+def get_state_if_exists(
+    tap_state: dict,
+    tap_stream_id: str,
+    state_partition_context: Optional[dict] = None,
+    key: Optional[str] = None,
+) -> Optional[Any]:
+    """Return the stream or partition state, creating a new one if it does not exist.
+
+    Parameters
+    ----------
+    tap_state : dict
+        the existing state dict which contains all streams.
+    tap_stream_id : str
+        the id of the stream
+    state_partition_context : Optional[dict], optional
+        keys which identify the partition context, by default None (not partitioned)
+    key : Optional[str], optional
+        name of the key searched for, by default None (return entire state if found)
+
+    Returns
+    -------
+    Optional[Any]
+        Returns the state if exists, otherwise None
+
+    Raises
+    ------
+    ValueError
+        Raised if state is invalid or cannot be parsed.
+
+    """
+    if "bookmarks" not in tap_state:
+        return None
+    if tap_stream_id not in tap_state["bookmarks"]:
+        return None
+
+    stream_state = tap_state["bookmarks"][tap_stream_id]
+    if not state_partition_context:
+        if key:
+            return stream_state.get(key, None)
+        return stream_state
+    if "partitions" not in stream_state:
+        return None  # No partitions defined
+
+    matched_partition = _find_in_partitions_list(
+        stream_state["partitions"], state_partition_context
+    )
+    if matched_partition is None:
+        return None  # Partition definition not present
+    if key:
+        return matched_partition.get(key, None)
+    return matched_partition
+
+
+def get_state_partitions_list(
+    tap_state: dict, tap_stream_id: str
+) -> Optional[List[dict]]:
+    """Return a list of partitions defined in the state, or None if not defined."""
+    return (get_state_if_exists(tap_state, tap_stream_id) or {}).get("partitions", None)
+
+
+def _find_in_partitions_list(
+    partitions: List[dict], state_partition_context: dict
+) -> Optional[dict]:
+    found = [
+        partition_state
+        for partition_state in partitions
+        if partition_state["context"] == state_partition_context
+    ]
+    if len(found) > 1:
+        raise ValueError(
+            f"State file contains duplicate entries for partition: "
+            "{state_partition_context}.\n"
+            f"Matching state values were: {str(found)}"
+        )
+    if found:
+        return cast(dict, found[0])
+
+    return None
+
+
+def _create_in_partitions_list(
+    partitions: List[dict], state_partition_context: dict
+) -> dict:
+    # Existing partition not found. Creating new state entry in partitions list...
+    new_partition_state = {"context": state_partition_context}
+    partitions.append(new_partition_state)
+    return new_partition_state
+
+
+def get_writeable_state_dict(
+    tap_state: dict, tap_stream_id: str, state_partition_context: Optional[dict] = None
+) -> dict:
+    """Return the stream or partition state, creating a new one if it does not exist.
+
+    Parameters
+    ----------
+    tap_state : dict
+        the existing state dict which contains all streams.
+    tap_stream_id : str
+        the id of the stream
+    state_partition_context : Optional[dict], optional
+        keys which identify the partition context, by default None (not partitioned)
+
+    Returns
+    -------
+    dict
+        Returns a writeable dict at the stream or partition level.
+
+    Raises
+    ------
+    ValueError
+        Raise an error if duplicate entries are found.
+
+    """
+    if tap_state is None:
+        raise ValueError("Cannot write state to missing state dictionary.")
+
+    if "bookmarks" not in tap_state:
+        tap_state["bookmarks"] = {}
+    if tap_stream_id not in tap_state["bookmarks"]:
+        tap_state["bookmarks"][tap_stream_id] = {}
+    stream_state = cast(dict, tap_state["bookmarks"][tap_stream_id])
+    if not state_partition_context:
+        return stream_state
+
+    if "partitions" not in stream_state:
+        stream_state["partitions"] = []
+    stream_state_partitions: List[dict] = stream_state["partitions"]
+    found = _find_in_partitions_list(stream_state_partitions, state_partition_context)
+    if found:
+        return found
+
+    return _create_in_partitions_list(stream_state_partitions, state_partition_context)
+
+
+def write_stream_state(
+    tap_state,
+    tap_stream_id: str,
+    key,
+    val,
+    *,
+    state_partition_context: Optional[dict] = None,
+) -> None:
+    """Write stream state."""
+    state_dict = get_writeable_state_dict(
+        tap_state, tap_stream_id, state_partition_context=state_partition_context
+    )
+    state_dict[key] = val
+
+
+def reset_state_progress_markers(stream_or_partition_state: dict) -> Optional[dict]:
+    """Wipe the state once sync is complete.
+
+    For logging purposes, return the wiped 'progress_markers' object if it existed.
+    """
+    progress_markers = stream_or_partition_state.pop(PROGRESS_MARKERS, {})
+    # Remove auto-generated human-readable note:
+    progress_markers.pop(PROGRESS_MARKER_NOTE, None)
+    # Return remaining 'progress_markers' if any:
+    return progress_markers or None
+
+
+def write_replication_key_signpost(
+    stream_or_partition_state: dict,
+    new_signpost_value: Any,
+) -> None:
+    """Write signpost value."""
+    stream_or_partition_state[SIGNPOST_MARKER] = to_json_compatible(new_signpost_value)
+
+
+def write_starting_replication_value(
+    stream_or_partition_state: dict,
+    initial_value: Any,
+) -> None:
+    """Write initial replication value to state."""
+    stream_or_partition_state[STARTING_MARKER] = to_json_compatible(initial_value)
+
+
+def get_starting_replication_value(stream_or_partition_state: dict):
+    """Retrieve initial replication marker value from state."""
+    if not stream_or_partition_state:
+        return None
+    return stream_or_partition_state.get(STARTING_MARKER)
+
+
+def increment_state(
+    stream_or_partition_state: dict,
+    latest_record: dict,
+    replication_key: str,
+    is_sorted: bool,
+    check_sorted: bool,
+) -> None:
+    """Update the state using data from the latest record.
+
+    Raises InvalidStreamSortException if is_sorted=True, check_sorted=True and unsorted
+    data is detected in the stream.
+    """
+    progress_dict = stream_or_partition_state
+    if not is_sorted:
+        if PROGRESS_MARKERS not in stream_or_partition_state:
+            stream_or_partition_state[PROGRESS_MARKERS] = {
+                PROGRESS_MARKER_NOTE: "Progress is not resumable if interrupted."
+            }
+        progress_dict = stream_or_partition_state[PROGRESS_MARKERS]
+    old_rk_value = to_json_compatible(progress_dict.get("replication_key_value"))
+    new_rk_value = to_json_compatible(latest_record[replication_key])
+    if old_rk_value is None or not check_sorted or new_rk_value >= old_rk_value:
+        progress_dict["replication_key"] = replication_key
+        progress_dict["replication_key_value"] = new_rk_value
+        return
+
+    if is_sorted:
+        raise InvalidStreamSortException(
+            f"Unsorted data detected in stream. Latest value '{new_rk_value}' is "
+            f"smaller than previous max '{old_rk_value}'."
+        )
+
+
+def _greater_than_signpost(
+    signpost: Union[datetime.datetime, str, int, float],
+    new_value: Union[datetime.datetime, str, int, float],
+) -> bool:
+    """Compare and return True if new_value is greater than signpost."""
+    return (  # fails if signpost and bookmark are incompatible types
+        new_value > signpost  # type: ignore
+    )
+
+
+def finalize_state_progress_markers(stream_or_partition_state: dict) -> Optional[dict]:
+    """Promote or wipe progress markers once sync is complete."""
+    signpost_value = stream_or_partition_state.pop(SIGNPOST_MARKER, None)
+    stream_or_partition_state.pop(STARTING_MARKER, None)
+    if PROGRESS_MARKERS in stream_or_partition_state:
+        if "replication_key" in stream_or_partition_state[PROGRESS_MARKERS]:
+            # Replication keys valid (only) after sync is complete
+            progress_markers = stream_or_partition_state[PROGRESS_MARKERS]
+            stream_or_partition_state["replication_key"] = progress_markers.pop(
+                "replication_key"
+            )
+            new_rk_value = progress_markers.pop("replication_key_value")
+            if signpost_value and _greater_than_signpost(signpost_value, new_rk_value):
+                new_rk_value = signpost_value
+            stream_or_partition_state["replication_key_value"] = new_rk_value
+    # Wipe and return any markers that have not been promoted
+    return reset_state_progress_markers(stream_or_partition_state)
+
+
+def log_sort_error(
+    ex: Exception,
+    log_fn: Callable,
+    stream_name: str,
+    current_context: Optional[dict],
+    state_partition_context: Optional[dict],
+    record_count: int,
+    partition_record_count: int,
+) -> None:
+    """Log a sort error."""
+    msg = f"Sorting error detected in '{stream_name}'." f"on record #{record_count}. "
+    if partition_record_count != record_count:
+        msg += (
+            f"Record was partition record "
+            f"#{partition_record_count} with"
+            f" state partition context {state_partition_context}. "
+        )
+    if current_context:
+        msg += f"Context was {str(current_context)}. "
+    msg += str(ex)
+    log_fn(msg)