apache-hamilton 1.90.0.dev0__py3-none-any.whl

hamilton/caching/cache_key.py

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import base64
+import zlib
+from collections.abc import Mapping
+
+
+def _compress_string(string: str) -> str:
+    return base64.b64encode(zlib.compress(string.encode(), level=3)).decode()
+
+
+def _decompress_string(string: str) -> str:
+    return zlib.decompress(base64.b64decode(string.encode())).decode()
+
+
+def _encode_str_dict(d: Mapping) -> str:
+    interleaved_tuple = tuple(item for pair in sorted(d.items()) for item in pair)
+    return ",".join(interleaved_tuple)
+
+
+def _decode_str_dict(s: str) -> Mapping:
+    interleaved_tuple = tuple(s.split(","))
+    d = {}
+    for i in range(0, len(interleaved_tuple), 2):
+        d[interleaved_tuple[i]] = interleaved_tuple[i + 1]
+    return d
+
+
+def decode_key(cache_key: str) -> dict:
+    node_name, _, code_and_data_string = cache_key.partition("-")
+    code_version, _, dep_encoded = code_and_data_string.partition("-")
+    data_stringified = _decompress_string(dep_encoded)
+
+    if data_stringified == "<none>":
+        dependencies_data_versions = {}
+    else:
+        dependencies_data_versions = _decode_str_dict(data_stringified)
+    return dict(
+        node_name=node_name,
+        code_version=code_version,
+        dependencies_data_versions=dependencies_data_versions,
+    )
+
+
+def create_cache_key(
+    node_name: str, code_version: str, dependencies_data_versions: dict[str, str]
+) -> str:
+    if len(dependencies_data_versions.keys()) > 0:
+        dependencies_stringified = _encode_str_dict(dependencies_data_versions)
+    else:
+        dependencies_stringified = "<none>"
+
+    safe_node_name = "".join(c for c in node_name if c.isalnum() or c == "_").rstrip()
+
+    return f"{safe_node_name}-{code_version}-{_compress_string(dependencies_stringified)}"

hamilton/caching/fingerprinting.py

@@ -0,0 +1,287 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+This module contains hashing functions for Python objects. It uses
+functools.singledispatch to allow specialized implementations based on type.
+Singledispatch automatically applies the most specific implementation
+
+This module houses implementations for the Python standard library. Supporting
+all types is considerable endeavor, so we'll add support as types are requested
+by users.
+
+Otherwise, 3rd party types can be supported via the `h_databackends` module.
+This registers abstract types that can be checked without having to import the
+3rd party library. For instance, there are implementations for pandas.DataFrame
+and polars.DataFrame despite these libraries not being imported here.
+
+IMPORTANT all container types that make a recursive call to `hash_value` or a specific
+implementation should pass the `depth` parameter to prevent `RecursionError`.
+"""
+
+import base64
+import datetime
+import functools
+import hashlib
+import logging
+import sys
+from collections.abc import Mapping, Sequence, Set
+
+from hamilton.experimental import h_databackends
+
+# NoneType is introduced in Python 3.10
+try:
+    from types import NoneType
+except ImportError:
+    NoneType = type(None)
+
+
+logger = logging.getLogger("hamilton.caching")
+
+
+MAX_DEPTH = 6
+UNHASHABLE = "<unhashable>"
+NONE_HASH = "<none>"
+
+
+def set_max_depth(depth: int) -> None:
+    """Set the maximum recursion depth for fingerprinting non-supported types.
+
+    :param depth: The maximum depth for fingerprinting.
+    """
+    global MAX_DEPTH
+    MAX_DEPTH = depth
+
+
+def _compact_hash(digest: bytes) -> str:
+    """Compact the hash to a string that's safe to pass around.
+
+    NOTE this is particularly relevant for the Hamilton UI and
+    passing hashes/fingerprints through web services.
+    """
+    return base64.urlsafe_b64encode(digest).decode()
+
+
+@functools.singledispatch
+def hash_value(obj, *args, depth=0, **kwargs) -> str:
+    """Fingerprinting strategy that computes a hash of the
+    full Python object.
+
+    The default case hashes the `__dict__` attribute of the
+    object (recursive).
+    """
+    if depth > MAX_DEPTH:
+        return UNHASHABLE
+
+    # __dict__ attribute contains the instance attributes of the object.
+    # this is typically sufficient to define the object and its behavior, so it's a good target
+    # for a hash in the default case.
+    # Objects that return an empty dict should be skipped (very odd behavior, happens with pandas type)
+    if getattr(obj, "__dict__", {}) != {}:
+        return hash_value(obj.__dict__, depth=depth + 1)
+
+    # check if the object comes from a module part of the standard library
+    # if it's the case, hash it's __repr__(), which is a string representation of the object
+    # __repr__() from the standard library should be well-formed and offer a reliable basis
+    # for fingerprinting.
+    # for example, this will catch: pathlib.Path, enum.Enum, argparse.Namespace
+    elif getattr(obj, "__module__", False):
+        if obj.__module__.partition(".")[0] in sys.builtin_module_names:
+            return hash_repr(obj, depth=depth)
+
+    # cover the datetime module, which doesn't have a __module__ attribute
+    elif type(obj) in vars(datetime).values():
+        return hash_repr(obj, depth=depth)
+
+    return UNHASHABLE
+
+
+@hash_value.register(NoneType)
+def hash_none(obj, *args, **kwargs) -> str:
+    """Hash for None is <none>
+
+    Primitive type returns a hash and doesn't have to handle depth.
+    """
+    return NONE_HASH
+
+
+def hash_repr(obj, *args, **kwargs) -> str:
+    """Use the built-in repr() to get a string representation of the object
+    and hash it.
+
+    While `.__repr__()` might not be implemented for all classes, the function
+    `repr()` will handle it, along with exceptions, to always return a value.
+
+    Primitive type returns a hash and doesn't have to handle depth.
+    """
+    return hash_primitive(repr(obj))
+
+
+# we need to use explicit multiple registration because older Python
+# versions don't support type annotations with Union types
+@hash_value.register(str)
+@hash_value.register(int)
+@hash_value.register(float)
+@hash_value.register(bool)
+def hash_primitive(obj, *args, **kwargs) -> str:
+    """Convert the primitive to a string and hash it
+
+    Primitive type returns a hash and doesn't have to handle depth.
+    """
+    hash_object = hashlib.md5(str(obj).encode())
+    return _compact_hash(hash_object.digest())
+
+
+@hash_value.register(bytes)
+def hash_bytes(obj, *args, **kwargs) -> str:
+    """Convert the primitive to a string and hash it
+
+    Primitive type returns a hash and doesn't have to handle depth.
+    """
+    hash_object = hashlib.md5(obj)
+    return _compact_hash(hash_object.digest())
+
+
+@hash_value.register(Sequence)
+def hash_sequence(obj, *args, depth: int = 0, **kwargs) -> str:
+    """Hash each object of the sequence.
+
+    Orders matters for the hash since orders matters in a sequence.
+    """
+    hash_object = hashlib.sha224()
+    for elem in obj:
+        hash_object.update(hash_value(elem, depth=depth + 1).encode())
+
+    return _compact_hash(hash_object.digest())
+
+
+def hash_unordered_mapping(obj, *args, depth: int = 0, **kwargs) -> str:
+    """
+
+    When hashing an unordered mapping, the two following dict have the same hash.
+
+    .. code-block:: python
+
+        foo = {"key": 3, "key2": 13}
+        bar = {"key2": 13, "key": 3}
+
+        hash_mapping(foo) == hash_mapping(bar)
+    """
+
+    hashed_mapping: dict[str, str] = {}
+    for key, value in obj.items():
+        hashed_mapping[hash_value(key, depth=depth + 1)] = hash_value(value, depth=depth + 1)
+
+    hash_object = hashlib.sha224()
+    for key, value in sorted(hashed_mapping.items()):
+        hash_object.update(key.encode())
+        hash_object.update(value.encode())
+
+    return _compact_hash(hash_object.digest())
+
+
+@hash_value.register(Mapping)
+def hash_mapping(obj, *, ignore_order: bool = True, depth: int = 0, **kwargs) -> str:
+    """Hash each key then its value.
+
+    The mapping is always sorted first because order shouldn't matter
+    in a mapping.
+
+    NOTE Since Python 3.7, dictionary store insertion order. However, this
+    function assumes that they key order doesn't matter to uniquely identify
+    the dictionary.
+
+    .. code-block:: python
+
+        foo = {"key": 3, "key2": 13}
+        bar = {"key2": 13, "key": 3}
+
+        hash_mapping(foo) == hash_mapping(bar)
+
+    """
+    if ignore_order:
+        # use the same depth because we're simply dispatching to another implementation
+        return hash_unordered_mapping(obj, depth=depth)
+
+    hash_object = hashlib.sha224()
+    for key, value in obj.items():
+        hash_object.update(hash_value(key, depth=depth + 1).encode())
+        hash_object.update(hash_value(value, depth=depth + 1).encode())
+
+    return _compact_hash(hash_object.digest())
+
+
+@hash_value.register(Set)
+def hash_set(obj, *args, depth: int = 0, **kwargs) -> str:
+    """Hash each element of the set, then sort hashes, and
+    create a hash of hashes.
+
+    For the same objects in the set, the hashes will be the
+    same.
+    """
+    hashes = [hash_value(elem, depth=depth + 1) for elem in obj]
+    sorted_hashes = sorted(hashes)
+
+    hash_object = hashlib.sha224()
+    for hash in sorted_hashes:
+        hash_object.update(hash.encode())
+
+    return _compact_hash(hash_object.digest())
+
+
+@hash_value.register(h_databackends.AbstractPandasDataFrame)
+@hash_value.register(h_databackends.AbstractPandasColumn)
+def hash_pandas_obj(obj, *args, depth: int = 0, **kwargs) -> str:
+    """Convert a pandas dataframe, series, or index to
+    a dictionary of {index: row_hash} then hash it.
+
+    Given the hashing for mappings, the physical ordering or rows doesn't matter.
+    For example, if the index is a date, the hash will represent the {date: row_hash},
+    and won't preserve how dates were ordered in the DataFrame.
+    """
+    from pandas.util import hash_pandas_object
+
+    hash_per_row = hash_pandas_object(obj)
+    return hash_mapping(hash_per_row.to_dict(), ignore_order=False, depth=depth + 1)
+
+
+@hash_value.register(h_databackends.AbstractPolarsDataFrame)
+def hash_polars_dataframe(obj, *args, depth: int = 0, **kwargs) -> str:
+    """Convert a polars dataframe, series, or index to
+    a list of hashes then hash it.
+    """
+    hash_per_row = obj.hash_rows()
+    return hash_sequence(hash_per_row.to_list(), depth=depth + 1)
+
+
+@hash_value.register(h_databackends.AbstractPolarsColumn)
+def hash_polars_column(obj, *args, depth: int = 0, **kwargs) -> str:
+    """Promote the single Series to a dataframe and hash it"""
+    # use the same depth because we're simply dispatching to another implementation
+    return hash_polars_dataframe(obj.to_frame(), depth=depth)
+
+
+@hash_value.register(h_databackends.AbstractNumpyArray)
+def hash_numpy_array(obj, *args, depth: int = 0, **kwargs) -> str:
+    """Get the bytes representation of the array raw data and hash it.
+
+    Might not be ideal because different higher-level numpy objects could have
+    the same underlying array representation (e.g., masked arrays).
+    Unsure, but it's an area to investigate.
+    """
+    # use the same depth because we're simply dispatching to another implementation
+    return hash_bytes(obj.tobytes(), depth=depth)

hamilton/caching/stores/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

hamilton/caching/stores/base.py

@@ -0,0 +1,242 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import abc
+import pickle
+from collections.abc import Sequence
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from hamilton.htypes import custom_subclass_check
+from hamilton.io.data_adapters import DataLoader, DataSaver
+from hamilton.registry import LOADER_REGISTRY, SAVER_REGISTRY
+
+
+class ResultRetrievalError(Exception):
+    """Raised by the SmartCacheAdapter when ResultStore.get() fails."""
+
+
+# TODO Currently, this check is done when data needs to be saved.
+# Ideally, it would be done earlier in the caching lifecycle.
+def search_data_adapter_registry(
+    name: str, type_: type
+) -> tuple[type[DataSaver], type[DataLoader]]:
+    """Find pair of DataSaver and DataLoader registered with `name` and supporting `type_`"""
+    if name not in SAVER_REGISTRY or name not in LOADER_REGISTRY:
+        raise KeyError(
+            f"{name} isn't associated to both a DataLoader and a DataSaver. "
+            "Default saver/loader pairs include `json`, `file`, `pickle`, `parquet`, `csv`, "
+            "`feather`, `orc`, `excel`. More pairs may be available through plugins."
+        )
+
+    try:
+        saver_cls = next(
+            saver_cls
+            for saver_cls in SAVER_REGISTRY[name]
+            if any(
+                custom_subclass_check(type_, applicable_type)
+                for applicable_type in saver_cls.applicable_types()
+            )
+        )
+    except StopIteration as e:
+        raise KeyError(f"{name} doesn't have any DataSaver supporting type {type_}") from e
+
+    try:
+        loader_cls = next(
+            loader_cls
+            for loader_cls in LOADER_REGISTRY[name]
+            if any(
+                custom_subclass_check(type_, applicable_type)
+                for applicable_type in loader_cls.applicable_types()
+            )
+        )
+    except StopIteration as e:
+        raise KeyError(f"{name} doesn't have any DataLoader supporting type {type_}") from e
+
+    return saver_cls, loader_cls
+
+
+class ResultStore(abc.ABC):
+    @abc.abstractmethod
+    def set(self, data_version: str, result: Any, **kwargs) -> None:
+        """Store ``result`` keyed by ``data_version``."""
+
+    @abc.abstractmethod
+    def get(self, data_version: str, **kwargs) -> Any | None:
+        """Try to retrieve ``result`` keyed by ``data_version``.
+        If retrieval misses, return ``None``.
+        """
+
+    @abc.abstractmethod
+    def delete(self, data_version: str) -> None:
+        """Delete ``result`` keyed by ``data_version``."""
+
+    @abc.abstractmethod
+    def delete_all(self) -> None:
+        """Delete all stored results."""
+
+    @abc.abstractmethod
+    def exists(self, data_version: str) -> bool:
+        """boolean check if a ``result`` is found for ``data_version``
+        If True, ``.get()`` should successfully retrieve the ``result``.
+        """
+
+
+class MetadataStore(abc.ABC):
+    @abc.abstractmethod
+    def __len__(self) -> int:
+        """Return the number of cache_keys in the metadata store"""
+
+    @abc.abstractmethod
+    def initialize(self, run_id: str) -> None:
+        """Setup the metadata store and log the start of the run"""
+
+    @abc.abstractmethod
+    def set(self, cache_key: str, data_version: str, **kwargs) -> Any | None:
+        """Store the mapping ``cache_key -> data_version``.
+        Can include other metadata (e.g., node name, run id, code version) depending
+        on the implementation.
+        """
+
+    @abc.abstractmethod
+    def get(self, cache_key: str, **kwargs) -> str | None:
+        """Try to retrieve ``data_version`` keyed by ``cache_key``.
+        If retrieval misses return ``None``.
+        """
+
+    @abc.abstractmethod
+    def delete(self, cache_key: str) -> None:
+        """Delete ``data_version`` keyed by ``cache_key``."""
+
+    @abc.abstractmethod
+    def delete_all(self) -> None:
+        """Delete all stored metadata."""
+
+    @abc.abstractmethod
+    def exists(self, cache_key: str) -> bool:
+        """boolean check if a ``data_version`` is found for ``cache_key``
+        If True, ``.get()`` should successfully retrieve the ``data_version``.
+        """
+
+    @abc.abstractmethod
+    def get_run_ids(self) -> Sequence[str]:
+        """Return a list of run ids, sorted from oldest to newest start time.
+        A ``run_id`` is registered when the metadata_store ``.initialize()`` is called.
+        """
+
+    @abc.abstractmethod
+    def get_run(self, run_id: str) -> Sequence[dict]:
+        """Return a list of node metadata associated with a run.
+
+        For each node, the metadata should include ``cache_key`` (created or used)
+        and ``data_version``. These values allow to manually query the MetadataStore
+        or ResultStore.
+
+        Decoding the ``cache_key`` gives the ``node_name``, ``code_version``, and
+        ``dependencies_data_versions``. Individual implementations may add more
+        information or decode the ``cache_key`` before returning metadata.
+        """
+
+    @property
+    def size(self) -> int:
+        """Number of unique entries (i.e., cache_keys) in the metadata_store"""
+        return self.__len__()
+
+    @property
+    def last_run_id(self) -> str:
+        """Return"""
+        return self.get_run_ids()[-1]
+
+    def get_last_run(self) -> Any:
+        """Return the metadata from the last started run."""
+        return self.get_run(self.last_run_id)
+
+
+# TODO refactor the association between StoredResult, MetadataStore, and ResultStore
+# to load data using the `DataLoader` class and kwargs instead of pickling the instantiated
+# DataLoader object. This would be safer across Hamilton versions.
+class StoredResult:
+    def __init__(
+        self,
+        value: Any,
+        expires_at=None,
+        saver=None,
+        loader=None,
+    ):
+        self.value = value
+        self.expires_at = expires_at
+        self.saver = saver
+        self.loader = loader
+
+    @classmethod
+    def new(
+        cls,
+        value: Any,
+        expires_in: timedelta | None = None,
+        saver: DataSaver | None = None,
+        loader: DataLoader | None = None,
+    ) -> "StoredResult":
+        if expires_in is not None and not isinstance(expires_in, timedelta):
+            expires_in = timedelta(seconds=expires_in)
+
+        # != operator on boolean is XOR
+        if bool(saver is not None) != bool(loader is not None):
+            raise ValueError(
+                "Must pass both `saver` and `loader` or neither. Currently received: "
+                f"`saver`: `{saver}`; `loader`: `{loader}`"
+            )
+
+        return cls(
+            value=value,
+            expires_at=(datetime.now(tz=timezone.utc) + expires_in) if expires_in else None,
+            saver=saver,
+            loader=loader,
+        )
+
+    @property
+    def expired(self) -> bool:
+        return self.expires_at is not None and datetime.now(tz=timezone.utc) >= self.expires_at
+
+    @property
+    def expires_in(self) -> int:
+        if self.expires_at:
+            return int(self.expires_at.timestamp() - datetime.now(tz=timezone.utc).timestamp())
+
+        return -1
+
+    def save(self) -> bytes:
+        """Receives pickleable data or DataLoader to use to load the real data"""
+        if self.saver is not None:
+            self.saver.save_data(data=self.value)
+            to_pickle = self.loader
+        else:
+            to_pickle = self.value
+
+        return pickle.dumps(to_pickle)
+
+    @classmethod
+    def load(cls, raw: bytes) -> "StoredResult":
+        """Reads the raw bytes from disk and sets `StoredResult.data`"""
+        loaded = pickle.loads(raw)
+        if isinstance(loaded, DataLoader):
+            loader = loaded
+            result, metadata = loader.load_data(None)
+        else:
+            loader = None
+            result = loaded
+
+        return StoredResult.new(value=result)