checkpointer 2.5.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

checkpointer/checkpoint.py

@@ -23,7 +23,7 @@ class CheckpointerOpts(TypedDict, total=False):
   root_path: Path | str | None
   when: bool
   verbosity: Literal[0, 1]
-  path: Callable[..., str] | None
+  hash_by: Callable | None
   should_expire: Callable[[datetime], bool] | None
   capture: bool
 
@@ -33,7 +33,7 @@ class Checkpointer:
     self.root_path = Path(opts.get("root_path", DEFAULT_DIR) or ".")
     self.when = opts.get("when", True)
     self.verbosity = opts.get("verbosity", 1)
-    self.path = opts.get("path")
+    self.hash_by = opts.get("hash_by")
     self.should_expire = opts.get("should_expire")
     self.capture = opts.get("capture", False)
 
@@ -66,7 +66,7 @@ class CheckpointFn(Generic[Fn]):
     store_format = self.checkpointer.format
     Storage = STORAGE_MAP[store_format] if isinstance(store_format, str) else store_format
     deep_hashes = [child._set_ident().fn_hash_raw for child in iterate_checkpoint_fns(self)]
-    self.fn_hash = str(ObjectHash().update_hash(self.fn_hash_raw, iter=deep_hashes))
+    self.fn_hash = str(ObjectHash().write_text(self.fn_hash_raw, iter=deep_hashes))
     self.fn_subdir = f"{fn_file}/{fn_name}/{self.fn_hash[:16]}"
     self.is_async = inspect.iscoroutinefunction(wrapped)
     self.storage = Storage(self)
@@ -88,13 +88,9 @@ class CheckpointFn(Generic[Fn]):
       pointfn._lazyinit()
 
   def get_checkpoint_id(self, args: tuple, kw: dict) -> str:
-    if not callable(self.checkpointer.path):
-      call_hash = ObjectHash(self.fn_hash, args, kw, digest_size=16)
-      return f"{self.fn_subdir}/{call_hash}"
-    checkpoint_id = self.checkpointer.path(*args, **kw)
-    if not isinstance(checkpoint_id, str):
-      raise CheckpointError(f"path function must return a string, got {type(checkpoint_id)}")
-    return checkpoint_id
+    hash_params = [self.checkpointer.hash_by(*args, **kw)] if self.checkpointer.hash_by else (args, kw)
+    call_hash = ObjectHash(self.fn_hash, *hash_params, digest_size=16)
+    return f"{self.fn_subdir}/{call_hash}"
 
   async def _store_on_demand(self, args: tuple, kw: dict, rerun: bool):
     checkpoint_id = self.get_checkpoint_id(args, kw)

checkpointer/fn_ident.py

@@ -11,7 +11,7 @@ from .utils import AttrDict, distinct, get_cell_contents, iterate_and_upcoming,
 cwd = Path.cwd()
 
 def is_class(obj) -> TypeGuard[Type]:
-  # isinstance works too, but needlessly triggers __getattribute__
+  # isinstance works too, but needlessly triggers _lazyinit()
   return issubclass(type(obj), type)
 
 def extract_classvars(code: CodeType, scope_vars: AttrDict) -> dict[str, dict[str, Type]]:

checkpointer/object_hash.py

@@ -3,7 +3,7 @@ import hashlib
 import io
 import re
 from collections.abc import Iterable
-from contextlib import nullcontext
+from contextlib import nullcontext, suppress
 from decimal import Decimal
 from itertools import chain
 from pickle import HIGHEST_PROTOCOL as PROTOCOL
@@ -11,19 +11,18 @@ from types import BuiltinFunctionType, FunctionType, GeneratorType, MethodType,
 from typing import Any, TypeAliasType, TypeVar
 from .utils import ContextVar, get_fn_body
 
-try:
+np, torch = None, None
+
+with suppress(Exception):
   import numpy as np
-except:
-  np = None
-try:
+
+with suppress(Exception):
   import torch
-except:
-  torch = None
 
 def encode_type(t: type | FunctionType) -> str:
   return f"{t.__module__}:{t.__qualname__}"
 
-def encode_val(v: Any) -> str:
+def encode_type_of(v: Any) -> str:
   return encode_type(type(v))
 
 class ObjectHashError(Exception):
@@ -32,11 +31,11 @@ class ObjectHashError(Exception):
     self.obj = obj
 
 class ObjectHash:
-  def __init__(self, *obj: Any, iter: Iterable[Any] = [], digest_size=64, tolerate_errors=False) -> None:
+  def __init__(self, *objs: Any, iter: Iterable[Any] = (), digest_size=64, tolerate_errors=False) -> None:
     self.hash = hashlib.blake2b(digest_size=digest_size)
     self.current: dict[int, int] = {}
     self.tolerate_errors = ContextVar(tolerate_errors)
-    self.update(iter=chain(obj, iter))
+    self.update(iter=chain(objs, iter))
 
   def copy(self) -> "ObjectHash":
     new = ObjectHash(tolerate_errors=self.tolerate_errors.value)
@@ -48,15 +47,21 @@ class ObjectHash:
 
   __str__ = hexdigest
 
-  def update_hash(self, *data: bytes | str, iter: Iterable[bytes | str] = []) -> "ObjectHash":
+  def nested_hash(self, *objs: Any) -> str:
+    return ObjectHash(iter=objs, tolerate_errors=self.tolerate_errors.value).hexdigest()
+
+  def write_bytes(self, *data: bytes, iter: Iterable[bytes] = ()) -> "ObjectHash":
     for d in chain(data, iter):
-      self.hash.update(d.encode() if isinstance(d, str) else d)
+      self.hash.update(d)
     return self
 
+  def write_text(self, *data: str, iter: Iterable[str] = ()) -> "ObjectHash":
+    return self.write_bytes(iter=(d.encode() for d in chain(data, iter)))
+
   def header(self, *args: Any) -> "ObjectHash":
-    return self.update_hash(":".join(map(str, args)))
+    return self.write_bytes(":".join(map(str, args)).encode())
 
-  def update(self, *objs: Any, iter: Iterable[Any] = [], tolerate_errors: bool | None=None) -> "ObjectHash":
+  def update(self, *objs: Any, iter: Iterable[Any] = (), tolerate_errors: bool | None=None) -> "ObjectHash":
     with nullcontext() if tolerate_errors is None else self.tolerate_errors.set(tolerate_errors):
       for obj in chain(objs, iter):
         try:
@@ -74,19 +79,20 @@ class ObjectHash:
         self.header("null")
 
       case bool() | int() | float() | complex() | Decimal() | ObjectHash():
-        self.header("number", encode_val(obj), obj)
+        self.header("number", encode_type_of(obj), obj)
 
       case str() | bytes() | bytearray() | memoryview():
-        self.header("bytes", encode_val(obj), len(obj)).update_hash(obj)
+        b = obj.encode() if isinstance(obj, str) else obj
+        self.header("bytes", encode_type_of(obj), len(b)).write_bytes(b)
 
       case set() | frozenset():
-        self.header("set", encode_val(obj), len(obj))
         try:
           items = sorted(obj)
+          header = "set"
         except:
-          self.header("unsortable")
-          items = sorted(str(ObjectHash(item, tolerate_errors=self.tolerate_errors.value)) for item in obj)
-        self.update(iter=items)
+          items = sorted(map(self.nested_hash, obj))
+          header = "set-unsortable"
+        self.header(header, encode_type_of(obj), len(obj)).update(iter=items)
 
       case TypeVar():
         self.header("TypeVar").update(obj.__name__, obj.__bound__, obj.__constraints__, obj.__contravariant__, obj.__covariant__)
@@ -113,7 +119,7 @@ class ObjectHash:
         self.header("generator", obj.__qualname__)._update_iterator(obj)
 
       case io.TextIOWrapper() | io.FileIO() | io.BufferedRandom() | io.BufferedWriter() | io.BufferedReader():
-        self.header("file", encode_val(obj)).update(obj.name, obj.mode, obj.tell())
+        self.header("file", encode_type_of(obj)).update(obj.name, obj.mode, obj.tell())
 
       case type():
         self.header("type", encode_type(obj))
@@ -122,20 +128,20 @@ class ObjectHash:
         self.header("dtype").update(obj.__class__, obj.descr)
 
       case _ if np and isinstance(obj, np.ndarray):
-        self.header("ndarray", encode_val(obj), obj.shape, obj.strides).update(obj.dtype)
+        self.header("ndarray", encode_type_of(obj), obj.shape, obj.strides).update(obj.dtype)
         if obj.dtype.hasobject:
           self.update(obj.__reduce_ex__(PROTOCOL))
         else:
           array = np.ascontiguousarray(obj if obj.base is None else obj.base).view(np.uint8)
-          self.update_hash(array.data)
+          self.write_bytes(array.data)
 
       case _ if torch and isinstance(obj, torch.Tensor):
-        self.header("tensor", encode_val(obj), str(obj.dtype), tuple(obj.shape), obj.stride(), str(obj.device))
+        self.header("tensor", encode_type_of(obj), obj.dtype, tuple(obj.shape), obj.stride(), obj.device)
         if obj.device.type != "cpu":
           obj = obj.cpu()
         storage = obj.storage()
-        buffer = (ctypes.c_ubyte * (storage.nbytes())).from_address(storage.data_ptr())
-        self.update_hash(memoryview(buffer))
+        buffer = (ctypes.c_ubyte * storage.nbytes()).from_address(storage.data_ptr())
+        self.write_bytes(memoryview(buffer))
 
       case _ if id(obj) in self.current:
         self.header("circular", self.current[id(obj)])
@@ -145,36 +151,36 @@ class ObjectHash:
           self.current[id(obj)] = len(self.current)
           match obj:
             case list() | tuple():
-              self.header("list", encode_val(obj), len(obj)).update(iter=obj)
+              self.header("list", encode_type_of(obj), len(obj)).update(iter=obj)
             case dict():
               try:
                 items = sorted(obj.items())
+                header = "dict"
               except:
-                items = sorted((str(ObjectHash(key, tolerate_errors=self.tolerate_errors.value)), val) for key, val in obj.items())
-              self.header("dict", encode_val(obj), len(obj)).update(iter=chain.from_iterable(items))
+                items = sorted((self.nested_hash(key), val) for key, val in obj.items())
+                header = "dict-unsortable"
+              self.header(header, encode_type_of(obj), len(obj)).update(iter=chain.from_iterable(items))
             case _:
               self._update_object(obj)
         finally:
           del self.current[id(obj)]
 
-  def _update_iterator(self, obj: Iterable) -> None:
-    self.header("iterator", encode_val(obj)).update(iter=obj).header(b"iterator-end")
+  def _update_iterator(self, obj: Iterable) -> "ObjectHash":
+    return self.header("iterator", encode_type_of(obj)).update(iter=obj).header("iterator-end")
 
   def _update_object(self, obj: object) -> "ObjectHash":
-    self.header("instance", encode_val(obj))
-    try:
-      reduced = obj.__reduce_ex__(PROTOCOL) if hasattr(obj, "__reduce_ex__") else obj.__reduce__()
-    except:
-      reduced = None
+    self.header("instance", encode_type_of(obj))
+    reduced = None
+    with suppress(Exception):
+      reduced = obj.__reduce_ex__(PROTOCOL)
+    with suppress(Exception):
+      reduced = reduced or obj.__reduce__()
     if isinstance(reduced, str):
       return self.header("reduce-str").update(reduced)
     if reduced:
       reduced = list(reduced)
       it = reduced.pop(3) if len(reduced) >= 4 else None
-      self.header("reduce").update(reduced)
-      if it is not None:
-        self._update_iterator(it)
-      return self
+      return self.header("reduce").update(reduced)._update_iterator(it or ())
     if state := hasattr(obj, "__getstate__") and obj.__getstate__():
       return self.header("getstate").update(state)
     if len(getattr(obj, "__slots__", [])):

checkpointer/test_checkpointer.py

@@ -83,17 +83,6 @@ async def test_async_caching():
 
   assert result1 == result2 == 9
 
-def test_custom_path_caching():
-  def custom_path(a, b):
-    return f"add/{a}-{b}"
-
-  @checkpoint(path=custom_path)
-  def add(a, b):
-    return a + b
-
-  add(3, 4)
-  assert (checkpoint.root_path / "add/3-4.pkl").exists()
-
 def test_force_recalculation():
   @checkpoint
   def square(x: int) -> int:

{checkpointer-2.5.0.dist-info → checkpointer-2.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: checkpointer
-Version: 2.5.0
+Version: 2.6.0
 Summary: A Python library for memoizing function results with support for multiple storage backends, async runtimes, and automatic cache invalidation
 Project-URL: Repository, https://github.com/Reddan/checkpointer.git
 Author: Hampus Hallman
@@ -12,6 +12,8 @@ License: Copyright 2018-2025 Hampus Hallman
         
         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 License-File: LICENSE
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 
@@ -28,6 +30,7 @@ Adding or removing `@checkpoint` doesn't change how your code works. You can app
 - ⏲️ **Custom Expiration Logic**: Automatically invalidate old checkpoints.
 - 📂 **Flexible Path Configuration**: Control where checkpoints are stored.
 - 📦 **Captured Variables Handling**: Optionally include captured variables in cache invalidation.
+- ⚡ **Custom Argument Hashing**: Override argument hashing for speed or specialized hashing logic.
 
 ---
 
@@ -61,9 +64,9 @@ When you use `@checkpoint`, the function's **arguments** (`args`, `kwargs`) are
 
 Additionally, `checkpointer` ensures that caches are invalidated when a function's implementation or any of its dependencies change. Each function is assigned a hash based on:
 
-1. **Its source code**: Changes to the function's code update its hash.
-2. **Dependent functions**: If a function calls others, changes in those dependencies will also update the hash.
-3. **Captured variables**: (Optional) If `capture=True`, changes to captured variables and global variables will also update the hash.
+1. **Function Code**: The hash updates when the function’s own source code changes.
+2. **Dependencies**: If the function calls other user-defined functions, changes in those dependencies also update the hash.
+3. **External Variables** *(with `capture=True`)*: Any global or closure-based variables used by the function are included in its hash, so changes to those variables also trigger cache invalidation.
 
 ### Example: Cache Invalidation
 
@@ -155,21 +158,19 @@ stored_result = expensive_function.get(4)
 
 ### Refresh Function Hash
 
-When using `capture=True`, changes to captured variables are included in the function's hash to determine cache invalidation. However, `checkpointer` does not automatically update this hash during a running Python session—it recalculates between sessions or when you explicitly refresh it.
-
-Use the `reinit` method to manually refresh the function's hash within the same session:
+If `capture=True`, you might need to re-hash a function during the same Python session. For that, call `reinit`:
 
 ```python
 expensive_function.reinit()
 ```
 
-This forces `checkpointer` to recalculate the hash of `expensive_function`, considering any changes to captured variables. It's useful when you've modified external variables that the function depends on and want the cache to reflect these changes immediately.
+This tells `checkpointer` to recalculate the function hash, reflecting changes in captured variables.
 
 ---
 
 ## Storage Backends
 
-`checkpointer` works with both built-in and custom storage backends, so you can use what's provided or roll your own as needed.
+`checkpointer` works with built-in and custom storage backends, so you can use what's provided or roll your own as needed.
 
 ### Built-In Backends
 
@@ -201,10 +202,10 @@ from checkpointer import checkpoint, Storage
 from datetime import datetime
 
 class CustomStorage(Storage):
-    def store(self, path, data): ...  # Save the checkpoint data
-    def exists(self, path) -> bool: ...  # Check if a checkpoint exists at the given path
-    def checkpoint_date(self, path) -> datetime: ...  # Return the date the checkpoint was created
-    def load(self, path): ...  # Return the checkpoint data
+    def exists(self, path) -> bool: ...  # Check if a checkpoint exists
+    def checkpoint_date(self, path) -> datetime: ...  # Get the checkpoint's timestamp
+    def store(self, path, data): ...  # Save data to the checkpoint
+    def load(self, path): ...  # Load data from the checkpoint
     def delete(self, path): ...  # Delete the checkpoint
 
 @checkpoint(format=CustomStorage)
@@ -212,21 +213,21 @@ def custom_cached(x: int):
     return x ** 2
 ```
 
-Using a custom backend lets you tailor storage to your application, whether it involves databases, cloud storage, or custom file formats.
+Use a custom backend to integrate with databases, cloud storage, or specialized file formats.
 
 ---
 
 ## Configuration Options ⚙️
 
-| Option          | Type                              | Default              | Description                                    |
-|-----------------|-----------------------------------|----------------------|------------------------------------------------|
-| `capture`       | `bool`                            | `False`              | Include captured variables in function hashes. |
-| `format`        | `"pickle"`, `"memory"`, `Storage` | `"pickle"`           | Storage backend format.                        |
-| `root_path`     | `Path`, `str`, or `None`          | ~/.cache/checkpoints | Root directory for storing checkpoints.        |
-| `when`          | `bool`                            | `True`               | Enable or disable checkpointing.               |
-| `verbosity`     | `0` or `1`                        | `1`                  | Logging verbosity.                             |
-| `path`          | `Callable[..., str]`              | `None`               | Custom path for checkpoint storage.            |
-| `should_expire` | `Callable[[datetime], bool]`      | `None`               | Custom expiration logic.                       |
+| Option          | Type                                | Default              | Description                                               |
+|-----------------|-------------------------------------|----------------------|-----------------------------------------------------------|
+| `capture`       | `bool`                              | `False`              | Include captured variables in function hashes.            |
+| `format`        | `"pickle"`, `"memory"`, `Storage`   | `"pickle"`           | Storage backend format.                                   |
+| `root_path`     | `Path`, `str`, or `None`            | ~/.cache/checkpoints | Root directory for storing checkpoints.                   |
+| `when`          | `bool`                              | `True`               | Enable or disable checkpointing.                          |
+| `verbosity`     | `0` or `1`                          | `1`                  | Logging verbosity.                                        |
+| `should_expire` | `Callable[[datetime], bool]`        | `None`               | Custom expiration logic.                                  |
+| `hash_by`       | `Callable[..., Any]`                | `None`               | Custom function that transforms arguments before hashing. |
 
 ---
 

{checkpointer-2.5.0.dist-info → checkpointer-2.6.0.dist-info}/RECORD

@@ -1,16 +1,16 @@
 checkpointer/__init__.py,sha256=ZJ6frUNgkklUi85b5uXTyTfRzMvZgQOJY-ZOnu7jh78,777
-checkpointer/checkpoint.py,sha256=FeizwZf0r6j_xy8EOyDKXqcfCNNZnUYBziVbxPu9kwE,6284
-checkpointer/fn_ident.py,sha256=_GyIfoUvEpjZ4dUAa04NK4pJdSmDAXuufjt7z2xQP8w,4316
-checkpointer/object_hash.py,sha256=ekpKXtbKtHBl5e9s-uyng8qOHTFl9CCT6QHlQTZTQn8,6860
+checkpointer/checkpoint.py,sha256=2CAlNPnMLeou8SwHWeIjJ4kQhSqnpM9EBP_f-cYdSlE,6123
+checkpointer/fn_ident.py,sha256=TEM_SdjzQ5OgRJnqNgyYZZKONx7tM7Dk4bNM5TB_RyY,4311
+checkpointer/object_hash.py,sha256=o7Qr9VViWPmG1Fy1S3RediCagRydEVgTVZrZ9nRN54E,7215
 checkpointer/print_checkpoint.py,sha256=21aeqgM9CMjNAJyScqFmXCWWfh3jBIn7o7i5zJkZGaA,1369
-checkpointer/test_checkpointer.py,sha256=qpG_p4DVMlOBxt71v93-GTKve08EQaExFAf6xSv0wUg,3821
+checkpointer/test_checkpointer.py,sha256=uJ2Pg9Miq1W0l28eNlRhMjuT_R8c-ygYwp3KP3VW8Os,3600
 checkpointer/utils.py,sha256=Rvm2NaJHtPTusM7fyHz_w9HUy_fqQfx8S1fr5CBWGL0,3047
 checkpointer/storages/__init__.py,sha256=Kl4Og5jhYxn6m3tB_kTMsabf4_eWVLmFVAoC-pikNQE,301
 checkpointer/storages/bcolz_storage.py,sha256=3QkSUSeG5s2kFuVV_LZpzMn1A5E7kqC7jk7w35c0NyQ,2314
 checkpointer/storages/memory_storage.py,sha256=S5ayOZE_CyaFQJ-vSgObTanldPzG3gh3NksjNAc7vsk,1282
 checkpointer/storages/pickle_storage.py,sha256=lJ0ton9ib3eifiny8XtPSNsx-w4Cm8oYUlbmKob34xU,1554
 checkpointer/storages/storage.py,sha256=_m18Z8TKrdAbi6YYYQmuNOnhna4RB2sJDn1v3liaU3U,721
-checkpointer-2.5.0.dist-info/METADATA,sha256=yd3D8zAWdCe4011MLBwknwDvSYx3LGGSmk6AzDXmMUg,10647
-checkpointer-2.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-checkpointer-2.5.0.dist-info/licenses/LICENSE,sha256=9xVsdtv_-uSyY9Xl9yujwAPm4-mjcCLeVy-ljwXEWbo,1059
-checkpointer-2.5.0.dist-info/RECORD,,
+checkpointer-2.6.0.dist-info/METADATA,sha256=RWzn1RNHN76iF4hNkomVLouWvtjHtOody5doC4ajRk8,10556
+checkpointer-2.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+checkpointer-2.6.0.dist-info/licenses/LICENSE,sha256=9xVsdtv_-uSyY9Xl9yujwAPm4-mjcCLeVy-ljwXEWbo,1059
+checkpointer-2.6.0.dist-info/RECORD,,