checkpointer 2.12.0__py3-none-any.whl → 2.14.0__py3-none-any.whl

checkpointer/__init__.py

@@ -1,5 +1,6 @@
 import gc
 import tempfile
+from datetime import timedelta
 from typing import Callable
 from .checkpoint import CachedFunction, Checkpointer, CheckpointError, FunctionIdent
 from .object_hash import ObjectHash
@@ -8,8 +9,8 @@ from .types import AwaitableValue, Captured, CapturedOnce, CaptureMe, CaptureMeO
 
 checkpoint = Checkpointer()
 capture_checkpoint = Checkpointer(capture=True)
-memory_checkpoint = Checkpointer(format="memory", verbosity=0)
-tmp_checkpoint = Checkpointer(root_path=f"{tempfile.gettempdir()}/checkpoints")
+memory_checkpoint = Checkpointer(storage="memory", verbosity=0)
+tmp_checkpoint = Checkpointer(directory=f"{tempfile.gettempdir()}/checkpoints")
 static_checkpoint = Checkpointer(fn_hash_from=())
 
 def cleanup_all(invalidated=True, expired=True):

checkpointer/checkpoint.py

@@ -1,19 +1,19 @@
 from __future__ import annotations
 import re
-from datetime import datetime
+from datetime import datetime, timedelta
 from functools import cached_property, update_wrapper
 from inspect import Parameter, iscoroutine, signature, unwrap
-from itertools import chain
 from pathlib import Path
 from typing import (
   Callable, Concatenate, Coroutine, Generic, Iterable,
-  Literal, Self, Type, TypedDict, Unpack, cast, overload,
+  Literal, Self, Type, TypedDict, Unpack, overload,
 )
 from .fn_ident import Capturable, RawFunctionIdent, get_fn_ident
 from .object_hash import ObjectHash
 from .print_checkpoint import print_checkpoint
 from .storages import STORAGE_MAP, Storage, StorageType
 from .types import AwaitableValue, C, Coro, Fn, P, R, hash_by_from_annotation
+from .utils import flatten
 
 DEFAULT_DIR = Path.home() / ".cache/checkpoints"
 
@@ -21,21 +21,21 @@ class CheckpointError(Exception):
   pass
 
 class CheckpointerOpts(TypedDict, total=False):
-  format: Type[Storage] | StorageType
-  root_path: Path | str | None
+  storage: Type[Storage] | StorageType
+  directory: Path | str | None
   when: bool
   verbosity: Literal[0, 1, 2]
-  should_expire: Callable[[datetime], bool] | None
+  expiry: Callable[[datetime], bool] | timedelta | None
   capture: bool
   fn_hash_from: object
 
 class Checkpointer:
   def __init__(self, **opts: Unpack[CheckpointerOpts]):
-    self.format = opts.get("format", "pickle")
-    self.root_path = Path(opts.get("root_path", DEFAULT_DIR) or ".")
+    self.storage = opts.get("storage", "pickle")
+    self.directory = Path(opts.get("directory", DEFAULT_DIR) or ".")
     self.when = opts.get("when", True)
     self.verbosity = opts.get("verbosity", 1)
-    self.should_expire = opts.get("should_expire")
+    self.expiry = opts.get("expiry")
     self.capture = opts.get("capture", False)
     self.fn_hash_from = opts.get("fn_hash_from")
 
@@ -56,24 +56,46 @@ class FunctionIdent:
   Separated from CachedFunction to prevent hash desynchronization
   among bound instances when `.reinit()` is called.
   """
-  def __init__(self, cached_fn: CachedFunction):
-    self.__dict__.clear()
+  __slots__ = (
+    "checkpointer", "cached_fn", "fn", "fn_dir", "pos_names",
+    "arg_names", "default_args", "hash_by_map", "__dict__",
+  )
+
+  def __init__(self, cached_fn: CachedFunction, checkpointer: Checkpointer, fn: Callable):
+    wrapped = unwrap(fn)
+    fn_file = Path(wrapped.__code__.co_filename).name
+    fn_name = re.sub(r"[^\w.]", "", wrapped.__qualname__)
+    params = list(signature(wrapped).parameters.values())
+    pos_param_types = (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD)
+    named_param_types = (Parameter.KEYWORD_ONLY,) + pos_param_types
+    name_by_kind = {Parameter.VAR_POSITIONAL: b"*", Parameter.VAR_KEYWORD: b"**"}
+    self.checkpointer = checkpointer
     self.cached_fn = cached_fn
+    self.fn = fn
+    self.fn_dir = f"{fn_file}/{fn_name}"
+    self.pos_names = [param.name for param in params if param.kind in pos_param_types]
+    self.arg_names = {param.name for param in params if param.kind in named_param_types}
+    self.default_args = {param.name: param.default for param in params if param.default is not Parameter.empty}
+    self.hash_by_map = {
+      name_by_kind.get(param.kind, param.name): hash_by
+      for param in params
+      if (hash_by := hash_by_from_annotation(param.annotation))
+    }
 
   def reset(self):
-    self.__init__(self.cached_fn)
+    self.__dict__.clear()
 
   def is_static(self) -> bool:
-    return self.cached_fn.checkpointer.fn_hash_from is not None
+    return self.checkpointer.fn_hash_from is not None
 
   @cached_property
   def raw_ident(self) -> RawFunctionIdent:
-    return get_fn_ident(unwrap(self.cached_fn.fn), self.cached_fn.checkpointer.capture)
+    return get_fn_ident(unwrap(self.fn), self.checkpointer.capture)
 
   @cached_property
   def fn_hash(self) -> str:
     if self.is_static():
-      return str(ObjectHash(self.cached_fn.checkpointer.fn_hash_from, digest_size=16))
+      return str(ObjectHash(self.checkpointer.fn_hash_from, digest_size=16))
     depends = self.deep_idents(past_static=False)
     deep_hashes = [d.fn_hash if d.is_static() else d.raw_ident.fn_hash for d in depends]
     return str(ObjectHash(digest_size=16).write_text(iter=deep_hashes))
@@ -105,29 +127,21 @@ class FunctionIdent:
 
 class CachedFunction(Generic[Fn]):
   def __init__(self, checkpointer: Checkpointer, fn: Fn):
-    wrapped = unwrap(fn)
-    fn_file = Path(wrapped.__code__.co_filename).name
-    fn_name = re.sub(r"[^\w.]", "", wrapped.__qualname__)
-    Storage = STORAGE_MAP[checkpointer.format] if isinstance(checkpointer.format, str) else checkpointer.format
-    update_wrapper(cast(Callable, self), wrapped)
-    self.checkpointer = checkpointer
-    self.fn = fn
-    self.fn_dir = f"{fn_file}/{fn_name}"
+    store_format = checkpointer.storage
+    Storage = STORAGE_MAP[store_format] if isinstance(store_format, str) else store_format
+    update_wrapper(self, unwrap(fn))  # type: ignore
+    self.ident = FunctionIdent(self, checkpointer, fn)
     self.storage = Storage(self)
-    self.cleanup = self.storage.cleanup
     self.bound = ()
 
-    params = list(signature(wrapped).parameters.values())
-    pos_params = (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD)
-    self.arg_names = [param.name for param in params if param.kind in pos_params]
-    self.default_args = {param.name: param.default for param in params if param.default is not Parameter.empty}
-    self.hash_by_map = get_hash_by_map(params)
-    self.ident = FunctionIdent(self)
-
   @overload
   def __get__(self: Self, instance: None, owner: Type[C]) -> Self: ...
   @overload
-  def __get__(self: CachedFunction[Callable[Concatenate[C, P], R]], instance: C, owner: Type[C]) -> CachedFunction[Callable[P, R]]: ...
+  def __get__(
+    self: CachedFunction[Callable[Concatenate[C, P], R]],
+    instance: C,
+    owner: Type[C],
+  ) -> CachedFunction[Callable[P, R]]: ...
   def __get__(self, instance, owner):
     if instance is None:
       return self
@@ -137,62 +151,65 @@ class CachedFunction(Generic[Fn]):
     return bound_fn
 
   @property
-  def depends(self) -> list[Callable]:
-    return self.ident.raw_ident.depends
+  def fn(self) -> Fn:
+    return self.ident.fn  # type: ignore
 
-  def reinit(self, recursive=False) -> CachedFunction[Fn]:
+  @property
+  def cleanup(self):
+    return self.storage.cleanup
+
+  def reinit(self, recursive=True) -> CachedFunction[Fn]:
     depend_idents = list(self.ident.deep_idents()) if recursive else [self.ident]
     for ident in depend_idents: ident.reset()
     for ident in depend_idents: ident.fn_hash
     return self
 
   def _get_call_hash(self, args: tuple, kw: dict[str, object]) -> str:
+    ident = self.ident
     args = self.bound + args
-    pos_args = args[len(self.arg_names):]
-    named_pos_args = dict(zip(self.arg_names, args))
-    named_args = {**self.default_args, **named_pos_args, **kw}
-    if hash_by_map := self.hash_by_map:
-      rest_hash_by = hash_by_map.get(b"**")
-      for key, value in named_args.items():
-        if hash_by := hash_by_map.get(key, rest_hash_by):
-          named_args[key] = hash_by(value)
-      if pos_hash_by := hash_by_map.get(b"*"):
-        pos_args = map(pos_hash_by, pos_args)
-    named_args_iter = chain.from_iterable(sorted(named_args.items()))
-    captured = chain.from_iterable(capturable.capture() for capturable in self.ident.capturables)
-    obj_hash = ObjectHash(digest_size=16) \
-      .update(iter=named_args_iter, header="NAMED") \
-      .update(iter=pos_args, header="POS") \
-      .update(iter=captured, header="CAPTURED")
-    return str(obj_hash)
+    pos_args = args[len(ident.pos_names):]
+    named_pos_args = dict(zip(ident.pos_names, args))
+    named_args = {**ident.default_args, **named_pos_args, **kw}
+    for key, hash_by in ident.hash_by_map.items():
+      if isinstance(key, str):
+        named_args[key] = hash_by(named_args[key])
+      elif key == b"*":
+        pos_args = map(hash_by, pos_args)
+      elif key == b"**":
+        for key in kw.keys() - ident.arg_names:
+          named_args[key] = hash_by(named_args[key])
+    call_hash = ObjectHash(digest_size=16) \
+      .update(header="NAMED", iter=flatten(sorted(named_args.items()))) \
+      .update(header="POS", iter=pos_args) \
+      .update(header="CAPTURED", iter=flatten(c.capture() for c in ident.capturables))
+    return str(call_hash)
 
   def get_call_hash(self: CachedFunction[Callable[P, R]], *args: P.args, **kw: P.kwargs) -> str:
     return self._get_call_hash(args, kw)
 
-  async def _resolve_coroutine(self, call_hash: str, coroutine: Coroutine):
+  async def _store_coroutine(self, call_hash: str, coroutine: Coroutine):
     return self.storage.store(call_hash, AwaitableValue(await coroutine)).value
 
   def _call(self: CachedFunction[Callable[P, R]], args: tuple, kw: dict, rerun=False) -> R:
     full_args = self.bound + args
-    params = self.checkpointer
+    params = self.ident.checkpointer
+    storage = self.storage
     if not params.when:
       return self.fn(*full_args, **kw)
 
     call_hash = self._get_call_hash(args, kw)
-    call_id = f"{self.storage.fn_id()}/{call_hash}"
-    refresh = rerun \
-      or not self.storage.exists(call_hash) \
-      or (params.should_expire and params.should_expire(self.storage.checkpoint_date(call_hash)))
+    call_id = f"{storage.fn_id()}/{call_hash}"
+    refresh = rerun or not storage.exists(call_hash) or storage.expired(call_hash)
 
     if refresh:
       print_checkpoint(params.verbosity >= 1, "MEMORIZING", call_id, "blue")
       data = self.fn(*full_args, **kw)
       if iscoroutine(data):
-        return self._resolve_coroutine(call_hash, data)
-      return self.storage.store(call_hash, data)
+        return self._store_coroutine(call_hash, data)
+      return storage.store(call_hash, data)
 
     try:
-      data = self.storage.load(call_hash)
+      data = storage.load(call_hash)
       print_checkpoint(params.verbosity >= 2, "REMEMBERED", call_id, "green")
       return data
     except (EOFError, FileNotFoundError):
@@ -232,15 +249,6 @@ class CachedFunction(Generic[Fn]):
     self.storage.store(self._get_call_hash(args, kw), value)
 
   def __repr__(self) -> str:
-    return f"<CachedFunction {self.fn.__name__} {self.ident.fn_hash[:6]}>"
-
-def get_hash_by_map(params: list[Parameter]) -> dict[str | bytes, Callable[[object], object]]:
-  hash_by_map = {}
-  for param in params:
-    name = param.name
-    if param.kind == Parameter.VAR_POSITIONAL:
-      name = b"*"
-    elif param.kind == Parameter.VAR_KEYWORD:
-      name = b"**"
-    hash_by_map[name] = hash_by_from_annotation(param.annotation)
-  return hash_by_map if any(hash_by_map.values()) else {}
+    initialized = "fn_hash" in self.ident.__dict__
+    fn_hash = self.ident.fn_hash[:6] if initialized else "- uninitialized"
+    return f"<CachedFunction {self.fn.__name__} {fn_hash}>"

checkpointer/fn_ident.py

@@ -2,11 +2,12 @@ import dis
 from inspect import Parameter, getmodule, signature, unwrap
 from types import CodeType, MethodType, ModuleType
 from typing import Annotated, Callable, Iterable, NamedTuple, Type, get_args, get_origin
+from .fn_string import get_fn_aststr
 from .import_mappings import resolve_annotation
 from .object_hash import ObjectHash
 from .types import hash_by_from_annotation, is_capture_me, is_capture_me_once, to_none
 from .utils import (
-  AttrDict, cwd, distinct, get_cell_contents,
+  cwd, distinct, get_at, get_cell_contents,
   get_file, is_class, is_user_fn, seekable, takewhile,
 )
 
@@ -28,77 +29,77 @@ class Capturable(NamedTuple):
   def capture(self) -> tuple[str, object]:
     if obj := self.hash:
       return self.key, obj
-    obj = AttrDict.get_at(self.module, *self.attr_path)
+    obj = get_at(self.module, *self.attr_path)
     obj = self.hash_by(obj) if self.hash_by else obj
     return self.key, obj
 
   @staticmethod
   def new(module: ModuleType, attr_path: AttrPath, hash_by: Callable | None, capture_once: bool) -> "Capturable":
     file = str(get_file(module).relative_to(cwd))
-    key = "-".join((file, *attr_path))
+    key = file + "/" + ".".join(attr_path)
     cap = Capturable(key, module, attr_path, hash_by)
     if not capture_once:
       return cap
     obj_hash = str(ObjectHash(cap.capture()[1]))
     return Capturable(key, module, attr_path, None, obj_hash)
 
-def extract_classvars(code: CodeType, scope_vars: AttrDict) -> dict[str, dict[str, Type]]:
+def extract_classvars(code: CodeType, scope_vars: dict) -> dict[str, dict[str, Type]]:
   attr_path = AttrPath(())
   scope_obj = None
   classvars: dict[str, dict[str, Type]] = {}
   instructs = seekable(dis.get_instructions(code))
-  for instr in instructs:
-    if instr.opname in scope_vars and not attr_path:
+  for instruct in instructs:
+    if instruct.opname in scope_vars and not attr_path:
       attrs = takewhile((x.opname == "LOAD_ATTR", x.argval) for x in instructs)
-      attr_path = AttrPath((instr.opname, instr.argval, *attrs))
+      attr_path = AttrPath((instruct.opname, instruct.argval, *attrs))
       instructs.step(-1)
-    elif instr.opname == "CALL":
-      obj = scope_vars.get_at(*attr_path)
+    elif instruct.opname == "CALL":
+      obj = get_at(scope_vars, *attr_path)
       attr_path = AttrPath(())
       if is_class(obj):
         scope_obj = obj
-    elif instr.opname in ("STORE_FAST", "STORE_DEREF") and scope_obj:
-      load_key = instr.opname.replace("STORE", "LOAD")
-      classvars.setdefault(load_key, {})[instr.argval] = scope_obj
+    elif instruct.opname in ("STORE_FAST", "STORE_DEREF") and scope_obj:
+      load_key = instruct.opname.replace("STORE", "LOAD")
+      classvars.setdefault(load_key, {})[instruct.argval] = scope_obj
       scope_obj = None
   return classvars
 
-def extract_scope_values(code: CodeType, scope_vars: AttrDict) -> Iterable[tuple[AttrPath, object]]:
+def extract_scope_values(code: CodeType, scope_vars: dict) -> Iterable[tuple[AttrPath, object]]:
   classvars = extract_classvars(code, scope_vars)
-  scope_vars = scope_vars.set({k: scope_vars[k].set(v) for k, v in classvars.items()})
+  scope_vars = {**scope_vars, **{k: {**scope_vars[k], **v} for k, v in classvars.items()}}
   instructs = seekable(dis.get_instructions(code))
-  for instr in instructs:
-    if instr.opname in scope_vars:
+  for instruct in instructs:
+    if instruct.opname in scope_vars:
       attrs = takewhile((x.opname in ("LOAD_ATTR", "LOAD_METHOD"), x.argval) for x in instructs)
-      attr_path = AttrPath((instr.opname, instr.argval, *attrs))
+      attr_path = AttrPath((instruct.opname, instruct.argval, *attrs))
       parent_path = attr_path[:-1]
       instructs.step(-1)
-      obj = scope_vars.get_at(*attr_path)
+      obj = get_at(scope_vars, *attr_path)
       if obj is not None:
         yield attr_path, obj
       if callable(obj) and parent_path[1:]:
-        parent_obj = scope_vars.get_at(*parent_path)
+        parent_obj = get_at(scope_vars, *parent_path)
         yield parent_path, parent_obj
   for const in code.co_consts:
     if isinstance(const, CodeType):
-      next_deref = scope_vars.LOAD_DEREF.set(scope_vars.LOAD_FAST)
-      next_scope_vars = AttrDict({**scope_vars, "LOAD_FAST": {}, "LOAD_DEREF": next_deref})
+      next_deref = {**scope_vars["LOAD_DEREF"], **scope_vars["LOAD_FAST"]}
+      next_scope_vars = {**scope_vars, "LOAD_FAST": {}, "LOAD_DEREF": next_deref}
       yield from extract_scope_values(const, next_scope_vars)
 
-def resolve_class_annotations(anno: object) -> Type | None:
+def class_from_annotation(anno: object) -> Type | None:
   if anno in (None, Annotated):
     return None
-  elif is_class(anno):
+  if is_class(anno):
     return anno
-  elif get_origin(anno) is Annotated:
-    return resolve_class_annotations(next(iter(get_args(anno)), None))
-  return resolve_class_annotations(get_origin(anno))
+  if get_origin(anno) is Annotated:
+    return class_from_annotation(next(iter(get_args(anno)), None))
+  return class_from_annotation(get_origin(anno))
 
-def get_self_value(fn: Callable) -> type | object | None:
+def get_self_value(fn: Callable) -> Type | object | None:
   if isinstance(fn, MethodType):
     return fn.__self__
   parts = fn.__qualname__.split(".")[:-1]
-  cls = parts and AttrDict(fn.__globals__).get_at(*parts)
+  cls = parts and get_at(fn.__globals__, *parts)
   if is_class(cls):
     return cls
 
@@ -106,9 +107,9 @@ def get_capturables(fn: Callable, capture: bool, captured_vars: dict[AttrPath, o
   module = getmodule(fn)
   if not module or not is_user_fn(fn):
     return
-  for (instruct, *attr_path), obj in captured_vars.items():
+  for (instruct_type, *attr_path), obj in captured_vars.items():
     attr_path = AttrPath(attr_path)
-    if instruct == "LOAD_GLOBAL" and not callable(obj) and not isinstance(obj, ModuleType):
+    if instruct_type == "LOAD_GLOBAL" and not callable(obj) and not isinstance(obj, ModuleType):
       anno = resolve_annotation(module, ".".join(attr_path))
       if capture or is_capture_me(anno) or is_capture_me_once(anno):
         hash_by = hash_by_from_annotation(anno)
@@ -116,19 +117,20 @@ def get_capturables(fn: Callable, capture: bool, captured_vars: dict[AttrPath, o
           yield Capturable.new(module, attr_path, hash_by, is_capture_me_once(anno))
 
 def get_fn_captures(fn: Callable, capture: bool) -> tuple[list[Callable], list[Capturable]]:
-  sig_scope = {
+  scope_vars_signature: dict[str, Type | object] = {
     param.name: class_anno
     for param in signature(fn).parameters.values()
     if param.annotation is not Parameter.empty
     if param.kind not in (Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD)
-    if (class_anno := resolve_class_annotations(param.annotation))
+    if (class_anno := class_from_annotation(param.annotation))
+  }
+  if self_obj := get_self_value(fn):
+    scope_vars_signature["self"] = self_obj
+  scope_vars = {
+    "LOAD_FAST": scope_vars_signature,
+    "LOAD_DEREF": dict(get_cell_contents(fn)),
+    "LOAD_GLOBAL": fn.__globals__,
   }
-  self_value = get_self_value(fn)
-  scope_vars = AttrDict({
-    "LOAD_FAST": AttrDict({**sig_scope, "self": self_value} if self_value else sig_scope),
-    "LOAD_DEREF": AttrDict(get_cell_contents(fn)),
-    "LOAD_GLOBAL": AttrDict(fn.__globals__),
-  })
   captured_vars = dict(extract_scope_values(fn.__code__, scope_vars))
   captured_callables = [obj for obj in captured_vars.values() if callable(obj)]
   capturables = list(get_capturables(fn, capture, captured_vars))
@@ -142,7 +144,7 @@ def get_depend_fns(fn: Callable, capture: bool, capturable_by_fn: CapturableByFn
   for depend_fn in captured_callables:
     depend_fn = unwrap(depend_fn, stop=lambda f: isinstance(f, CachedFunction))
     if isinstance(depend_fn, CachedFunction):
-      capturable_by_fn[depend_fn] = []
+      capturable_by_fn[depend_fn.ident.cached_fn] = []
     elif depend_fn not in capturable_by_fn and is_user_fn(depend_fn):
       get_depend_fns(depend_fn, capture, capturable_by_fn)
   return capturable_by_fn
@@ -150,10 +152,10 @@ def get_depend_fns(fn: Callable, capture: bool, capturable_by_fn: CapturableByFn
 def get_fn_ident(fn: Callable, capture: bool) -> RawFunctionIdent:
   from .checkpoint import CachedFunction
   capturable_by_fn = get_depend_fns(fn, capture)
-  capturables = {capt for capts in capturable_by_fn.values() for capt in capts}
   depends = capturable_by_fn.keys()
   depends = distinct(fn.__func__ if isinstance(fn, MethodType) else fn for fn in depends)
-  unwrapped_depends = [fn for fn in depends if not isinstance(fn, CachedFunction)]
-  assert fn == unwrapped_depends[0]
-  fn_hash = str(ObjectHash(iter=unwrapped_depends))
+  depend_callables = [fn for fn in depends if not isinstance(fn, CachedFunction)]
+  fn_hash = str(ObjectHash(iter=map(get_fn_aststr, depend_callables)))
+  capturables = {capt for capts in capturable_by_fn.values() for capt in capts}
+  assert fn == depend_callables[0]
   return RawFunctionIdent(fn_hash, depends, capturables)

checkpointer/fn_string.py

@@ -0,0 +1,75 @@
+import ast
+import sys
+from inspect import getsource
+from textwrap import dedent
+from typing import Callable
+from .utils import drop_none, get_at
+
+def get_decorator_path(node: ast.AST) -> tuple[str, ...]:
+  if isinstance(node, ast.Call):
+    return get_decorator_path(node.func)
+  elif isinstance(node, ast.Attribute):
+    return get_decorator_path(node.value) + (node.attr,)
+  elif isinstance(node, ast.Name):
+    return (node.id,)
+  else:
+    return ()
+
+def is_lone_expression(node: ast.AST) -> bool:
+  # Filter out docstrings
+  return isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant)
+
+class CleanFunctionTransform(ast.NodeTransformer):
+  def __init__(self, fn_globals: dict):
+    self.is_root = True
+    self.fn_globals = fn_globals
+
+  def is_checkpointer(self, node: ast.AST) -> bool:
+    from .checkpoint import Checkpointer
+    decorator = get_at(self.fn_globals, *get_decorator_path(node))
+    return isinstance(decorator, Checkpointer) or decorator is Checkpointer
+
+  def visit_FunctionDef(self, node: ast.FunctionDef | ast.AsyncFunctionDef):
+    fn_type = type(node).__name__
+    fn_name = None if self.is_root else node.name
+    args_by_type = [
+      node.args.posonlyargs + node.args.args,
+      drop_none([node.args.vararg]),
+      sorted(node.args.kwonlyargs, key=lambda x: x.arg),
+      drop_none([node.args.kwarg]),
+    ]
+    arg_kind_names = ",".join(f"{i}:{arg.arg}" for i, args in enumerate(args_by_type) for arg in args)
+    header = " ".join(drop_none((fn_type, fn_name, arg_kind_names or None)))
+
+    self.is_root = False
+
+    return ast.List([
+      ast.Constant(header),
+      ast.List([child for child in node.decorator_list if not self.is_checkpointer(child)], ast.Load()),
+      ast.List([self.visit(child) for child in node.body if not is_lone_expression(child)], ast.Load()),
+    ], ast.Load())
+
+  def visit_AsyncFunctionDef(self, node):
+    return self.visit_FunctionDef(node)
+
+def get_fn_aststr(fn: Callable) -> str:
+  try:
+    source = getsource(fn)
+  except OSError:
+    return ""
+  try:
+    tree = ast.parse(dedent(source), mode="exec")
+    tree = tree.body[0]
+  except SyntaxError:
+    # lambda functions can cause SyntaxError in ast.parse
+    return source.strip()
+
+  if fn.__name__ != "<lambda>":
+    tree = CleanFunctionTransform(fn.__globals__).visit(tree)
+  else:
+    tree = ast.List([node for node in ast.walk(tree) if isinstance(node, ast.Lambda)], ast.Load())
+
+  if sys.version_info >= (3, 13):
+    return ast.dump(tree, annotate_fields=False, show_empty=True)
+  else:
+    return ast.dump(tree, annotate_fields=False)

checkpointer/object_hash.py

@@ -5,32 +5,38 @@ import io
 import re
 import sys
 import tokenize
+import sysconfig
+from collections import OrderedDict
 from collections.abc import Iterable
 from contextlib import nullcontext, suppress
 from decimal import Decimal
 from io import StringIO
+from inspect import getfile
 from itertools import chain
+from pathlib import Path
 from pickle import HIGHEST_PROTOCOL as PICKLE_PROTOCOL
-from types import BuiltinFunctionType, FunctionType, GeneratorType, MethodType, ModuleType, UnionType
+from types import BuiltinFunctionType, FunctionType, GeneratorType, MappingProxyType, MethodType, ModuleType, UnionType
 from typing import Callable, Self, TypeVar
-from .utils import ContextVar
+from .utils import ContextVar, flatten
 
 np, torch = None, None
 
-with suppress(Exception):
-  import numpy as np
-with suppress(Exception):
-  import torch
-
 class _Never:
   def __getattribute__(self, _: str):
     pass
 
+with suppress(Exception):
+  import numpy as np
+with suppress(Exception):
+  import torch
 if sys.version_info >= (3, 12):
   from typing import TypeAliasType
 else:
   TypeAliasType = _Never
 
+nc = nullcontext()
+stdlib = Path(sysconfig.get_paths()["stdlib"]).resolve()
+
 def encode_type(t: type | FunctionType) -> str:
   return f"{t.__module__}:{t.__qualname__}"
 
@@ -77,7 +83,7 @@ class ObjectHash:
     return self.write_bytes(":".join(map(str, args)).encode())
 
   def update(self, *objs: object, iter: Iterable[object] = (), tolerable: bool | None=None, header: str | None = None) -> Self:
-    with nullcontext() if tolerable is None else self.tolerable.set(tolerable):
+    with nc if tolerable is None else self.tolerable.set(tolerable):
       for obj in chain(objs, iter):
         if header is not None:
           self.write_bytes(header.encode())
@@ -105,11 +111,11 @@ class ObjectHash:
 
       case set() | frozenset():
         try:
-          items = sorted(obj)
           header = "set"
+          items = sorted(obj)
         except:
-          items = sorted(map(self.nested_hash, obj))
           header = "set-unsortable"
+          items = sorted(map(self.nested_hash, obj))
         self.header(header, encode_type_of(obj), len(obj)).update(iter=items)
 
       case TypeVar():
@@ -125,7 +131,11 @@ class ObjectHash:
         self.header("builtin", obj.__qualname__)
 
       case FunctionType():
-        self.header("function", encode_type(obj)).update(get_fn_body(obj), obj.__defaults__, obj.__kwdefaults__, obj.__annotations__)
+        fn_file = Path(getfile(obj)).resolve()
+        if fn_file.is_relative_to(stdlib):
+          self.header("function-std", obj.__qualname__)
+        else:
+          self.header("function", encode_type(obj)).update(get_fn_body(obj), obj.__defaults__, obj.__kwdefaults__, obj.__annotations__)
 
       case MethodType():
         self.header("method").update(obj.__func__, obj.__self__.__class__)
@@ -170,14 +180,16 @@ class ObjectHash:
           match obj:
             case list() | tuple():
               self.header("list", encode_type_of(obj), len(obj)).update(iter=obj)
-            case dict():
-              try:
-                items = sorted(obj.items())
-                header = "dict"
-              except:
-                items = sorted((self.nested_hash(key), val) for key, val in obj.items())
-                header = "dict-unsortable"
-              self.header(header, encode_type_of(obj), len(obj)).update(iter=chain.from_iterable(items))
+            case dict() | MappingProxyType():
+              header = "dict"
+              items = obj.items()
+              if not isinstance(obj, OrderedDict):
+                try:
+                  items = sorted(items)
+                except:
+                  header = "dict-unsortable"
+                  items = sorted((self.nested_hash(key), val) for key, val in items)
+              self.header(header, encode_type_of(obj), len(obj)).update(iter=flatten(items))
             case _:
               self._update_object(obj)
         finally:

checkpointer/storages/memory_storage.py

@@ -31,7 +31,7 @@ class MemoryStorage(Storage):
       if key.parent == curr_key.parent:
         if invalidated and key != curr_key:
           del item_map[key]
-        elif expired and self.checkpointer.should_expire:
+        elif expired and self.checkpointer.expiry:
           for call_hash, (date, _) in list(calldict.items()):
-            if self.checkpointer.should_expire(date):
+            if self.expired_dt(date):
               del calldict[call_hash]

checkpointer/storages/pickle_storage.py

@@ -9,7 +9,7 @@ def filedate(path: Path) -> datetime:
 
 class PickleStorage(Storage):
   def get_path(self, call_hash: str):
-    return self.fn_dir() / f"{call_hash}.pkl"
+    return self.fn_dir() / f"{call_hash[:2]}/{call_hash[2:]}.pkl"
 
   def store(self, call_hash, data):
     path = self.get_path(call_hash)
@@ -40,10 +40,10 @@ class PickleStorage(Storage):
       for path in old_dirs:
         shutil.rmtree(path)
       print(f"Removed {len(old_dirs)} invalidated directories for {self.cached_fn.__qualname__}")
-    if expired and self.checkpointer.should_expire:
+    if expired and self.checkpointer.expiry:
       count = 0
       for pkl_path in fn_path.glob("**/*.pkl"):
-        if self.checkpointer.should_expire(filedate(pkl_path)):
+        if self.expired_dt(filedate(pkl_path)):
           count += 1
           pkl_path.unlink(missing_ok=True)
       print(f"Removed {count} expired checkpoints for {self.cached_fn.__qualname__}")

checkpointer/storages/storage.py

@@ -1,24 +1,38 @@
 from __future__ import annotations
 from typing import Any, TYPE_CHECKING
 from pathlib import Path
-from datetime import datetime
+from datetime import datetime, timedelta
 
 if TYPE_CHECKING:
   from ..checkpoint import Checkpointer, CachedFunction
 
 class Storage:
   checkpointer: Checkpointer
-  cached_fn: CachedFunction
+  ident: CachedFunction
 
   def __init__(self, cached_fn: CachedFunction):
-    self.checkpointer = cached_fn.checkpointer
+    self.checkpointer = cached_fn.ident.checkpointer
     self.cached_fn = cached_fn
 
   def fn_id(self) -> str:
-    return f"{self.cached_fn.fn_dir}/{self.cached_fn.ident.fn_hash}"
+    ident = self.cached_fn.ident
+    return f"{ident.fn_dir}/{ident.fn_hash}"
 
   def fn_dir(self) -> Path:
-    return self.checkpointer.root_path / self.fn_id()
+    return self.checkpointer.directory / self.fn_id()
+
+  def expired(self, call_hash: str) -> bool:
+    if not self.checkpointer.expiry:
+      return False
+    return self.expired_dt(self.checkpoint_date(call_hash))
+
+  def expired_dt(self, dt: datetime) -> bool:
+    expiry = self.checkpointer.expiry
+    if isinstance(expiry, timedelta):
+      return dt < datetime.now() - expiry
+    else:
+      if TYPE_CHECKING: assert expiry
+      return expiry(dt)
 
   def store(self, call_hash: str, data: Any) -> Any: ...
 

checkpointer/utils.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
 import inspect
-from contextlib import contextmanager
-from itertools import islice
+from contextlib import contextmanager, suppress
+from itertools import chain, islice
 from pathlib import Path
 from types import FunctionType, MethodType, ModuleType
 from typing import Callable, Generic, Iterable, Self, Type, TypeGuard
 from .types import T
 
 cwd = Path.cwd().resolve()
+flatten = chain.from_iterable
 
 def is_class(obj) -> TypeGuard[Type]:
   return isinstance(obj, type)
@@ -22,11 +23,12 @@ def is_user_fn(obj) -> TypeGuard[Callable]:
   return isinstance(obj, (FunctionType, MethodType)) and is_user_file(get_file(obj))
 
 def get_cell_contents(fn: Callable) -> Iterable[tuple[str, object]]:
-  for key, cell in zip(fn.__code__.co_freevars, fn.__closure__ or []):
-    try:
+  for key, cell in zip(fn.__code__.co_freevars, fn.__closure__ or ()):
+    with suppress(ValueError):
       yield (key, cell.cell_contents)
-    except ValueError:
-      pass
+
+def drop_none(iterable: Iterable[T | None]) -> list[T]:
+  return [x for x in iterable if x is not None]
 
 def distinct(seq: Iterable[T]) -> list[T]:
   return list(dict.fromkeys(seq))
@@ -80,6 +82,14 @@ class seekable(Generic[T]):
     with self.freeze():
       return list(islice(self, count))
 
+def get_at(obj: object, *attrs: str) -> object:
+  for attr in attrs:
+    if type(obj) is dict:
+      obj = obj.get(attr, None)
+    else:
+      obj = getattr(obj, attr, None)
+  return obj
+
 class AttrDict(dict):
   def __init__(self, *args, **kwargs):
     super().__init__(*args, **kwargs)
@@ -91,17 +101,6 @@ class AttrDict(dict):
   def __setattr__(self, name: str, value: object):
     super().__setattr__(name, value)
 
-  def set(self, d: dict) -> AttrDict:
-    if not d:
-      return self
-    return AttrDict({**self, **d})
-
-  def get_at(self: object, *attrs: str) -> object:
-    obj = self
-    for attr in attrs:
-      obj = getattr(obj, attr, None)
-    return obj
-
 class ContextVar(Generic[T]):
   def __init__(self, value: T):
     self.value = value

checkpointer-2.14.0.dist-info/METADATA

@@ -0,0 +1,243 @@
+Metadata-Version: 2.4
+Name: checkpointer
+Version: 2.14.0
+Summary: checkpointer adds code-aware caching to Python functions, maintaining correctness and speeding up execution as your code changes.
+Project-URL: Repository, https://github.com/Reddan/checkpointer.git
+Author: Hampus Hallman
+License-Expression: MIT
+License-File: ATTRIBUTION.md
+License-File: LICENSE
+Keywords: async,cache,caching,data analysis,data processing,fast,hashing,invalidation,memoization,optimization,performance,workflow
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# checkpointer · [![License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/Reddan/checkpointer/blob/master/LICENSE) [![pypi](https://img.shields.io/pypi/v/checkpointer)](https://pypi.org/project/checkpointer/) [![pypi](https://img.shields.io/pypi/pyversions/checkpointer)](https://pypi.org/project/checkpointer/)
+
+`checkpointer` is a Python library offering a decorator-based API for memoizing (caching) function results with code-aware cache invalidation. It works with sync and async functions, supports multiple storage backends, and invalidates caches automatically when your code or dependencies change - helping you maintain correctness, speed up execution, and smooth out your workflows by skipping redundant, costly operations.
+
+## 📦 Installation
+
+```bash
+pip install checkpointer
+```
+
+## 🚀 Quick Start
+
+Apply the `@checkpoint` decorator to any function:
+
+```python
+from checkpointer import checkpoint
+
+@checkpoint
+def expensive_function(x: int) -> int:
+    print("Computing...")
+    return x ** 2
+
+result = expensive_function(4)  # Computes and stores the result
+result = expensive_function(4)  # Loads from the cache
+```
+
+## 🧠 How It Works
+
+When you decorate a function with `@checkpoint` and call it, `checkpointer` computes a unique identifier that represents that specific call. This identifier is based on:
+
+* The function's source code and all its user-defined dependencies,
+* Global variables used by the function (if capturing is enabled or explicitly annotated),
+* The actual arguments passed to the function.
+
+`checkpointer` then looks up this identifier in its cache. If a valid cached result exists, it returns that immediately. Otherwise, it runs the original function, stores the result, and returns it.
+
+`checkpointer` is designed to be flexible through features like:
+
+* **Support for decorated methods**, correctly caching results bound to instances.
+* **Support for decorated async functions**, compatible with any async runtime.
+* **Robust hashing**, covering complex Python objects and large **NumPy**/**PyTorch** arrays via its internal `ObjectHash`.
+* **Targeted hashing**, allowing you to optimize how arguments and captured variables are hashed.
+* **Multi-layered caching**, letting you stack decorators for layered caching strategies without losing cache consistency.
+
+### 🚨 What Causes Cache Invalidation?
+
+To ensure cache correctness, `checkpointer` tracks two types of hashes:
+
+#### 1. Function Identity Hash (Computed Once Per Function)
+
+This hash represents the decorated function itself and is computed once (usually on first invocation). It covers:
+
+* **Function Code and Signature:**\
+    The actual logic and parameters of the function are hashed - but *not* parameter type annotations or formatting details like whitespace, newlines, comments, or trailing commas, which do **not** trigger invalidation.
+
+* **Dependencies:**\
+    All user-defined functions and methods that the decorated function calls or relies on, including indirect dependencies, are included recursively. Dependencies are identified by:
+    * Inspecting the function's global scope for referenced functions and objects.
+    * Inferring from the function's argument type annotations.
+    * Analyzing object constructions and method calls to identify classes and methods used.
+
+* **Exclusions:**\
+    Changes elsewhere in the module unrelated to the function or its dependencies do **not** cause invalidation.
+
+#### 2. Call Hash (Computed on Every Function Call)
+
+Every function call produces a call hash, combining:
+
+* **Passed Arguments:**\
+    Includes positional and keyword arguments, combined with default values. Changing defaults alone doesn't necessarily trigger invalidation unless it affects actual call values.
+
+* **Captured Global Variables:**\
+    When `capture=True` or explicit capture annotations are used, `checkpointer` includes referenced global variables in the call hash. Variables annotated with `CaptureMe` are hashed on every call, causing immediate cache invalidation if they change. Variables annotated with `CaptureMeOnce` are hashed only once per Python session, improving performance by avoiding repeated hashing.
+
+* **Custom Argument Hashing:**\
+    Using `HashBy` annotations, arguments or captured variables can be transformed before hashing (e.g., sorting lists to ignore order), allowing more precise or efficient call hashes.
+
+## 💡 Usage
+
+Once a function is decorated with `@checkpoint`, you can interact with its caching behavior using the following methods:
+
+* **`expensive_function(...)`**:\
+    Call the function normally. This will compute and cache the result or load it from cache.
+
+* **`expensive_function.rerun(...)`**:\
+    Force the original function to execute and overwrite any existing cached result.
+
+* **`expensive_function.fn(...)`**:\
+    Call the undecorated function directly, bypassing the cache (useful in recursion to prevent caching intermediate steps).
+
+* **`expensive_function.get(...)`**:\
+    Retrieve the cached result without executing the function. Raises `CheckpointError` if no valid cache exists.
+
+* **`expensive_function.exists(...)`**:\
+    Check if a cached result exists without computing or loading it.
+
+* **`expensive_function.delete(...)`**:\
+    Remove the cached entry for given arguments.
+
+* **`expensive_function.reinit(recursive: bool = True)`**:\
+    Recalculate the function identity hash and recapture `CaptureMeOnce` variables, updating the cached function state within the same Python session.
+
+## ⚙️ Configuration & Customization
+
+The `@checkpoint` decorator accepts the following parameters:
+
+* **`storage`** (Type: `str` or `checkpointer.Storage`, Default: `"pickle"`)\
+    Storage backend to use: `"pickle"` (disk-based, persistent), `"memory"` (in-memory, non-persistent), or a custom `Storage` class.
+
+* **`directory`** (Type: `str` or `pathlib.Path` or `None`, Default: `~/.cache/checkpoints`)\
+    Base directory for disk-based checkpoints (only for `"pickle"` storage).
+
+* **`capture`** (Type: `bool`, Default: `False`)\
+    If `True`, includes global variables referenced by the function in call hashes (except those excluded via `NoHash`).
+
+* **`expiry`** (Type: `Callable[[datetime.datetime], bool]` or `datetime.timedelta`, Default: `None`)\
+    A custom callable that receives the `datetime` timestamp of a cached result. It should return `True` if the cached result is considered expired and needs recomputation, or `False` otherwise.
+
+* **`fn_hash_from`** (Type: `Any`, Default: `None`)\
+    Override the computed function identity hash with any hashable object you provide (e.g., version strings, config IDs). This gives you explicit control over the function's version and when its cache should be invalidated.
+
+* **`when`** (Type: `bool`, Default: `True`)\
+    Enable or disable checkpointing dynamically, useful for environment-based toggling.
+
+* **`verbosity`** (Type: `int` (`0`, `1`, or `2`), Default: `1`)\
+    Controls the level of logging output from `checkpointer`.
+    * `0`: No output.
+    * `1`: Shows when functions are computed and cached.
+    * `2`: Also shows when cached results are remembered (loaded from cache).
+
+## 🔬 Customize Argument Hashing
+
+You can customize how arguments are hashed without modifying the actual argument values to improve cache hit rates or speed up hashing.
+
+* **`Annotated[T, HashBy[fn]]`**:\
+    Transform the argument via `fn(argument)` before hashing. Useful for normalization (e.g., sorting lists) or optimized hashing for complex inputs.
+
+* **`NoHash[T]`**:\
+    Exclude the argument from hashing completely, so changes to it won't trigger cache invalidation.
+
+**Example:**
+
+```python
+from typing import Annotated
+from checkpointer import checkpoint, HashBy, NoHash
+from pathlib import Path
+import logging
+
+def file_bytes(path: Path) -> bytes:
+    return path.read_bytes()
+
+@checkpoint
+def process(
+    numbers: Annotated[list[int], HashBy[sorted]],   # Hash by sorted list
+    data_file: Annotated[Path, HashBy[file_bytes]],  # Hash by file content
+    log: NoHash[logging.Logger],                     # Exclude logger from hashing
+):
+    ...
+```
+
+In this example, the hash for `numbers` ignores order, `data_file` is hashed based on its contents rather than path, and changes to `log` don't affect caching.
+
+## 🎯 Capturing Global Variables
+
+`checkpointer` can include **captured global variables** in call hashes - these are globals your function reads during execution that may affect results.
+
+Use `capture=True` on `@checkpoint` to capture **all** referenced globals (except those explicitly excluded with `NoHash`).
+
+Alternatively, you can **opt-in selectively** by annotating globals with:
+
+* **`CaptureMe[T]`**:\
+    Capture the variable on every call (triggers invalidation on changes).
+
+* **`CaptureMeOnce[T]`**:\
+    Capture once per Python session (for expensive, immutable globals).
+
+You can also combine these with `HashBy` to customize how captured variables are hashed (e.g., hash by subset of attributes).
+
+**Example:**
+
+```python
+from typing import Annotated
+from checkpointer import checkpoint, CaptureMe, CaptureMeOnce, HashBy
+from pathlib import Path
+
+def file_bytes(path: Path) -> bytes:
+    return path.read_bytes()
+
+captured_data: CaptureMe[Annotated[Path, HashBy[file_bytes]]] = Path("data.txt")
+session_config: CaptureMeOnce[dict] = {"mode": "prod"}
+
+@checkpoint
+def process():
+    # `captured_data` is included in the call hash on every call, hashed by file content
+    # `session_config` is hashed once per session
+    ...
+```
+
+## 🗄️ Custom Storage Backends
+
+Implement your own storage backend by subclassing `checkpointer.Storage` and overriding required methods.
+
+Within storage methods, `call_hash` identifies calls by arguments. Use `self.fn_id()` to get function identity (name + hash/version), important for organizing checkpoints.
+
+**Example:**
+
+```python
+from checkpointer import checkpoint, Storage
+from datetime import datetime
+
+class MyCustomStorage(Storage):
+    def exists(self, call_hash):
+        fn_dir = self.checkpointer.directory / self.fn_id()
+        return (fn_dir / call_hash).exists()
+
+    def store(self, call_hash, data):
+        ...  # Store serialized data
+        return data  # Must return data to checkpointer
+
+    def checkpoint_date(self, call_hash): ...
+    def load(self, call_hash): ...
+    def delete(self, call_hash): ...
+
+@checkpoint(storage=MyCustomStorage)
+def custom_cached_function(x: int):
+    return x ** 2
+```

checkpointer-2.14.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+checkpointer/__init__.py,sha256=mT706hd-BoTpQXYH_MiqreplONVDvjCWjwwGQ7oRe_8,921
+checkpointer/checkpoint.py,sha256=SUHu5ADRtSrapspk2DpBLaYnAQ7yuTc05lUELig_Tpc,9972
+checkpointer/fn_ident.py,sha256=Z2HtkoG8n0ddqNzjejnKW6Lo-ID2unKQZa1tOpRGujs,6911
+checkpointer/fn_string.py,sha256=YldjAU91cwiZNwuE_AQN_DMiQCPAhGiiC3lPi0uvM0g,2579
+checkpointer/import_mappings.py,sha256=ESqWvZTzYAmaVnJ6NulUvn3_8CInOOPmEKUXO2WD_WA,1794
+checkpointer/object_hash.py,sha256=_VgyTEoe3H6268KBAOFZxSgWHb6otgG-3NzhHyehxe4,8595
+checkpointer/print_checkpoint.py,sha256=uUQ493fJCaB4nhp4Ox60govSCiBTIPbBX15zt2QiRGo,1356
+checkpointer/types.py,sha256=GFqbGACdDxzQX3bb2LmF9UxQVWOEisGvdtobnqCBAOA,1129
+checkpointer/utils.py,sha256=CF28pGytCWV4URhRDFaIDawX8qmCAAByQp0ZkEwrA6c,3014
+checkpointer/storages/__init__.py,sha256=p-r4YrPXn505_S3qLrSXHSlsEtb13w_DFnCt9IiUomk,296
+checkpointer/storages/memory_storage.py,sha256=f242kLEsB4TcqEkEBmJGIewJE9mvv9aCAZ0V9pPB2EQ,1110
+checkpointer/storages/pickle_storage.py,sha256=burbgX-ICPLNMAF9iU3HRZroxJibKfmgJLqTCm6YbbI,1649
+checkpointer/storages/storage.py,sha256=PgpwW_VLkiwD361Kjsz8aoq5FHZLBTqF43yD2mw9dTI,1326
+checkpointer-2.14.0.dist-info/METADATA,sha256=MSJ--2X9N_Ug8sd9VhffONSzhIPA_njmzJhCe3fa_l8,11215
+checkpointer-2.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+checkpointer-2.14.0.dist-info/licenses/ATTRIBUTION.md,sha256=WF6L7-sD4s9t9ytVJOhjhpDoZ6TrWpqE3_bMdDIeJxI,1078
+checkpointer-2.14.0.dist-info/licenses/LICENSE,sha256=drXs6vIb7uW49r70UuMz2A1VtOCl626kiTbcmrar1Xo,1072
+checkpointer-2.14.0.dist-info/RECORD,,

checkpointer-2.12.0.dist-info/METADATA

@@ -1,236 +0,0 @@
-Metadata-Version: 2.4
-Name: checkpointer
-Version: 2.12.0
-Summary: checkpointer adds code-aware caching to Python functions, maintaining correctness and speeding up execution as your code changes.
-Project-URL: Repository, https://github.com/Reddan/checkpointer.git
-Author: Hampus Hallman
-License-Expression: MIT
-License-File: ATTRIBUTION.md
-License-File: LICENSE
-Keywords: async,cache,caching,code-aware,decorator,fast,hashing,invalidation,memoization,memoize,memory,optimization,performance,workflow
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown
-
-# checkpointer · [![License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/Reddan/checkpointer/blob/master/LICENSE) [![pypi](https://img.shields.io/pypi/v/checkpointer)](https://pypi.org/project/checkpointer/) [![pypi](https://img.shields.io/pypi/pyversions/checkpointer)](https://pypi.org/project/checkpointer/)
-
-`checkpointer` is a Python library offering a decorator-based API for memoizing (caching) function results with code-aware cache invalidation. It works with sync and async functions, supports multiple storage backends, and refreshes caches automatically when your code or dependencies change - helping you maintain correctness, speed up execution, and smooth out your workflows by skipping redundant, costly operations.
-
-## 📦 Installation
-
-```bash
-pip install checkpointer
-```
-
-## 🚀 Quick Start
-
-Apply the `@checkpoint` decorator to any function:
-
-```python
-from checkpointer import checkpoint
-
-@checkpoint
-def expensive_function(x: int) -> int:
-    print("Computing...")
-    return x ** 2
-
-result = expensive_function(4)  # Computes and stores the result
-result = expensive_function(4)  # Loads from the cache
-```
-
-## 🧠 How It Works
-
-When a `@checkpoint`-decorated function is called, `checkpointer` first computes a unique identifier (hash) for the call. This hash is derived from the function's source code, its dependencies, and the arguments passed.
-
-It then tries to retrieve a cached result using this ID. If a valid cached result is found, it's returned immediately. Otherwise, the original function executes, its result is stored, and then returned.
-
-Cache validity is determined by this function's hash, which automatically updates if:
-
-* **Function Code Changes**: The decorated function's source code is modified.
-* **Dependencies Change**: Any user-defined function in its dependency tree (direct or indirect, even across modules or not decorated) is modified.
-* **Captured Variables Change** (with `capture=True`): Global or closure-based variables used within the function are altered.
-
-**Example: Dependency Invalidation**
-
-```python
-def multiply(a, b):
-    return a * b
-
-@checkpoint
-def helper(x):
-    # Depends on `multiply`
-    return multiply(x + 1, 2)
-
-@checkpoint
-def compute(a, b):
-    # Depends on `helper` and `multiply`
-    return helper(a) + helper(b)
-```
-
-If `multiply` is modified, caches for both `helper` and `compute` are automatically invalidated and recomputed.
-
-## 💡 Usage
-
-Once a function is decorated with `@checkpoint`, you can interact with its caching behavior using the following methods:
-
-* **`expensive_function(...)`**:\
-    Call the function normally. This will either compute and cache the result or load it from the cache if available.
-
-* **`expensive_function.rerun(...)`**:\
-    Forces the original function to execute, compute a new result, and overwrite any existing cached value for the given arguments.
-
-* **`expensive_function.fn(...)`**:\
-    Calls the original, undecorated function directly, bypassing the cache entirely. This is particularly useful within recursive functions to prevent caching intermediate steps.
-
-* **`expensive_function.get(...)`**:\
-    Attempts to retrieve the cached result for the given arguments without executing the original function. Raises `CheckpointError` if no valid cached result exists.
-
-* **`expensive_function.exists(...)`**:\
-    Checks if a cached result exists for the given arguments without attempting to compute or load it. Returns `True` if a valid checkpoint exists, `False` otherwise.
-
-* **`expensive_function.delete(...)`**:\
-    Removes the cached entry for the specified arguments.
-
-* **`expensive_function.reinit()`**:\
-    Recalculates the function's internal hash. This is primarily used when `capture=True` and you need to update the cache based on changes to external variables within the same Python session.
-
-## ⚙️ Configuration & Customization
-
-The `@checkpoint` decorator accepts the following parameters to customize its behavior:
-
-* **`format`** (Type: `str` or `checkpointer.Storage`, Default: `"pickle"`)\
-    Defines the storage backend to use. Built-in options are `"pickle"` (disk-based, persistent) and `"memory"` (in-memory, non-persistent). You can also provide a custom `Storage` class.
-
-* **`root_path`** (Type: `str` or `pathlib.Path` or `None`, Default: `~/.cache/checkpoints`)\
-    The base directory for storing disk-based checkpoints. This parameter is only relevant when `format` is set to `"pickle"`.
-
-* **`when`** (Type: `bool`, Default: `True`)\
-    A boolean flag to enable or disable checkpointing for the decorated function. This is particularly useful for toggling caching based on environment variables (e.g., `when=os.environ.get("ENABLE_CACHING", "false").lower() == "true"`).
-
-* **`capture`** (Type: `bool`, Default: `False`)\
-    If set to `True`, `checkpointer` includes global or closure-based variables used by the function in its hash calculation. This ensures that changes to these external variables also trigger cache invalidation and recomputation.
-
-* **`should_expire`** (Type: `Callable[[datetime.datetime], bool]`, Default: `None`)\
-    A custom callable that receives the `datetime` timestamp of a cached result. It should return `True` if the cached result is considered expired and needs recomputation, or `False` otherwise.
-
-* **`fn_hash_from`** (Type: `Any`, Default: `None`)\
-    This allows you to override the automatically computed function hash, giving you explicit control over when the function's cache should be invalidated. You can pass any object relevant to your invalidation logic (e.g., version strings, config parameters). The object you provide will be hashed internally by `checkpointer`.
-
-* **`verbosity`** (Type: `int` (`0`, `1`, or `2`), Default: `1`)\
-    Controls the level of logging output from `checkpointer`.
-    * `0`: No output.
-    * `1`: Shows when functions are computed and cached.
-    * `2`: Also shows when cached results are remembered (loaded from cache).
-
-## 🔬 Customize Argument Hashing
-
-You can customize how individual function arguments are hashed without changing their actual values when passed in.
-
-* **`Annotated[T, HashBy[fn]]`**:\
-    Hashes the argument by applying `fn(argument)` before hashing. This enables custom normalization (e.g., sorting lists to ignore order) or optimized hashing for complex types, improving cache hit rates or speeding up hashing.
-
-* **`NoHash[T]`**:\
-    Completely excludes the argument from hashing, so changes to it won’t trigger cache invalidation.
-
-**Example:**
-
-```python
-from typing import Annotated
-from checkpointer import checkpoint, HashBy, NoHash
-from pathlib import Path
-import logging
-
-def file_bytes(path: Path) -> bytes:
-    return path.read_bytes()
-
-@checkpoint
-def process(
-    numbers: Annotated[list[int], HashBy[sorted]],   # Hash by sorted list
-    data_file: Annotated[Path, HashBy[file_bytes]],  # Hash by file content
-    log: NoHash[logging.Logger],                     # Exclude logger from hashing
-):
-    ...
-```
-
-In this example, the cache key for `numbers` ignores order, `data_file` is hashed based on its contents rather than path, and changes to `log` don’t affect caching.
-
-## 🗄️ Custom Storage Backends
-
-For integration with databases, cloud storage, or custom serialization, implement your own storage backend by inheriting from `checkpointer.Storage` and implementing its abstract methods.
-
-Within custom storage methods, `call_hash` identifies calls by arguments. Use `self.fn_id()` to get the function's unique identity (name + hash/version), crucial for organizing stored checkpoints (e.g., by function version). Access global `Checkpointer` config via `self.checkpointer`.
-
-**Example: Custom Storage Backend**
-
-```python
-from checkpointer import checkpoint, Storage
-from datetime import datetime
-
-class MyCustomStorage(Storage):
-    def exists(self, call_hash):
-        # Example: Constructing a path based on function ID and call ID
-        fn_dir = self.checkpointer.root_path / self.fn_id()
-        return (fn_dir / call_hash).exists()
-
-    def store(self, call_hash, data):
-        ... # Store the serialized data for `call_hash`
-        return data # This method must return the data back to checkpointer
-
-    def checkpoint_date(self, call_hash): ...
-    def load(self, call_hash): ...
-    def delete(self, call_hash): ...
-
-@checkpoint(format=MyCustomStorage)
-def custom_cached_function(x: int):
-    return x ** 2
-```
-
-## 🧱 Layered Caching
-
-You can apply multiple `@checkpoint` decorators to a single function to create layered caching strategies. `checkpointer` processes these decorators from bottom to top, meaning the decorator closest to the function definition is evaluated first.
-
-This is useful for scenarios like combining a fast, ephemeral cache (e.g., in-memory) with a persistent, slower cache (e.g., disk-based).
-
-**Example: Memory Cache over Disk Cache**
-
-```python
-from checkpointer import checkpoint
-
-@checkpoint(format="memory") # Layer 2: Fast, ephemeral in-memory cache
-@checkpoint(format="pickle") # Layer 1: Persistent disk cache
-def some_expensive_operation():
-    print("Performing a time-consuming operation...")
-    return sum(i for i in range(10**7))
-```
-
-## ⚡ Async Support
-
-`checkpointer` works seamlessly with Python's `asyncio` and other async runtimes.
-
-```python
-import asyncio
-from checkpointer import checkpoint
-
-@checkpoint
-async def async_compute_sum(a: int, b: int) -> int:
-    print(f"Asynchronously computing {a} + {b}...")
-    await asyncio.sleep(1)
-    return a + b
-
-async def main():
-    # First call computes and caches
-    result1 = await async_compute_sum(3, 7)
-    print(f"Result 1: {result1}")
-
-    # Second call loads from cache
-    result2 = await async_compute_sum(3, 7)
-    print(f"Result 2: {result2}")
-
-    # Retrieve from cache without re-running the async function
-    result3 = async_compute_sum.get(3, 7)
-    print(f"Result 3 (from cache): {result3}")
-
-asyncio.run(main())
-```

checkpointer-2.12.0.dist-info/RECORD

@@ -1,17 +0,0 @@
-checkpointer/__init__.py,sha256=x8LbL-URPg-afn0O-HMxPsJkV9cmW-Cw5epKSCGClOM,889
-checkpointer/checkpoint.py,sha256=2K2D_aYHpI1XBcNrWQxta06wRdUj81TeCXshBVxl7cA,9869
-checkpointer/fn_ident.py,sha256=4qg4NIvcWRV5GduO70an_iu12dn8LLIlw3Q8F3gm0Mo,6826
-checkpointer/import_mappings.py,sha256=ESqWvZTzYAmaVnJ6NulUvn3_8CInOOPmEKUXO2WD_WA,1794
-checkpointer/object_hash.py,sha256=MkrwSJJYVlWOjIDpGfYpAeYFPgCJhXHYBdlKVcUy2kw,8145
-checkpointer/print_checkpoint.py,sha256=uUQ493fJCaB4nhp4Ox60govSCiBTIPbBX15zt2QiRGo,1356
-checkpointer/types.py,sha256=GFqbGACdDxzQX3bb2LmF9UxQVWOEisGvdtobnqCBAOA,1129
-checkpointer/utils.py,sha256=rq7AjR0WJql1o8clKMdXj4p3wrPYMLyS6G11nvNNjFE,2934
-checkpointer/storages/__init__.py,sha256=p-r4YrPXn505_S3qLrSXHSlsEtb13w_DFnCt9IiUomk,296
-checkpointer/storages/memory_storage.py,sha256=aQRSOmAfS0UudubCpv8cdfu2ycM8mlsO9tFMcD2kmgo,1133
-checkpointer/storages/pickle_storage.py,sha256=je1LM2lTSs5yzm25Apg5tJ9jU9T6nXCgD9SlqQRIFaM,1652
-checkpointer/storages/storage.py,sha256=5Jel7VlmCG9gnIbZFKT1NrEiAePz8ZD8hfsD-tYEiP4,905
-checkpointer-2.12.0.dist-info/METADATA,sha256=B-R8Aq2caTM7fclANY7MJVZJQaU0nB1Mdf7wMr6HSTI,10705
-checkpointer-2.12.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-checkpointer-2.12.0.dist-info/licenses/ATTRIBUTION.md,sha256=WF6L7-sD4s9t9ytVJOhjhpDoZ6TrWpqE3_bMdDIeJxI,1078
-checkpointer-2.12.0.dist-info/licenses/LICENSE,sha256=drXs6vIb7uW49r70UuMz2A1VtOCl626kiTbcmrar1Xo,1072
-checkpointer-2.12.0.dist-info/RECORD,,