pickle-debug 1.0__tar.gz → 1.2__tar.gz

{pickle_debug-1.0/src/pickle_debug.egg-info → pickle_debug-1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pickle_debug
-Version: 1.0
+Version: 1.2
 Summary: Store expensive objects from debugging
 Author: Gerard
 Author-email: gweatherby@uchc.edu
@@ -77,6 +77,19 @@ Decorator. Returns a wrapper (using `functools.wraps`) that:
 Results are written to a temporary file and then renamed, so an interrupted run does not leave a partial
 cache entry.
 
+#### Unpicklable attributes
+If an object (in the result or in the arguments) has attributes that cannot be pickled, such as locks,
+open files, sockets or lambdas, those attributes are left out and the rest of the object is cached. This
+applies to nested objects too, and covers both `__dict__` and `__slots__` attributes. Each dropped
+attribute is logged at `DEBUG` level.
+
+An object loaded from the cache does not have the dropped attributes, so accessing them raises
+`AttributeError`. If the code you are debugging needs them, recreate them after the call.
+
+Only attributes are filtered. An unpicklable value at the top level, or directly inside a list, tuple,
+dict or set, still raises an exception. So does an object with a custom `__reduce__` or a `__getstate__`
+that does not return a dict.
+
 ### `reset()`
 Delete the whole cache directory. It refuses (raises `RuntimeError`) if the directory contains anything
 other than `.pickle` and `.tmp` files, to avoid deleting unrelated data if `enable()` was pointed at the
@@ -84,8 +97,10 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 
 ## Caveats
 
-- **Arguments and results must be picklable.** The arguments are pickled to build the key, and the result
-  is pickled to store it.
+- **Arguments and results must be picklable**, apart from object attributes, which are left out (see
+  [Unpicklable attributes](#unpicklable-attributes)). The arguments are pickled to build the key, and the
+  result is pickled to store it. Because dropped attributes are not part of the key, two arguments that
+  differ only in unpicklable attributes share a cache entry.
 - **The key is the pickled arguments.** Equal arguments that pickle differently (for example sets with a
   different iteration order) may miss the cache. Objects whose pickle changes between calls never hit it.
 - **The key does not include the function's code or any files it reads.** If you change the function or
@@ -99,4 +114,4 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 ## Logging
 
 The module logs to the `pickle_debug` logger at `DEBUG` level when it enables caching, reads from the
-cache, or stores a result.
+cache, stores a result, or leaves an unpicklable attribute out.

{pickle_debug-1.0 → pickle_debug-1.2}/README.md

@@ -65,6 +65,19 @@ Decorator. Returns a wrapper (using `functools.wraps`) that:
 Results are written to a temporary file and then renamed, so an interrupted run does not leave a partial
 cache entry.
 
+#### Unpicklable attributes
+If an object (in the result or in the arguments) has attributes that cannot be pickled, such as locks,
+open files, sockets or lambdas, those attributes are left out and the rest of the object is cached. This
+applies to nested objects too, and covers both `__dict__` and `__slots__` attributes. Each dropped
+attribute is logged at `DEBUG` level.
+
+An object loaded from the cache does not have the dropped attributes, so accessing them raises
+`AttributeError`. If the code you are debugging needs them, recreate them after the call.
+
+Only attributes are filtered. An unpicklable value at the top level, or directly inside a list, tuple,
+dict or set, still raises an exception. So does an object with a custom `__reduce__` or a `__getstate__`
+that does not return a dict.
+
 ### `reset()`
 Delete the whole cache directory. It refuses (raises `RuntimeError`) if the directory contains anything
 other than `.pickle` and `.tmp` files, to avoid deleting unrelated data if `enable()` was pointed at the
@@ -72,8 +85,10 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 
 ## Caveats
 
-- **Arguments and results must be picklable.** The arguments are pickled to build the key, and the result
-  is pickled to store it.
+- **Arguments and results must be picklable**, apart from object attributes, which are left out (see
+  [Unpicklable attributes](#unpicklable-attributes)). The arguments are pickled to build the key, and the
+  result is pickled to store it. Because dropped attributes are not part of the key, two arguments that
+  differ only in unpicklable attributes share a cache entry.
 - **The key is the pickled arguments.** Equal arguments that pickle differently (for example sets with a
   different iteration order) may miss the cache. Objects whose pickle changes between calls never hit it.
 - **The key does not include the function's code or any files it reads.** If you change the function or
@@ -87,4 +102,4 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 ## Logging
 
 The module logs to the `pickle_debug` logger at `DEBUG` level when it enables caching, reads from the
-cache, or stores a result.
+cache, stores a result, or leaves an unpicklable attribute out.

{pickle_debug-1.0 → pickle_debug-1.2}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "pickle_debug"
 description = "Store expensive objects from debugging"
 license = {text = "MIT" }
-version = "1.0"
+version = "1.2"
 dependencies = []
 requires-python= ">= 3.12"
 readme = "README.md"

pickle_debug-1.2/src/pickle_debug/__init__.py

@@ -0,0 +1,174 @@
+import functools
+import hashlib
+import importlib.metadata
+import io
+import logging
+import os
+import pickle
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Any, Callable
+
+pickle_debug_logger = logging.getLogger(__name__)
+
+__version__ = importlib.metadata.version('pickle-debug')
+
+_CACHE_DIRECTORY: Path | None = None
+_SUFFIX = '.pickle'
+
+
+def enable(directory: Path | str) -> None:
+    """Enable caching in specified directory. Raises OSError if directory can't be created or written"""
+    global _CACHE_DIRECTORY
+    directory = Path(directory)
+    directory.mkdir(parents=True, exist_ok=True)
+    # verify we can create and write files
+    with tempfile.TemporaryFile(dir=directory) as f:
+        f.write(b'test')
+    _CACHE_DIRECTORY = directory
+    pickle_debug_logger.debug(f"caching to {directory}")
+
+
+def reset() -> None:
+    """Remove entire cache directory. Raises exception if it exists and can't be removed"""
+    if _CACHE_DIRECTORY is None or not _CACHE_DIRECTORY.exists():
+        return
+    unexpected = [p for p in _CACHE_DIRECTORY.iterdir()
+                  if not (p.is_file() and not p.is_symlink() and p.suffix in (_SUFFIX, '.tmp'))]
+    if unexpected:
+        raise RuntimeError(f"Not removing {_CACHE_DIRECTORY}, unexpected contents: "
+                           + ', '.join(p.name for p in unexpected))
+    try:
+        shutil.rmtree(_CACHE_DIRECTORY)
+    except OSError as e:
+        raise RuntimeError(f"Unable to remove cache {_CACHE_DIRECTORY}") from e
+    if _CACHE_DIRECTORY.exists():
+        raise RuntimeError(f"Unable to remove cache {_CACHE_DIRECTORY}")
+
+
+class _FilteringPickler(pickle.Pickler):
+    """Pickler that leaves out object attributes which can't be pickled"""
+
+    def __init__(self, file, memo: dict[int, list] | None = None):
+        # C Pickler doesn't expose its protocol, so keep our own copy
+        self._protocol = pickle.DEFAULT_PROTOCOL
+        super().__init__(file, self._protocol)
+        # id -> [value, picklable], shared with nested test picklers. Holding value keeps id from being reused.
+        # picklable is True while test in progress to handle cycles
+        self._picklable_memo = {} if memo is None else memo
+
+    def _picklable(self, value: Any) -> bool:
+        key = id(value)
+        if key not in self._picklable_memo:
+            entry = self._picklable_memo[key] = [value, True]
+            try:
+                _FilteringPickler(io.BytesIO(), self._picklable_memo).dump(value)
+            except Exception:
+                entry[1] = False
+        return self._picklable_memo[key][1]
+
+    def _filter(self, obj: Any, state: dict) -> dict:
+        kept = {}
+        for name, value in state.items():
+            if self._picklable(value):
+                kept[name] = value
+            else:
+                pickle_debug_logger.debug(f"leaving unpicklable {type(obj).__qualname__}.{name} out of cache")
+        return kept
+
+    def reducer_override(self, obj):
+        if isinstance(obj, type) or not hasattr(obj, '__dict__') and not hasattr(type(obj), '__slots__'):
+            return NotImplemented
+        try:
+            reduced = obj.__reduce_ex__(self._protocol)
+        except Exception:
+            return NotImplemented
+        if not isinstance(reduced, tuple) or len(reduced) < 3:
+            return NotImplemented
+        state = reduced[2]
+        if isinstance(state, dict):
+            state = self._filter(obj, state)
+        elif (isinstance(state, tuple) and len(state) == 2
+              and all(s is None or isinstance(s, dict) for s in state)):
+            state = tuple(None if s is None else self._filter(obj, s) for s in state)
+        else:
+            return NotImplemented
+        return reduced[:2] + (state,) + reduced[3:]
+
+
+def _dumps(obj: Any) -> bytes:
+    buffer = io.BytesIO()
+    _FilteringPickler(buffer).dump(obj)
+    return buffer.getvalue()
+
+
+def _save(path: Path, result: Any) -> None:
+    """Write via temporary file so an interrupted write doesn't leave partial result"""
+    data = _dumps(result)
+    fd, tmp = tempfile.mkstemp(dir=path.parent, suffix='.tmp')
+    try:
+        with os.fdopen(fd, 'wb') as f:
+            f.write(data)
+        os.replace(tmp, path)
+    except BaseException:
+        Path(tmp).unlink(missing_ok=True)
+        raise
+
+
+def _is_method_receiver(obj: Any, name: str, wrapper: Callable) -> bool:
+    """True if obj is the self or cls that wrapper was called through as a method"""
+    cls = obj if isinstance(obj, type) else type(obj)
+    for klass in cls.__mro__:
+        attr = vars(klass).get(name)
+        if isinstance(attr, classmethod):
+            attr = attr.__func__
+        if attr is wrapper:
+            return True
+    return False
+
+
+def cached(func: Callable) -> Callable:
+    """Decorator: cache result of func in directory set by enable(), keyed by arguments.
+    When used on a method, self (or cls) is keyed by its class name rather than its current value"""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        directory = _CACHE_DIRECTORY
+        if directory is None:
+            return func(*args, **kwargs)
+        key_args = args
+        if args and _is_method_receiver(args[0], func.__name__, wrapper):
+            cls = args[0] if isinstance(args[0], type) else type(args[0])
+            key_args = (f"{cls.__module__}.{cls.__qualname__}",) + args[1:]
+        key = _dumps((func.__module__, func.__qualname__, key_args, sorted(kwargs.items())))
+        path = directory / (hashlib.sha256(key).hexdigest() + _SUFFIX)
+        try:
+            with open(path, 'rb') as f:
+                result = pickle.load(f)
+            pickle_debug_logger.debug(f"{func.__qualname__} read from {path}")
+            return result
+        except FileNotFoundError:
+            pass
+        result = func(*args, **kwargs)
+        directory.mkdir(parents=True, exist_ok=True)
+        _save(path, result)
+        pickle_debug_logger.debug(f"{func.__qualname__} stored in {path}")
+        return result
+
+    return wrapper
+
+def record(name:str,fn,*args,**kwargs):
+    p = None
+    if _CACHE_DIRECTORY is not None:
+        p = _CACHE_DIRECTORY / f"{name}.pickle"
+        if p.is_file():
+            with open(p,'rb') as f:
+                result = pickle.load(f)
+                return result
+    result = fn(*args,**kwargs)
+    if p is not None: 
+        with open(p,'wb') as f:
+            pickle.dump(result,f)
+    return result
+

{pickle_debug-1.0 → pickle_debug-1.2/src/pickle_debug.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pickle_debug
-Version: 1.0
+Version: 1.2
 Summary: Store expensive objects from debugging
 Author: Gerard
 Author-email: gweatherby@uchc.edu
@@ -77,6 +77,19 @@ Decorator. Returns a wrapper (using `functools.wraps`) that:
 Results are written to a temporary file and then renamed, so an interrupted run does not leave a partial
 cache entry.
 
+#### Unpicklable attributes
+If an object (in the result or in the arguments) has attributes that cannot be pickled, such as locks,
+open files, sockets or lambdas, those attributes are left out and the rest of the object is cached. This
+applies to nested objects too, and covers both `__dict__` and `__slots__` attributes. Each dropped
+attribute is logged at `DEBUG` level.
+
+An object loaded from the cache does not have the dropped attributes, so accessing them raises
+`AttributeError`. If the code you are debugging needs them, recreate them after the call.
+
+Only attributes are filtered. An unpicklable value at the top level, or directly inside a list, tuple,
+dict or set, still raises an exception. So does an object with a custom `__reduce__` or a `__getstate__`
+that does not return a dict.
+
 ### `reset()`
 Delete the whole cache directory. It refuses (raises `RuntimeError`) if the directory contains anything
 other than `.pickle` and `.tmp` files, to avoid deleting unrelated data if `enable()` was pointed at the
@@ -84,8 +97,10 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 
 ## Caveats
 
-- **Arguments and results must be picklable.** The arguments are pickled to build the key, and the result
-  is pickled to store it.
+- **Arguments and results must be picklable**, apart from object attributes, which are left out (see
+  [Unpicklable attributes](#unpicklable-attributes)). The arguments are pickled to build the key, and the
+  result is pickled to store it. Because dropped attributes are not part of the key, two arguments that
+  differ only in unpicklable attributes share a cache entry.
 - **The key is the pickled arguments.** Equal arguments that pickle differently (for example sets with a
   different iteration order) may miss the cache. Objects whose pickle changes between calls never hit it.
 - **The key does not include the function's code or any files it reads.** If you change the function or
@@ -99,4 +114,4 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 ## Logging
 
 The module logs to the `pickle_debug` logger at `DEBUG` level when it enables caching, reads from the
-cache, or stores a result.
+cache, stores a result, or leaves an unpicklable attribute out.

{pickle_debug-1.0 → pickle_debug-1.2}/src/pickle_debug.egg-info/SOURCES.txt

@@ -8,4 +8,6 @@ src/pickle_debug.egg-info/dependency_links.txt
 src/pickle_debug.egg-info/top_level.txt
 src/test/patch_test.py
 src/test/pd_test.py
-src/test/ttest.py
+src/test/record.py
+src/test/ttest.py
+src/test/unpicklable_test.py

{pickle_debug-1.0 → pickle_debug-1.2}/src/test/patch_test.py

@@ -8,9 +8,10 @@ def f(x, y=1):
     time.sleep(1.5)
     return x + y
 
+
 class Slow:
 
-    def invoke(self,x):
+    def invoke(self, x):
         time.sleep(1.5)
         return x * x
 
@@ -18,8 +19,8 @@ class Slow:
 pickle_debug.enable("/tmp/mycache")
 
 f = pickle_debug.cached(f)
-print(f(5,6))
-print(f(5,6))
+print(f(5, 6))
+print(f(5, 6))
 
 slow = Slow()
 slow.invoke = pickle_debug.cached(slow.invoke)

pickle_debug-1.2/src/test/record.py

@@ -0,0 +1,11 @@
+import pickle_debug
+
+def sum(x,y):
+    print(f"{x} + {y}")
+    return x + y
+
+pickle_debug.enable("/tmp/sum")
+ans = pickle_debug.record("sum",sum,3,4)
+print(ans)
+ans = pickle_debug.record("sum",sum,3,4)
+print(ans)

{pickle_debug-1.0 → pickle_debug-1.2}/src/test/ttest.py

@@ -7,14 +7,14 @@ import pickle_debug
 def f(x, y=1):
     print(f"call {x} {y}")
     time.sleep(1.5)
-    return x + y  
+    return x + y
 
-print(f(1,2))
-print(f(1,2))
-pickle_debug.enable("/tmp/mycache")
-print(f(1,2))
-print(f(1,2))
-print(f(3,4))
-print(f(1,2))
-print(f(3,4))
 
+print(f(1, 2))
+print(f(1, 2))
+pickle_debug.enable("/tmp/mycache")
+print(f(1, 2))
+print(f(1, 2))
+print(f(3, 4))
+print(f(1, 2))
+print(f(3, 4))

pickle_debug-1.2/src/test/unpicklable_test.py

@@ -0,0 +1,46 @@
+import pathlib
+import tempfile
+import threading
+
+from pickle_debug import cached, enable
+
+
+class Holder:
+    def __init__(self, value):
+        self.value = value
+        self.lock = threading.Lock()
+        self.inner = Inner()
+        self.me = self
+
+
+class Inner:
+    def __init__(self):
+        self.ok = 'inner'
+        self.callback = lambda: None
+
+
+class Slotted:
+    __slots__ = ('a', 'lock')
+
+    def __init__(self):
+        self.a = 1
+        self.lock = threading.Lock()
+
+
+calls = []
+
+
+@cached
+def make(x):
+    calls.append(x)
+    return Holder(x), Slotted()
+
+
+enable(pathlib.Path(tempfile.mkdtemp()) / 'cache')
+make(5)
+h, s = make(5)
+assert calls == [5], calls
+assert h.value == 5 and not hasattr(h, 'lock') and h.me is h
+assert h.inner.ok == 'inner' and not hasattr(h.inner, 'callback')
+assert s.a == 1 and not hasattr(s, 'lock')
+print('ok')

pickle_debug-1.0/src/pickle_debug/__init__.py

@@ -1,84 +0,0 @@
-import functools
-import hashlib
-import importlib.metadata
-import logging
-import os
-import pickle
-import shutil
-import tempfile
-from pathlib import Path
-from typing import Any, Callable
-
-pickle_debug_logger = logging.getLogger(__name__)
-
-__version__ = importlib.metadata.version('pickle-debug')
-
-_CACHE_DIRECTORY: Path | None = None
-_SUFFIX = '.pickle'
-
-
-def enable(directory: Path | str) -> None:
-    """Enable caching in specified directory. Raises OSError if directory can't be created or written"""
-    global _CACHE_DIRECTORY
-    directory = Path(directory)
-    directory.mkdir(parents=True, exist_ok=True)
-    # verify we can create and write files
-    with tempfile.TemporaryFile(dir=directory) as f:
-        f.write(b'test')
-    _CACHE_DIRECTORY = directory
-    pickle_debug_logger.debug(f"caching to {directory}")
-
-
-def reset() -> None:
-    """Remove entire cache directory. Raises exception if it exists and can't be removed"""
-    if _CACHE_DIRECTORY is None or not _CACHE_DIRECTORY.exists():
-        return
-    unexpected = [p for p in _CACHE_DIRECTORY.iterdir()
-                  if not (p.is_file() and not p.is_symlink() and p.suffix in (_SUFFIX, '.tmp'))]
-    if unexpected:
-        raise RuntimeError(f"Not removing {_CACHE_DIRECTORY}, unexpected contents: "
-                           + ', '.join(p.name for p in unexpected))
-    try:
-        shutil.rmtree(_CACHE_DIRECTORY)
-    except OSError as e:
-        raise RuntimeError(f"Unable to remove cache {_CACHE_DIRECTORY}") from e
-    if _CACHE_DIRECTORY.exists():
-        raise RuntimeError(f"Unable to remove cache {_CACHE_DIRECTORY}")
-
-
-def _save(path: Path, result: Any) -> None:
-    """Write via temporary file so an interrupted write doesn't leave partial result"""
-    fd, tmp = tempfile.mkstemp(dir=path.parent, suffix='.tmp')
-    try:
-        with os.fdopen(fd, 'wb') as f:
-            pickle.dump(result, f)
-        os.replace(tmp, path)
-    except BaseException:
-        Path(tmp).unlink(missing_ok=True)
-        raise
-
-
-def cached(func: Callable) -> Callable:
-    """Decorator: cache result of func in directory set by enable(), keyed by arguments"""
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        directory = _CACHE_DIRECTORY
-        if directory is None:
-            return func(*args, **kwargs)
-        key = pickle.dumps((func.__module__, func.__qualname__, args, sorted(kwargs.items())))
-        path = directory / (hashlib.sha256(key).hexdigest() + _SUFFIX)
-        try:
-            with open(path, 'rb') as f:
-                result = pickle.load(f)
-            pickle_debug_logger.debug(f"{func.__qualname__} read from {path}")
-            return result
-        except FileNotFoundError:
-            pass
-        result = func(*args, **kwargs)
-        directory.mkdir(parents=True, exist_ok=True)
-        _save(path, result)
-        pickle_debug_logger.debug(f"{func.__qualname__} stored in {path}")
-        return result
-
-    return wrapper