pickle-debug 1.0__tar.gz → 1.1__tar.gz

{pickle_debug-1.0/src/pickle_debug.egg-info → pickle_debug-1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pickle_debug
-Version: 1.0
+Version: 1.1
 Summary: Store expensive objects from debugging
 Author: Gerard
 Author-email: gweatherby@uchc.edu
@@ -77,6 +77,19 @@ Decorator. Returns a wrapper (using `functools.wraps`) that:
 Results are written to a temporary file and then renamed, so an interrupted run does not leave a partial
 cache entry.
 
+#### Unpicklable attributes
+If an object (in the result or in the arguments) has attributes that cannot be pickled, such as locks,
+open files, sockets or lambdas, those attributes are left out and the rest of the object is cached. This
+applies to nested objects too, and covers both `__dict__` and `__slots__` attributes. Each dropped
+attribute is logged at `DEBUG` level.
+
+An object loaded from the cache does not have the dropped attributes, so accessing them raises
+`AttributeError`. If the code you are debugging needs them, recreate them after the call.
+
+Only attributes are filtered. An unpicklable value at the top level, or directly inside a list, tuple,
+dict or set, still raises an exception. So does an object with a custom `__reduce__` or a `__getstate__`
+that does not return a dict.
+
 ### `reset()`
 Delete the whole cache directory. It refuses (raises `RuntimeError`) if the directory contains anything
 other than `.pickle` and `.tmp` files, to avoid deleting unrelated data if `enable()` was pointed at the
@@ -84,8 +97,10 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 
 ## Caveats
 
-- **Arguments and results must be picklable.** The arguments are pickled to build the key, and the result
-  is pickled to store it.
+- **Arguments and results must be picklable**, apart from object attributes, which are left out (see
+  [Unpicklable attributes](#unpicklable-attributes)). The arguments are pickled to build the key, and the
+  result is pickled to store it. Because dropped attributes are not part of the key, two arguments that
+  differ only in unpicklable attributes share a cache entry.
 - **The key is the pickled arguments.** Equal arguments that pickle differently (for example sets with a
   different iteration order) may miss the cache. Objects whose pickle changes between calls never hit it.
 - **The key does not include the function's code or any files it reads.** If you change the function or
@@ -99,4 +114,4 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 ## Logging
 
 The module logs to the `pickle_debug` logger at `DEBUG` level when it enables caching, reads from the
-cache, or stores a result.
+cache, stores a result, or leaves an unpicklable attribute out.

{pickle_debug-1.0 → pickle_debug-1.1}/README.md

@@ -65,6 +65,19 @@ Decorator. Returns a wrapper (using `functools.wraps`) that:
 Results are written to a temporary file and then renamed, so an interrupted run does not leave a partial
 cache entry.
 
+#### Unpicklable attributes
+If an object (in the result or in the arguments) has attributes that cannot be pickled, such as locks,
+open files, sockets or lambdas, those attributes are left out and the rest of the object is cached. This
+applies to nested objects too, and covers both `__dict__` and `__slots__` attributes. Each dropped
+attribute is logged at `DEBUG` level.
+
+An object loaded from the cache does not have the dropped attributes, so accessing them raises
+`AttributeError`. If the code you are debugging needs them, recreate them after the call.
+
+Only attributes are filtered. An unpicklable value at the top level, or directly inside a list, tuple,
+dict or set, still raises an exception. So does an object with a custom `__reduce__` or a `__getstate__`
+that does not return a dict.
+
 ### `reset()`
 Delete the whole cache directory. It refuses (raises `RuntimeError`) if the directory contains anything
 other than `.pickle` and `.tmp` files, to avoid deleting unrelated data if `enable()` was pointed at the
@@ -72,8 +85,10 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 
 ## Caveats
 
-- **Arguments and results must be picklable.** The arguments are pickled to build the key, and the result
-  is pickled to store it.
+- **Arguments and results must be picklable**, apart from object attributes, which are left out (see
+  [Unpicklable attributes](#unpicklable-attributes)). The arguments are pickled to build the key, and the
+  result is pickled to store it. Because dropped attributes are not part of the key, two arguments that
+  differ only in unpicklable attributes share a cache entry.
 - **The key is the pickled arguments.** Equal arguments that pickle differently (for example sets with a
   different iteration order) may miss the cache. Objects whose pickle changes between calls never hit it.
 - **The key does not include the function's code or any files it reads.** If you change the function or
@@ -87,4 +102,4 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 ## Logging
 
 The module logs to the `pickle_debug` logger at `DEBUG` level when it enables caching, reads from the
-cache, or stores a result.
+cache, stores a result, or leaves an unpicklable attribute out.

{pickle_debug-1.0 → pickle_debug-1.1}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "pickle_debug"
 description = "Store expensive objects from debugging"
 license = {text = "MIT" }
-version = "1.0"
+version = "1.1"
 dependencies = []
 requires-python= ">= 3.12"
 readme = "README.md"

{pickle_debug-1.0 → pickle_debug-1.1}/src/pickle_debug/__init__.py

@@ -1,6 +1,7 @@
 import functools
 import hashlib
 import importlib.metadata
+import io
 import logging
 import os
 import pickle
@@ -46,12 +47,69 @@ def reset() -> None:
         raise RuntimeError(f"Unable to remove cache {_CACHE_DIRECTORY}")
 
 
+class _FilteringPickler(pickle.Pickler):
+    """Pickler that leaves out object attributes which can't be pickled"""
+
+    def __init__(self, file, memo: dict[int, list] | None = None):
+        # C Pickler doesn't expose its protocol, so keep our own copy
+        self._protocol = pickle.DEFAULT_PROTOCOL
+        super().__init__(file, self._protocol)
+        # id -> [value, picklable], shared with nested test picklers. Holding value keeps id from being reused.
+        # picklable is True while test in progress to handle cycles
+        self._picklable_memo = {} if memo is None else memo
+
+    def _picklable(self, value: Any) -> bool:
+        key = id(value)
+        if key not in self._picklable_memo:
+            entry = self._picklable_memo[key] = [value, True]
+            try:
+                _FilteringPickler(io.BytesIO(), self._picklable_memo).dump(value)
+            except Exception:
+                entry[1] = False
+        return self._picklable_memo[key][1]
+
+    def _filter(self, obj: Any, state: dict) -> dict:
+        kept = {}
+        for name, value in state.items():
+            if self._picklable(value):
+                kept[name] = value
+            else:
+                pickle_debug_logger.debug(f"leaving unpicklable {type(obj).__qualname__}.{name} out of cache")
+        return kept
+
+    def reducer_override(self, obj):
+        if isinstance(obj, type) or not hasattr(obj, '__dict__') and not hasattr(type(obj), '__slots__'):
+            return NotImplemented
+        try:
+            reduced = obj.__reduce_ex__(self._protocol)
+        except Exception:
+            return NotImplemented
+        if not isinstance(reduced, tuple) or len(reduced) < 3:
+            return NotImplemented
+        state = reduced[2]
+        if isinstance(state, dict):
+            state = self._filter(obj, state)
+        elif (isinstance(state, tuple) and len(state) == 2
+              and all(s is None or isinstance(s, dict) for s in state)):
+            state = tuple(None if s is None else self._filter(obj, s) for s in state)
+        else:
+            return NotImplemented
+        return reduced[:2] + (state,) + reduced[3:]
+
+
+def _dumps(obj: Any) -> bytes:
+    buffer = io.BytesIO()
+    _FilteringPickler(buffer).dump(obj)
+    return buffer.getvalue()
+
+
 def _save(path: Path, result: Any) -> None:
     """Write via temporary file so an interrupted write doesn't leave partial result"""
+    data = _dumps(result)
     fd, tmp = tempfile.mkstemp(dir=path.parent, suffix='.tmp')
     try:
         with os.fdopen(fd, 'wb') as f:
-            pickle.dump(result, f)
+            f.write(data)
         os.replace(tmp, path)
     except BaseException:
         Path(tmp).unlink(missing_ok=True)
@@ -66,7 +124,7 @@ def cached(func: Callable) -> Callable:
         directory = _CACHE_DIRECTORY
         if directory is None:
             return func(*args, **kwargs)
-        key = pickle.dumps((func.__module__, func.__qualname__, args, sorted(kwargs.items())))
+        key = _dumps((func.__module__, func.__qualname__, args, sorted(kwargs.items())))
         path = directory / (hashlib.sha256(key).hexdigest() + _SUFFIX)
         try:
             with open(path, 'rb') as f:

{pickle_debug-1.0 → pickle_debug-1.1/src/pickle_debug.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pickle_debug
-Version: 1.0
+Version: 1.1
 Summary: Store expensive objects from debugging
 Author: Gerard
 Author-email: gweatherby@uchc.edu
@@ -77,6 +77,19 @@ Decorator. Returns a wrapper (using `functools.wraps`) that:
 Results are written to a temporary file and then renamed, so an interrupted run does not leave a partial
 cache entry.
 
+#### Unpicklable attributes
+If an object (in the result or in the arguments) has attributes that cannot be pickled, such as locks,
+open files, sockets or lambdas, those attributes are left out and the rest of the object is cached. This
+applies to nested objects too, and covers both `__dict__` and `__slots__` attributes. Each dropped
+attribute is logged at `DEBUG` level.
+
+An object loaded from the cache does not have the dropped attributes, so accessing them raises
+`AttributeError`. If the code you are debugging needs them, recreate them after the call.
+
+Only attributes are filtered. An unpicklable value at the top level, or directly inside a list, tuple,
+dict or set, still raises an exception. So does an object with a custom `__reduce__` or a `__getstate__`
+that does not return a dict.
+
 ### `reset()`
 Delete the whole cache directory. It refuses (raises `RuntimeError`) if the directory contains anything
 other than `.pickle` and `.tmp` files, to avoid deleting unrelated data if `enable()` was pointed at the
@@ -84,8 +97,10 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 
 ## Caveats
 
-- **Arguments and results must be picklable.** The arguments are pickled to build the key, and the result
-  is pickled to store it.
+- **Arguments and results must be picklable**, apart from object attributes, which are left out (see
+  [Unpicklable attributes](#unpicklable-attributes)). The arguments are pickled to build the key, and the
+  result is pickled to store it. Because dropped attributes are not part of the key, two arguments that
+  differ only in unpicklable attributes share a cache entry.
 - **The key is the pickled arguments.** Equal arguments that pickle differently (for example sets with a
   different iteration order) may miss the cache. Objects whose pickle changes between calls never hit it.
 - **The key does not include the function's code or any files it reads.** If you change the function or
@@ -99,4 +114,4 @@ wrong place. It does nothing if caching is not enabled or the directory does not
 ## Logging
 
 The module logs to the `pickle_debug` logger at `DEBUG` level when it enables caching, reads from the
-cache, or stores a result.
+cache, stores a result, or leaves an unpicklable attribute out.

{pickle_debug-1.0 → pickle_debug-1.1}/src/pickle_debug.egg-info/SOURCES.txt

@@ -8,4 +8,5 @@ src/pickle_debug.egg-info/dependency_links.txt
 src/pickle_debug.egg-info/top_level.txt
 src/test/patch_test.py
 src/test/pd_test.py
-src/test/ttest.py
+src/test/ttest.py
+src/test/unpicklable_test.py

{pickle_debug-1.0 → pickle_debug-1.1}/src/test/patch_test.py

@@ -8,9 +8,10 @@ def f(x, y=1):
     time.sleep(1.5)
     return x + y
 
+
 class Slow:
 
-    def invoke(self,x):
+    def invoke(self, x):
         time.sleep(1.5)
         return x * x
 
@@ -18,8 +19,8 @@ class Slow:
 pickle_debug.enable("/tmp/mycache")
 
 f = pickle_debug.cached(f)
-print(f(5,6))
-print(f(5,6))
+print(f(5, 6))
+print(f(5, 6))
 
 slow = Slow()
 slow.invoke = pickle_debug.cached(slow.invoke)

{pickle_debug-1.0 → pickle_debug-1.1}/src/test/ttest.py

@@ -7,14 +7,14 @@ import pickle_debug
 def f(x, y=1):
     print(f"call {x} {y}")
     time.sleep(1.5)
-    return x + y  
+    return x + y
 
-print(f(1,2))
-print(f(1,2))
-pickle_debug.enable("/tmp/mycache")
-print(f(1,2))
-print(f(1,2))
-print(f(3,4))
-print(f(1,2))
-print(f(3,4))
 
+print(f(1, 2))
+print(f(1, 2))
+pickle_debug.enable("/tmp/mycache")
+print(f(1, 2))
+print(f(1, 2))
+print(f(3, 4))
+print(f(1, 2))
+print(f(3, 4))

pickle_debug-1.1/src/test/unpicklable_test.py

@@ -0,0 +1,46 @@
+import pathlib
+import tempfile
+import threading
+
+from pickle_debug import cached, enable
+
+
+class Holder:
+    def __init__(self, value):
+        self.value = value
+        self.lock = threading.Lock()
+        self.inner = Inner()
+        self.me = self
+
+
+class Inner:
+    def __init__(self):
+        self.ok = 'inner'
+        self.callback = lambda: None
+
+
+class Slotted:
+    __slots__ = ('a', 'lock')
+
+    def __init__(self):
+        self.a = 1
+        self.lock = threading.Lock()
+
+
+calls = []
+
+
+@cached
+def make(x):
+    calls.append(x)
+    return Holder(x), Slotted()
+
+
+enable(pathlib.Path(tempfile.mkdtemp()) / 'cache')
+make(5)
+h, s = make(5)
+assert calls == [5], calls
+assert h.value == 5 and not hasattr(h, 'lock') and h.me is h
+assert h.inner.ok == 'inner' and not hasattr(h.inner, 'callback')
+assert s.a == 1 and not hasattr(s, 'lock')
+print('ok')