cachier 3.3.0__py3-none-any.whl → 3.4.1__py3-none-any.whl

cachier/_types.py

@@ -1,9 +1,11 @@
-from typing import TYPE_CHECKING, Callable, Literal
+from typing import TYPE_CHECKING, Callable, Literal, Union
 
 if TYPE_CHECKING:
     import pymongo.collection
+    import redis
 
 
 HashFunc = Callable[..., str]
 Mongetter = Callable[[], "pymongo.collection.Collection"]
-Backend = Literal["pickle", "mongo", "memory"]
+RedisClient = Union["redis.Redis", Callable[[], "redis.Redis"]]
+Backend = Literal["pickle", "mongo", "memory", "redis"]

cachier/core.py

@@ -17,6 +17,7 @@ from functools import wraps
 from typing import Any, Callable, Optional, Union
 from warnings import warn
 
+from ._types import RedisClient
 from .config import (
     Backend,
     HashFunc,
@@ -27,6 +28,7 @@ from .cores.base import RecalculationNeeded, _BaseCore
 from .cores.memory import _MemoryCore
 from .cores.mongo import _MongoCore
 from .cores.pickle import _PickleCore
+from .cores.redis import _RedisCore
 from .cores.sql import _SQLCore
 
 MAX_WORKERS_ENVAR_NAME = "CACHIER_MAX_WORKERS"
@@ -110,6 +112,7 @@ def cachier(
     backend: Optional[Backend] = None,
     mongetter: Optional[Mongetter] = None,
     sql_engine: Optional[Union[str, Any, Callable[[], Any]]] = None,
+    redis_client: Optional["RedisClient"] = None,
     stale_after: Optional[timedelta] = None,
     next_time: Optional[bool] = None,
     cache_dir: Optional[Union[str, os.PathLike]] = None,
@@ -137,9 +140,9 @@ def cachier(
     hash_params : callable, optional
     backend : str, optional
         The name of the backend to use. Valid options currently include
-        'pickle', 'mongo', 'memory', and 'sql'. If not provided, defaults to
-        'pickle' unless the 'mongetter' argument is passed, in which
-        case the mongo backend is automatically selected.
+        'pickle', 'mongo', 'memory', 'sql', and 'redis'. If not provided,
+        defaults to 'pickle', unless a core-associated parameter is provided
+
     mongetter : callable, optional
         A callable that takes no arguments and returns a pymongo.Collection
         object with writing permissions. If unset a local pickle cache is used
@@ -147,6 +150,9 @@ def cachier(
     sql_engine : str, Engine, or callable, optional
         SQLAlchemy connection string, Engine, or callable returning an Engine.
         Used for the SQL backend.
+    redis_client : redis.Redis or callable, optional
+        Redis client instance or callable returning a Redis client.
+        Used for the Redis backend.
     stale_after : datetime.timedelta, optional
         The time delta after which a cached result is considered stale. Calls
         made after the result goes stale will trigger a recalculation of the
@@ -220,6 +226,12 @@ def cachier(
             sql_engine=sql_engine,
             wait_for_calc_timeout=wait_for_calc_timeout,
         )
+    elif backend == "redis":
+        core = _RedisCore(
+            hash_func=hash_func,
+            redis_client=redis_client,
+            wait_for_calc_timeout=wait_for_calc_timeout,
+        )
     else:
         raise ValueError("specified an invalid core: %s" % backend)
 

cachier/cores/mongo.py

@@ -77,7 +77,7 @@ class _MongoCore(_BaseCore):
             return key, None
         val = None
         if "value" in res:
-            val = pickle.loads(res["value"])  # noqa: S301
+            val = pickle.loads(res["value"])
         entry = CacheEntry(
             value=val,
             time=res.get("time", None),

cachier/cores/pickle.py

@@ -6,8 +6,10 @@
 # Licensed under the MIT license:
 # http://www.opensource.org/licenses/MIT-license
 # Copyright (c) 2016, Shay Palachy <shaypal5@gmail.com>
+import logging
 import os
 import pickle  # for local caching
+import time
 from datetime import datetime
 from typing import Any, Dict, Optional, Tuple, Union
 
@@ -51,12 +53,14 @@ class _PickleCore(_BaseCore):
                 if not entry._processing:
                     # print('stopping observer!')
                     self.value = entry.value
-                    self.observer.stop()
+                    if self.observer is not None:
+                        self.observer.stop()
                 # else:
                 #     print('NOT stopping observer... :(')
             except AttributeError:  # catching entry being None
                 self.value = None
-                self.observer.stop()
+                if self.observer is not None:
+                    self.observer.stop()
 
         def on_created(self, event) -> None:
             """A Watchdog Event Handler method."""  # noqa: D401
@@ -114,7 +118,7 @@ class _PickleCore(_BaseCore):
     def _load_cache_dict(self) -> Dict[str, CacheEntry]:
         try:
             with portalocker.Lock(self.cache_fpath, mode="rb") as cf:
-                cache = pickle.load(cf)  # noqa: S301
+                cache = pickle.load(cf)
             self._cache_used_fpath = str(self.cache_fpath)
         except (FileNotFoundError, EOFError):
             cache = {}
@@ -141,7 +145,7 @@ class _PickleCore(_BaseCore):
         fpath += f"_{hash_str or key}"
         try:
             with portalocker.Lock(fpath, mode="rb") as cache_file:
-                entry = pickle.load(cache_file)  # noqa: S301
+                entry = pickle.load(cache_file)
             return _PickleCore._convert_legacy_cache_entry(entry)
         except (FileNotFoundError, EOFError):
             return None
@@ -256,29 +260,93 @@ class _PickleCore(_BaseCore):
                 cache[key]._processing = False
                 self._save_cache(cache)
 
+    def _create_observer(self) -> Observer:
+        """Create a new observer instance."""
+        return Observer()
+
+    def _cleanup_observer(self, observer: Observer) -> None:
+        """Clean up observer properly."""
+        try:
+            if observer.is_alive():
+                observer.stop()
+                observer.join(timeout=1.0)
+        except Exception as e:
+            logging.debug("Observer cleanup failed: %s", e)
+
     def wait_on_entry_calc(self, key: str) -> Any:
+        """Wait for entry calculation to complete with inotify protection."""
         if self.separate_files:
             entry = self._load_cache_by_key(key)
             filename = f"{self.cache_fname}_{key}"
         else:
             with self.lock:
-                entry = self.get_cache_dict()[key]
+                entry = self.get_cache_dict().get(key)
             filename = self.cache_fname
+
         if entry and not entry._processing:
             return entry.value
+
+        # Try to use inotify-based waiting
+        try:
+            return self._wait_with_inotify(key, filename)
+        except OSError as e:
+            if "inotify instance limit reached" in str(e):
+                # Fall back to polling if inotify limit is reached
+                return self._wait_with_polling(key)
+            else:
+                raise
+
+    def _wait_with_inotify(self, key: str, filename: str) -> Any:
+        """Wait for calculation using inotify with proper cleanup."""
         event_handler = _PickleCore.CacheChangeHandler(
             filename=filename, core=self, key=key
         )
-        observer = Observer()
+
+        observer = self._create_observer()
         event_handler.inject_observer(observer)
-        observer.schedule(event_handler, path=self.cache_dir, recursive=True)
-        observer.start()
+
+        try:
+            observer.schedule(
+                event_handler, path=self.cache_dir, recursive=True
+            )
+            observer.start()
+
+            time_spent = 0
+            while observer.is_alive():
+                observer.join(timeout=1.0)
+                time_spent += 1
+                self.check_calc_timeout(time_spent)
+
+                # Check if calculation is complete
+                if event_handler.value is not None:
+                    break
+
+            return event_handler.value
+        finally:
+            # Always cleanup the observer
+            self._cleanup_observer(observer)
+
+    def _wait_with_polling(self, key: str) -> Any:
+        """Fallback method using polling instead of inotify."""
         time_spent = 0
-        while observer.is_alive():
-            observer.join(timeout=1.0)
+        while True:
+            time.sleep(1)  # Poll every 1 second (matching other cores)
             time_spent += 1
-            self.check_calc_timeout(time_spent)
-        return event_handler.value
+
+            try:
+                if self.separate_files:
+                    entry = self._load_cache_by_key(key)
+                else:
+                    with self.lock:
+                        entry = self.get_cache_dict().get(key)
+
+                if entry and not entry._processing:
+                    return entry.value
+
+                self.check_calc_timeout(time_spent)
+            except (FileNotFoundError, EOFError):
+                # Continue polling even if there are file errors
+                pass
 
     def clear_cache(self) -> None:
         if self.separate_files:

cachier/cores/redis.py

@@ -0,0 +1,225 @@
+"""A Redis-based caching core for cachier."""
+
+import pickle
+import time
+import warnings
+from datetime import datetime
+from typing import Any, Callable, Optional, Tuple, Union
+
+try:
+    import redis
+
+    REDIS_AVAILABLE = True
+except ImportError:
+    REDIS_AVAILABLE = False
+
+from .._types import HashFunc
+from ..config import CacheEntry
+from .base import RecalculationNeeded, _BaseCore, _get_func_str
+
+REDIS_SLEEP_DURATION_IN_SEC = 1
+
+
+class MissingRedisClient(ValueError):
+    """Thrown when the redis_client keyword argument is missing."""
+
+
+class _RedisCore(_BaseCore):
+    """Redis-based core for Cachier, supporting Redis backends."""
+
+    def __init__(
+        self,
+        hash_func: Optional[HashFunc],
+        redis_client: Optional[
+            Union["redis.Redis", Callable[[], "redis.Redis"]]
+        ],
+        wait_for_calc_timeout: Optional[int] = None,
+        key_prefix: str = "cachier",
+    ):
+        if not REDIS_AVAILABLE:
+            warnings.warn(
+                "`redis` was not found. Redis cores will not function. "
+                "Install with `pip install redis`.",
+                ImportWarning,
+                stacklevel=2,
+            )
+
+        super().__init__(
+            hash_func=hash_func, wait_for_calc_timeout=wait_for_calc_timeout
+        )
+        if redis_client is None:
+            raise MissingRedisClient(
+                "must specify ``redis_client`` when using the redis core"
+            )
+        self.redis_client = redis_client
+        self.key_prefix = key_prefix
+        self._func_str = None
+
+    def _resolve_redis_client(self):
+        """Resolve the Redis client from the provided parameter."""
+        if callable(self.redis_client):
+            return self.redis_client()
+        return self.redis_client
+
+    def _get_redis_key(self, key: str) -> str:
+        """Generate a Redis key for the given cache key."""
+        return f"{self.key_prefix}:{self._func_str}:{key}"
+
+    def set_func(self, func):
+        """Set the function this core will use."""
+        super().set_func(func)
+        self._func_str = _get_func_str(func)
+
+    def get_entry_by_key(self, key: str) -> Tuple[str, Optional[CacheEntry]]:
+        """Get entry based on given key from Redis."""
+        redis_client = self._resolve_redis_client()
+        redis_key = self._get_redis_key(key)
+
+        try:
+            # Get the cached data from Redis
+            cached_data = redis_client.hgetall(redis_key)
+            if not cached_data:
+                return key, None
+
+            # Deserialize the value
+            value = None
+            if cached_data.get(b"value"):
+                value = pickle.loads(cached_data[b"value"])
+
+            # Parse timestamp
+            timestamp_str = cached_data.get(b"timestamp", b"").decode("utf-8")
+            timestamp = (
+                datetime.fromisoformat(timestamp_str)
+                if timestamp_str
+                else datetime.now()
+            )
+
+            # Parse boolean fields
+            stale = (
+                cached_data.get(b"stale", b"false").decode("utf-8").lower()
+                == "true"
+            )
+            processing = (
+                cached_data.get(b"processing", b"false")
+                .decode("utf-8")
+                .lower()
+                == "true"
+            )
+            completed = (
+                cached_data.get(b"completed", b"false").decode("utf-8").lower()
+                == "true"
+            )
+
+            entry = CacheEntry(
+                value=value,
+                time=timestamp,
+                stale=stale,
+                _processing=processing,
+                _completed=completed,
+            )
+            return key, entry
+        except Exception as e:
+            warnings.warn(f"Redis get_entry_by_key failed: {e}", stacklevel=2)
+            return key, None
+
+    def set_entry(self, key: str, func_res: Any) -> None:
+        """Map the given result to the given key in Redis."""
+        redis_client = self._resolve_redis_client()
+        redis_key = self._get_redis_key(key)
+
+        try:
+            # Serialize the value
+            value_bytes = pickle.dumps(func_res)
+            now = datetime.now()
+
+            # Store in Redis using hash
+            redis_client.hset(
+                redis_key,
+                mapping={
+                    "value": value_bytes,
+                    "timestamp": now.isoformat(),
+                    "stale": "false",
+                    "processing": "false",
+                    "completed": "true",
+                },
+            )
+        except Exception as e:
+            warnings.warn(f"Redis set_entry failed: {e}", stacklevel=2)
+
+    def mark_entry_being_calculated(self, key: str) -> None:
+        """Mark the entry mapped by the given key as being calculated."""
+        redis_client = self._resolve_redis_client()
+        redis_key = self._get_redis_key(key)
+
+        try:
+            now = datetime.now()
+            redis_client.hset(
+                redis_key,
+                mapping={
+                    "timestamp": now.isoformat(),
+                    "stale": "false",
+                    "processing": "true",
+                    "completed": "false",
+                },
+            )
+        except Exception as e:
+            warnings.warn(
+                f"Redis mark_entry_being_calculated failed: {e}", stacklevel=2
+            )
+
+    def mark_entry_not_calculated(self, key: str) -> None:
+        """Mark the entry mapped by the given key as not being calculated."""
+        redis_client = self._resolve_redis_client()
+        redis_key = self._get_redis_key(key)
+
+        try:
+            redis_client.hset(redis_key, "processing", "false")
+        except Exception as e:
+            warnings.warn(
+                f"Redis mark_entry_not_calculated failed: {e}", stacklevel=2
+            )
+
+    def wait_on_entry_calc(self, key: str) -> Any:
+        """Wait on the entry with keys being calculated and returns result."""
+        time_spent = 0
+        while True:
+            time.sleep(REDIS_SLEEP_DURATION_IN_SEC)
+            time_spent += REDIS_SLEEP_DURATION_IN_SEC
+            key, entry = self.get_entry_by_key(key)
+            if entry is None:
+                raise RecalculationNeeded()
+            if not entry._processing:
+                return entry.value
+            self.check_calc_timeout(time_spent)
+
+    def clear_cache(self) -> None:
+        """Clear the cache of this core."""
+        redis_client = self._resolve_redis_client()
+        pattern = f"{self.key_prefix}:{self._func_str}:*"
+
+        try:
+            # Find all keys matching the pattern
+            keys = redis_client.keys(pattern)
+            if keys:
+                redis_client.delete(*keys)
+        except Exception as e:
+            warnings.warn(f"Redis clear_cache failed: {e}", stacklevel=2)
+
+    def clear_being_calculated(self) -> None:
+        """Mark all entries in this cache as not being calculated."""
+        redis_client = self._resolve_redis_client()
+        pattern = f"{self.key_prefix}:{self._func_str}:*"
+
+        try:
+            # Find all keys matching the pattern
+            keys = redis_client.keys(pattern)
+            if keys:
+                # Use pipeline for efficiency
+                pipe = redis_client.pipeline()
+                for key in keys:
+                    pipe.hset(key, "processing", "false")
+                pipe.execute()
+        except Exception as e:
+            warnings.warn(
+                f"Redis clear_being_calculated failed: {e}", stacklevel=2
+            )

cachier/version.info

@@ -1 +1 @@
-3.3.0
+3.4.1

{cachier-3.3.0.dist-info → cachier-3.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cachier
-Version: 3.3.0
+Version: 3.4.1
 Summary: Persistent, stale-free, local and cross-machine caching for Python functions.
 Author-email: Shay Palachy Affek <shay.palachy@gmail.com>
 License: MIT License
@@ -99,6 +99,7 @@ Features
 * Defining "shelf life" for cached values.
 * Local caching using pickle files.
 * Cross-machine caching using MongoDB.
+* Redis-based caching for high-performance scenarios.
 * Thread-safety.
 * **Per-call max age:** Specify a maximum age for cached values per call.
 
@@ -448,6 +449,56 @@ Cachier supports a generic SQL backend via SQLAlchemy, allowing you to use SQLit
     def my_func(x):
         return x * 2
 
+Redis Core
+----------
+
+**Note:** The Redis core requires the redis package to be installed. It is not installed by default with cachier. To use the Redis backend, run::
+
+    pip install redis
+
+Cachier supports Redis-based caching for high-performance scenarios. Redis provides fast in-memory storage with optional persistence.
+
+**Usage Example (Local Redis):**
+
+.. code-block:: python
+
+    import redis
+    from cachier import cachier
+
+    # Create Redis client
+    redis_client = redis.Redis(host='localhost', port=6379, db=0)
+
+    @cachier(backend="redis", redis_client=redis_client)
+    def my_func(x):
+        return x * 2
+
+**Usage Example (Redis with custom key prefix):**
+
+.. code-block:: python
+
+    import redis
+    from cachier import cachier
+
+    redis_client = redis.Redis(host='localhost', port=6379, db=0)
+
+    @cachier(backend="redis", redis_client=redis_client, key_prefix="myapp")
+    def my_func(x):
+        return x * 2
+
+**Usage Example (Redis with callable client):**
+
+.. code-block:: python
+
+    import redis
+    from cachier import cachier
+
+    def get_redis_client():
+        return redis.Redis(host='localhost', port=6379, db=0)
+
+    @cachier(backend="redis", redis_client=get_redis_client)
+    def my_func(x):
+        return x * 2
+
 **Configuration Options:**
 
 - ``sql_engine``: SQLAlchemy connection string, Engine, or callable returning an Engine.

cachier-3.4.1.dist-info/RECORD

@@ -0,0 +1,21 @@
+cachier/__init__.py,sha256=1isxXaP2l6Vq7gC1Gob6hduRLC07dHfWze1-oCjwSP0,415
+cachier/__main__.py,sha256=upg-TlHs1vngKYvkjoPpl3Pvl6xOx4ut-M1mElMiAo0,443
+cachier/_types.py,sha256=u-1NWfZqcvwLa5qDmiTxIN4btM6NiWta212u8yPo02o,335
+cachier/_version.py,sha256=jnPPRn_qmjNi-qmQjlHnzNGf3LSBTYkMmJdGjxMTOBM,1089
+cachier/config.py,sha256=6hyQtn9T6UXu2UQhKJltWT0Nu4OBS4ION1x7Lt1i8Og,3838
+cachier/core.py,sha256=W1O_a9rkgsErG3spVPZpnfRu9k0MbV94rGkv79KDgBA,16243
+cachier/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cachier/version.info,sha256=zHFPXgIkp4MR9INen7uI5pujX2Xn3Vy32GyrtKhLvXI,6
+cachier/cores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cachier/cores/base.py,sha256=s7qgmDJA4LGub6ydGfMk9vVJW4fgeU0EXl-9gmpuh28,3683
+cachier/cores/memory.py,sha256=fsvqq9rwwmAaMBvYo-oUNAxB6UOyfBpuf8ACW_XTaU0,3572
+cachier/cores/mongo.py,sha256=wVit36jQeNkP1KyxMtm6jDglazpdORYzTOPJ0q10BWI,4979
+cachier/cores/pickle.py,sha256=L_IwKSnskJqlGsulkhMQukKdylq_ghpyG5BvdB_7WzM,12901
+cachier/cores/redis.py,sha256=rWrkEwWPzfFvH1eLkgBdhzxqPtJtx6N_sYcqRPy_P9Y,7553
+cachier/cores/sql.py,sha256=nuf2-Szo7VTPRa7IC3JGWEtGsBtdkIrx0bhOm3U0mfE,9895
+cachier-3.4.1.dist-info/licenses/LICENSE,sha256=-2WrMJkIa0gVP6YQHXXDT7ws-S3M2NEVEF4XF3K8qrY,1069
+cachier-3.4.1.dist-info/METADATA,sha256=__yRE190T79aUMOS9uaXSWj8q-YJp-3LpwLoLpOZxt8,24446
+cachier-3.4.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cachier-3.4.1.dist-info/entry_points.txt,sha256=x4Y7t6Y0Qev_3fgG-Jv7TrsvVdJty3FnGAdkT8-_5mY,49
+cachier-3.4.1.dist-info/top_level.txt,sha256=_rW_HiJumDCch67YT-WAgzcyvKg5RiYDMZq9d-0ZpaE,8
+cachier-3.4.1.dist-info/RECORD,,

cachier-3.3.0.dist-info/RECORD

@@ -1,20 +0,0 @@
-cachier/__init__.py,sha256=1isxXaP2l6Vq7gC1Gob6hduRLC07dHfWze1-oCjwSP0,415
-cachier/__main__.py,sha256=upg-TlHs1vngKYvkjoPpl3Pvl6xOx4ut-M1mElMiAo0,443
-cachier/_types.py,sha256=EGJMiw-oCIC_cDLyzw7YC40lfo8jnD3zMmoJpA9Y8Iw,238
-cachier/_version.py,sha256=jnPPRn_qmjNi-qmQjlHnzNGf3LSBTYkMmJdGjxMTOBM,1089
-cachier/config.py,sha256=6hyQtn9T6UXu2UQhKJltWT0Nu4OBS4ION1x7Lt1i8Og,3838
-cachier/core.py,sha256=PHLDA6Mabih-mi4y3CBsb8_vBIpvYQt547_Y57fo8uI,15825
-cachier/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cachier/version.info,sha256=Xu5sDFxpAfXlI_CkNLykEG4f9xNt60UfmCsLsaexNxY,6
-cachier/cores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-cachier/cores/base.py,sha256=s7qgmDJA4LGub6ydGfMk9vVJW4fgeU0EXl-9gmpuh28,3683
-cachier/cores/memory.py,sha256=fsvqq9rwwmAaMBvYo-oUNAxB6UOyfBpuf8ACW_XTaU0,3572
-cachier/cores/mongo.py,sha256=pCBrxLsmGr68Q50JVD_CUPAYwhaLDrJUQs_6A-_GYLA,4993
-cachier/cores/pickle.py,sha256=FgfvZWAFdWQPOo3G-L57iEV2ujEkIDH8TyGzbarsZeE,10678
-cachier/cores/sql.py,sha256=nuf2-Szo7VTPRa7IC3JGWEtGsBtdkIrx0bhOm3U0mfE,9895
-cachier-3.3.0.dist-info/licenses/LICENSE,sha256=-2WrMJkIa0gVP6YQHXXDT7ws-S3M2NEVEF4XF3K8qrY,1069
-cachier-3.3.0.dist-info/METADATA,sha256=P8GoF2LSZKA5Tp8NunrJvS_f1XlytO4HsCmDsUy4oxQ,23136
-cachier-3.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cachier-3.3.0.dist-info/entry_points.txt,sha256=x4Y7t6Y0Qev_3fgG-Jv7TrsvVdJty3FnGAdkT8-_5mY,49
-cachier-3.3.0.dist-info/top_level.txt,sha256=_rW_HiJumDCch67YT-WAgzcyvKg5RiYDMZq9d-0ZpaE,8
-cachier-3.3.0.dist-info/RECORD,,