rapyer 1.1.5__tar.gz → 1.1.7__tar.gz

{rapyer-1.1.5 → rapyer-1.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rapyer
-Version: 1.1.5
+Version: 1.1.7
 Summary: Pydantic models with Redis as the backend
 License: MIT
 Keywords: redis,redis-json,pydantic,pydantic-v2,orm,database,async,nosql,cache,key-value,data-modeling,python,backend,storage,serialization,validation

{rapyer-1.1.5 → rapyer-1.1.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "rapyer"
-version = "1.1.5"
+version = "1.1.7"
 description = "Pydantic models with Redis as the backend"
 authors = [{name = "YedidyaHKfir", email = "yedidyakfir@gmail.com"}]
 readme = "README.md"
@@ -69,12 +69,17 @@ Repository = "https://github.com/imaginary-cherry/rapyer"
 
 [tool.poetry]
 packages = [{include = "rapyer"}]
+include = ["rapyer/scripts/lua/**/*.lua"]
 
 
 [tool.poetry.group.dev.dependencies]
 black = "^25.9.0"
 mypy = "^1.0.0"
 
+[tool.bandit]
+exclude_dirs = ["tests"]
+skips = ["B101"]  # assert_used - valid in test files
+
 [tool.coverage.run]
 source = ["rapyer"]
 omit = ["*/tests/*", "*/test_*"]

{rapyer-1.1.5 → rapyer-1.1.7}/rapyer/__init__.py

@@ -3,6 +3,7 @@
 from rapyer.base import (
     AtomicRedisModel,
     aget,
+    afind,
     find_redis_models,
     ainsert,
     get,
@@ -15,6 +16,7 @@ __all__ = [
     "init_rapyer",
     "teardown_rapyer",
     "aget",
+    "afind",
     "get",
     "find_redis_models",
     "ainsert",

{rapyer-1.1.5 → rapyer-1.1.7}/rapyer/base.py

@@ -6,7 +6,7 @@ import logging
 import pickle
 import uuid
 from contextlib import AbstractAsyncContextManager
-from typing import ClassVar, Any, get_origin
+from typing import ClassVar, Any, get_origin, Optional
 
 from pydantic import (
     BaseModel,
@@ -15,11 +15,12 @@ from pydantic import (
     model_validator,
     field_serializer,
     field_validator,
+    ValidationError,
 )
 from pydantic_core.core_schema import FieldSerializationInfo, ValidationInfo
 from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import Query
-from redis.exceptions import NoScriptError
+from redis.exceptions import NoScriptError, ResponseError
 from typing_extensions import deprecated
 
 from rapyer.config import RedisConfig
@@ -29,6 +30,7 @@ from rapyer.errors.base import (
     PersistentNoScriptError,
     UnsupportedIndexedFieldError,
     CantSerializeRedisValueError,
+    RapyerModelDoesntExistError,
 )
 from rapyer.fields.expression import ExpressionField, AtomicField, Expression
 from rapyer.fields.index import IndexAnnotation
@@ -54,7 +56,6 @@ from rapyer.utils.pythonic import safe_issubclass
 from rapyer.utils.redis import (
     acquire_lock,
     update_keys_in_pipeline,
-    refresh_ttl_if_needed,
 )
 
 logger = logging.getLogger("rapyer")
@@ -169,6 +170,14 @@ class AtomicRedisModel(BaseModel):
         field_path = self.field_path
         return f"${field_path}" if field_path else "$"
 
+    @classmethod
+    def should_refresh(cls):
+        return cls.Meta.refresh_ttl and cls.Meta.ttl is not None
+
+    async def refresh_ttl_if_needed(self):
+        if self.should_refresh():
+            await self.Meta.redis.expire(self.key, self.Meta.ttl)
+
     @classmethod
     def redis_schema(cls, redis_name: str = ""):
         fields = []
@@ -395,9 +404,12 @@ class AtomicRedisModel(BaseModel):
         async with self.Meta.redis.pipeline() as pipe:
             update_keys_in_pipeline(pipe, self.key, **json_path_kwargs)
             await pipe.execute()
-        await refresh_ttl_if_needed(
-            self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
-        )
+        await self.refresh_ttl_if_needed()
+
+    async def aset_ttl(self, ttl: int) -> None:
+        if self.is_inner_model():
+            raise RuntimeError("Can only set TTL from top level model")
+        await self.Meta.redis.expire(self.key, ttl)
 
     @classmethod
     @deprecated(
@@ -419,9 +431,8 @@ class AtomicRedisModel(BaseModel):
         instance = cls.model_validate(model_dump, context=context)
         instance.key = key
         instance._failed_fields = context.get(FAILED_FIELDS_KEY, set())
-        await refresh_ttl_if_needed(
-            cls.Meta.redis, key, cls.Meta.ttl, cls.Meta.refresh_ttl
-        )
+        if cls.should_refresh():
+            await cls.Meta.redis.expire(key, cls.Meta.ttl)
         return instance
 
     @deprecated(
@@ -440,55 +451,83 @@ class AtomicRedisModel(BaseModel):
         instance._pk = self._pk
         instance._base_model_link = self._base_model_link
         instance._failed_fields = context.get(FAILED_FIELDS_KEY, set())
-        await refresh_ttl_if_needed(
-            self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
-        )
+        await self.refresh_ttl_if_needed()
         return instance
 
     @classmethod
-    async def afind(cls, *expressions):
-        # Original behavior when no expressions provided - return all
-        if not expressions:
-            keys = await cls.afind_keys()
-            if not keys:
-                return []
+    def create_redis_model(cls, model_dump: dict, key: str) -> Optional[Self]:
+        context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
+        try:
+            model = cls.model_validate(model_dump, context=context)
+            model.key = key
+        except ValidationError as exc:
+            logger.debug(
+                "Skipping key %s due to validation error during afind: %s",
+                key,
+                exc,
+            )
+            return None
+        model.key = key
+        model._failed_fields = context.get(FAILED_FIELDS_KEY, set())
+        return model
 
-            models = await cls.Meta.redis.json().mget(keys=keys, path="$")
-        else:
-            # With expressions - use Redis Search
-            # Combine all expressions with & operator
+    @classmethod
+    async def afind(cls, *args):
+        # Separate keys (str) from expressions (Expression)
+        provided_keys = [arg for arg in args if isinstance(arg, str)]
+        expressions = [arg for arg in args if isinstance(arg, Expression)]
+        raise_on_missing = bool(provided_keys)
+
+        if provided_keys and expressions:
+            logger.warning(
+                "afind called with both keys and expressions; expressions ignored"
+            )
+
+        if provided_keys:
+            # Case 1: Extract by keys
+            targeted_keys = [
+                k if ":" in k else f"{cls.class_key_initials()}:{k}"
+                for k in provided_keys
+            ]
+        elif expressions:
+            # Case 2: Extract by expressions
             combined_expression = functools.reduce(lambda a, b: a & b, expressions)
             query_string = combined_expression.create_filter()
-
-            # Create a Query object
             query = Query(query_string).no_content()
-
-            # Try to search using the index
             index_name = cls.index_name()
             search_result = await cls.Meta.redis.ft(index_name).search(query)
-
             if not search_result.docs:
                 return []
+            targeted_keys = [doc.id for doc in search_result.docs]
+        else:
+            # Case 3: Extract all
+            targeted_keys = await cls.afind_keys()
+
+        if not targeted_keys:
+            return []
 
-            # Get the keys from search results
-            keys = [doc.id for doc in search_result.docs]
+        # Fetch the actual documents
+        models = await cls.Meta.redis.json().mget(keys=targeted_keys, path="$")
 
-            # Fetch the actual documents
-            models = await cls.Meta.redis.json().mget(keys=keys, path="$")
+        instances = []
+        for model, key in zip(models, targeted_keys):
+            if model is None:
+                if raise_on_missing:
+                    raise KeyNotFound(f"{key} is missing in redis")
+                continue
+            if not cls.Meta.is_fake_redis:
+                model = model[0]
+            model = cls.create_redis_model(model, key)
+            if model is None:
+                continue
+            instances.append(model)
 
-        if cls.Meta.ttl is not None and cls.Meta.refresh_ttl:
+        if cls.should_refresh():
             async with cls.Meta.redis.pipeline() as pipe:
-                for key in keys:
-                    pipe.expire(key, cls.Meta.ttl)
+                for model in instances:
+                    pipe.expire(model.key, cls.Meta.ttl)
                 await pipe.execute()
 
-        instances = []
-        for model, key in zip(models, keys):
-            context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
-            model = cls.model_validate(model[0], context=context)
-            model.key = key
-            model._failed_fields = context.get(FAILED_FIELDS_KEY, set())
-            instances.append(model)
         return instances
 
     @classmethod
@@ -616,9 +655,21 @@ class AtomicRedisModel(BaseModel):
             noscript_on_retry = False
 
             try:
+                if self.should_refresh():
+                    pipe.expire(self.key, self.Meta.ttl)
                 await pipe.execute()
             except NoScriptError:
                 noscript_on_first_attempt = True
+            except ResponseError as exc:
+                if ignore_if_deleted:
+                    logger.warning(
+                        "Swallowed ResponseError during pipeline.execute() with "
+                        "ignore_if_deleted=True for key %r: %s",
+                        getattr(self, "key", None),
+                        exc,
+                    )
+                else:
+                    raise
 
             if noscript_on_first_attempt:
                 await handle_noscript_error(self.Meta.redis)
@@ -642,9 +693,6 @@ class AtomicRedisModel(BaseModel):
                     "This indicates a server-side problem with Redis."
                 )
 
-            await refresh_ttl_if_needed(
-                self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
-            )
             _context_var.set(None)
             _context_xx_pipe.set(False)
 
@@ -659,6 +707,16 @@ class AtomicRedisModel(BaseModel):
             if isinstance(attr, RedisType):
                 attr._base_model_link = self
 
+        pipeline = _context_var.get()
+        if pipeline is not None:
+            serialized = self.model_dump(
+                mode="json",
+                context={REDIS_DUMP_FLAG_NAME: True},
+                include={name},
+            )
+            json_path = f"{self.json_path}.{name}"
+            pipeline.json().set(self.key, json_path, serialized[name])
+
     def __eq__(self, other):
         if not isinstance(other, BaseModel):
             return False
@@ -702,6 +760,52 @@ async def aget(redis_key: str) -> AtomicRedisModel:
     return await klass.aget(redis_key)
 
 
+async def afind(*redis_keys: str, skip_missing: bool = False) -> list[AtomicRedisModel]:
+    if not redis_keys:
+        return []
+
+    redis_model_mapping = {klass.__name__: klass for klass in REDIS_MODELS}
+
+    key_to_class: dict[str, type[AtomicRedisModel]] = {}
+    for key in redis_keys:
+        class_name = key.split(":")[0]
+        if class_name not in redis_model_mapping:
+            raise RapyerModelDoesntExistError(
+                class_name, f"Unknown model class: {class_name}"
+            )
+        key_to_class[key] = redis_model_mapping[class_name]
+
+    models_data = await AtomicRedisModel.Meta.redis.json().mget(
+        keys=redis_keys, path="$"
+    )
+
+    instances = []
+    instances_by_class: dict[type[AtomicRedisModel], list[AtomicRedisModel]] = {}
+
+    for data, key in zip(models_data, redis_keys):
+        if data is None:
+            if not skip_missing:
+                raise KeyNotFound(f"{key} is missing in redis")
+            continue
+        klass = key_to_class[key]
+        if not klass.Meta.is_fake_redis:
+            data = data[0]
+        model = klass.create_redis_model(data, key)
+        if model is None:
+            continue
+        instances.append(model)
+        instances_by_class.setdefault(klass, []).append(model)
+
+    async with AtomicRedisModel.Meta.redis.pipeline() as pipe:
+        for klass, class_instances in instances_by_class.items():
+            if klass.should_refresh():
+                for model in class_instances:
+                    pipe.expire(model.key, klass.Meta.ttl)
+        await pipe.execute()
+
+    return instances
+
+
 def find_redis_models() -> list[type[AtomicRedisModel]]:
     return REDIS_MODELS
 
@@ -710,6 +814,8 @@ async def ainsert(*models: Unpack[AtomicRedisModel]) -> list[AtomicRedisModel]:
     async with AtomicRedisModel.Meta.redis.pipeline() as pipe:
         for model in models:
             pipe.json().set(model.key, model.json_path, model.redis_dump())
+            if model.Meta.ttl is not None:
+                pipe.expire(model.key, model.Meta.ttl)
         await pipe.execute()
     return models
 

{rapyer-1.1.5 → rapyer-1.1.7}/rapyer/config.py

@@ -28,3 +28,5 @@ class RedisConfig:
     safe_load_all: bool = False
     # If True, use JSON serialization for fields that support it instead of pickle
     prefer_normal_json_dump: bool = False
+    # Set to True when using FakeRedis to normalize JSON responses
+    is_fake_redis: bool = False

{rapyer-1.1.5 → rapyer-1.1.7}/rapyer/errors/__init__.py

@@ -5,6 +5,9 @@ from rapyer.errors.base import (
     RapyerError,
     ScriptsNotInitializedError,
     UnsupportedIndexedFieldError,
+    RapyerModelDoesntExistError,
+    CantSerializeRedisValueError,
+    KeyNotFound,
 )
 
 __all__ = [
@@ -12,6 +15,9 @@ __all__ = [
     "FindError",
     "PersistentNoScriptError",
     "RapyerError",
+    "KeyNotFound",
     "ScriptsNotInitializedError",
     "UnsupportedIndexedFieldError",
+    "RapyerModelDoesntExistError",
+    "CantSerializeRedisValueError",
 ]

{rapyer-1.1.5 → rapyer-1.1.7}/rapyer/errors/base.py

@@ -10,6 +10,14 @@ class KeyNotFound(RapyerError):
     pass
 
 
+class RapyerModelDoesntExistError(RapyerError):
+    """Raised when a model doesn't exist."""
+
+    def __init__(self, model_name: str, *args):
+        super().__init__(*args)
+        self.model_name = model_name
+
+
 class FindError(RapyerError):
     """Raised when a model cannot be found."""
 

{rapyer-1.1.5 → rapyer-1.1.7}/rapyer/init.py

@@ -1,3 +1,5 @@
+import logging
+
 import redis.asyncio as redis_async
 from redis import ResponseError
 from redis.asyncio.client import Redis
@@ -6,21 +8,35 @@ from rapyer.base import REDIS_MODELS
 from rapyer.scripts import register_scripts
 
 
+def is_fakeredis(client) -> bool:
+    return "fakeredis" in type(client).__module__
+
+
 async def init_rapyer(
     redis: str | Redis = None,
     ttl: int = None,
     override_old_idx: bool = True,
     prefer_normal_json_dump: bool = None,
+    logger: logging.Logger = None,
 ):
+    if logger is not None:
+        rapyer_logger = logging.getLogger("rapyer")
+        rapyer_logger.setLevel(logger.level)
+        rapyer_logger.handlers.clear()
+        for handler in logger.handlers:
+            rapyer_logger.addHandler(handler)
+
     if isinstance(redis, str):
         redis = redis_async.from_url(redis, decode_responses=True, max_connections=20)
 
+    is_fake_redis = is_fakeredis(redis)
     if redis is not None:
-        await register_scripts(redis)
+        await register_scripts(redis, is_fake_redis)
 
     for model in REDIS_MODELS:
         if redis is not None:
             model.Meta.redis = redis
+            model.Meta.is_fake_redis = is_fake_redis
         if ttl is not None:
             model.Meta.ttl = ttl
         if prefer_normal_json_dump is not None:

rapyer-1.1.7/rapyer/scripts/__init__.py

@@ -0,0 +1,41 @@
+from rapyer.scripts.constants import (
+    DATETIME_ADD_SCRIPT_NAME,
+    NUM_FLOORDIV_SCRIPT_NAME,
+    NUM_MOD_SCRIPT_NAME,
+    NUM_MUL_SCRIPT_NAME,
+    NUM_POW_FLOAT_SCRIPT_NAME,
+    NUM_POW_SCRIPT_NAME,
+    NUM_TRUEDIV_SCRIPT_NAME,
+    REMOVE_RANGE_SCRIPT_NAME,
+    STR_APPEND_SCRIPT_NAME,
+    STR_MUL_SCRIPT_NAME,
+)
+from rapyer.scripts.registry import (
+    _REGISTERED_SCRIPT_SHAS,
+    get_scripts,
+    get_scripts_fakeredis,
+    handle_noscript_error,
+    register_scripts,
+    run_sha,
+)
+
+SCRIPTS = get_scripts()
+SCRIPTS_FAKEREDIS = get_scripts_fakeredis()
+
+__all__ = [
+    "DATETIME_ADD_SCRIPT_NAME",
+    "NUM_FLOORDIV_SCRIPT_NAME",
+    "NUM_MOD_SCRIPT_NAME",
+    "NUM_MUL_SCRIPT_NAME",
+    "NUM_POW_FLOAT_SCRIPT_NAME",
+    "NUM_POW_SCRIPT_NAME",
+    "NUM_TRUEDIV_SCRIPT_NAME",
+    "REMOVE_RANGE_SCRIPT_NAME",
+    "SCRIPTS",
+    "SCRIPTS_FAKEREDIS",
+    "STR_APPEND_SCRIPT_NAME",
+    "STR_MUL_SCRIPT_NAME",
+    "handle_noscript_error",
+    "register_scripts",
+    "run_sha",
+]

rapyer-1.1.7/rapyer/scripts/constants.py

@@ -0,0 +1,10 @@
+REMOVE_RANGE_SCRIPT_NAME = "remove_range"
+NUM_MUL_SCRIPT_NAME = "num_mul"
+NUM_FLOORDIV_SCRIPT_NAME = "num_floordiv"
+NUM_MOD_SCRIPT_NAME = "num_mod"
+NUM_POW_SCRIPT_NAME = "num_pow"
+NUM_POW_FLOAT_SCRIPT_NAME = "num_pow_float"
+NUM_TRUEDIV_SCRIPT_NAME = "num_truediv"
+STR_APPEND_SCRIPT_NAME = "str_append"
+STR_MUL_SCRIPT_NAME = "str_mul"
+DATETIME_ADD_SCRIPT_NAME = "datetime_add"

rapyer-1.1.7/rapyer/scripts/loader.py

@@ -0,0 +1,34 @@
+from functools import lru_cache
+from importlib import resources
+
+
+VARIANTS = {
+    "redis": {
+        "EXTRACT_ARRAY": "local arr = cjson.decode(arr_json)[1]",
+        "EXTRACT_VALUE": "local value = tonumber(cjson.decode(current_json)[1])",
+        "EXTRACT_STR": "local value = cjson.decode(current_json)[1]",
+        "EXTRACT_DATETIME": "local value = cjson.decode(current_json)[1]",
+    },
+    "fakeredis": {
+        "EXTRACT_ARRAY": "local arr = cjson.decode(arr_json)",
+        "EXTRACT_VALUE": "local value = tonumber(cjson.decode(current_json)[1])",
+        "EXTRACT_STR": "local value = cjson.decode(current_json)[1]",
+        "EXTRACT_DATETIME": "local value = cjson.decode(current_json)[1]",
+    },
+}
+
+
+@lru_cache(maxsize=None)
+def _load_template(category: str, name: str) -> str:
+    package = f"rapyer.scripts.lua.{category}"
+    filename = f"{name}.lua"
+    return resources.files(package).joinpath(filename).read_text()
+
+
+def load_script(category: str, name: str, variant: str = "redis") -> str:
+    template = _load_template(category, name)
+    replacements = VARIANTS[variant]
+    result = template
+    for placeholder, value in replacements.items():
+        result = result.replace(f"--[[{placeholder}]]", value)
+    return result

rapyer-1.1.7/rapyer/scripts/lua/datetime/add.lua

@@ -0,0 +1,80 @@
+local key = KEYS[1]
+local path = ARGV[1]
+local delta_seconds = tonumber(ARGV[2])
+
+local current_json = redis.call('JSON.GET', key, path)
+if not current_json or current_json == 'null' then
+    return nil
+end
+
+--[[EXTRACT_DATETIME]]
+local pattern = "(%d+)-(%d+)-(%d+)T(%d+):(%d+):(%d+)"
+local y, m, d, h, mi, s = string.match(value, pattern)
+if not y then return nil end
+
+y = tonumber(y)
+m = tonumber(m)
+d = tonumber(d)
+h = tonumber(h)
+mi = tonumber(mi)
+s = tonumber(s)
+
+s = s + delta_seconds
+
+while s >= 60 do
+    s = s - 60
+    mi = mi + 1
+end
+while s < 0 do
+    s = s + 60
+    mi = mi - 1
+end
+while mi >= 60 do
+    mi = mi - 60
+    h = h + 1
+end
+while mi < 0 do
+    mi = mi + 60
+    h = h - 1
+end
+while h >= 24 do
+    h = h - 24
+    d = d + 1
+end
+while h < 0 do
+    h = h + 24
+    d = d - 1
+end
+
+local function is_leap_year(year)
+    return (year % 4 == 0 and year % 100 ~= 0) or (year % 400 == 0)
+end
+
+local function days_in_month(year, month)
+    local days = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
+    if month == 2 and is_leap_year(year) then
+        return 29
+    end
+    return days[month]
+end
+
+while d > days_in_month(y, m) do
+    d = d - days_in_month(y, m)
+    m = m + 1
+    if m > 12 then
+        m = 1
+        y = y + 1
+    end
+end
+while d < 1 do
+    m = m - 1
+    if m < 1 then
+        m = 12
+        y = y - 1
+    end
+    d = d + days_in_month(y, m)
+end
+
+local new_date = string.format("%04d-%02d-%02dT%02d:%02d:%02d", y, m, d, h, mi, s)
+redis.call('JSON.SET', key, path, cjson.encode(new_date))
+return new_date

rapyer-1.1.7/rapyer/scripts/lua/list/remove_range.lua

@@ -0,0 +1,36 @@
+local key = KEYS[1]
+local path = ARGV[1]
+local start_idx = tonumber(ARGV[2])
+local end_idx = tonumber(ARGV[3])
+
+local arr_json = redis.call('JSON.GET', key, path)
+if not arr_json or arr_json == 'null' then
+    return nil
+end
+
+--[[EXTRACT_ARRAY]]
+local n = #arr
+
+if start_idx < 0 then start_idx = n + start_idx end
+if end_idx < 0 then end_idx = n + end_idx end
+if start_idx < 0 then start_idx = 0 end
+if end_idx < 0 then end_idx = 0 end
+if end_idx > n then end_idx = n end
+if start_idx >= n or start_idx >= end_idx then return true end
+
+local new_arr = {}
+local j = 1
+
+for i = 1, start_idx do
+    new_arr[j] = arr[i]
+    j = j + 1
+end
+
+for i = end_idx + 1, n do
+    new_arr[j] = arr[i]
+    j = j + 1
+end
+
+local encoded = j == 1 and '[]' or cjson.encode(new_arr)
+redis.call('JSON.SET', key, path, encoded)
+return true

rapyer-1.1.7/rapyer/scripts/lua/numeric/floordiv.lua

@@ -0,0 +1,13 @@
+local key = KEYS[1]
+local path = ARGV[1]
+local operand = tonumber(ARGV[2])
+
+local current_json = redis.call('JSON.GET', key, path)
+if not current_json or current_json == 'null' then
+    return nil
+end
+
+--[[EXTRACT_VALUE]]
+local result = math.floor(value / operand)
+redis.call('JSON.SET', key, path, result)
+return result

rapyer-1.1.7/rapyer/scripts/lua/numeric/mod.lua

@@ -0,0 +1,13 @@
+local key = KEYS[1]
+local path = ARGV[1]
+local operand = tonumber(ARGV[2])
+
+local current_json = redis.call('JSON.GET', key, path)
+if not current_json or current_json == 'null' then
+    return nil
+end
+
+--[[EXTRACT_VALUE]]
+local result = value % operand
+redis.call('JSON.SET', key, path, result)
+return result