rapyer 1.1.4__tar.gz → 1.1.6__tar.gz

{rapyer-1.1.4 → rapyer-1.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rapyer
-Version: 1.1.4
+Version: 1.1.6
 Summary: Pydantic models with Redis as the backend
 License: MIT
 Keywords: redis,redis-json,pydantic,pydantic-v2,orm,database,async,nosql,cache,key-value,data-modeling,python,backend,storage,serialization,validation
@@ -23,7 +23,12 @@ Classifier: Topic :: Database :: Database Engines/Servers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
 Classifier: Operating System :: OS Independent
+Provides-Extra: test
+Requires-Dist: fakeredis[json,lua] (>=2.20.0) ; extra == "test"
 Requires-Dist: pydantic (>=2.11.0,<2.13.0)
+Requires-Dist: pytest (>=8.4.2) ; extra == "test"
+Requires-Dist: pytest-asyncio (>=0.25.0) ; extra == "test"
+Requires-Dist: pytest-cov (>=6.0.0) ; extra == "test"
 Requires-Dist: redis[async] (>=6.0.0,<7.1.0)
 Project-URL: Bug Tracker, https://github.com/imaginary-cherry/rapyer/issues
 Project-URL: Changelog, https://github.com/imaginary-cherry/rapyer/releases

{rapyer-1.1.4 → rapyer-1.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "rapyer"
-version = "1.1.4"
+version = "1.1.6"
 description = "Pydantic models with Redis as the backend"
 authors = [{name = "YedidyaHKfir", email = "yedidyakfir@gmail.com"}]
 readme = "README.md"
@@ -51,6 +51,14 @@ dependencies = [
     "pydantic>=2.11.0, <2.13.0",
 ]
 
+[project.optional-dependencies]
+test = [
+    "pytest>=8.4.2",
+    "pytest-asyncio>=0.25.0",
+    "pytest-cov>=6.0.0",
+    "fakeredis[lua,json]>=2.20.0",
+]
+
 [project.urls]
 Homepage = "https://imaginary-cherry.github.io/rapyer/"
 Documentation = "https://imaginary-cherry.github.io/rapyer/"
@@ -67,10 +75,9 @@ packages = [{include = "rapyer"}]
 black = "^25.9.0"
 mypy = "^1.0.0"
 
-[tool.poetry.group.tests.dependencies]
-pytest = "^8.4.2"
-pytest-asyncio = "^0.25.0"
-pytest-cov = "^6.0.0"
+[tool.bandit]
+exclude_dirs = ["tests"]
+skips = ["B101"]  # assert_used - valid in test files
 
 [tool.coverage.run]
 source = ["rapyer"]

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/base.py

@@ -2,6 +2,7 @@ import asyncio
 import base64
 import contextlib
 import functools
+import logging
 import pickle
 import uuid
 from contextlib import AbstractAsyncContextManager
@@ -14,20 +15,24 @@ from pydantic import (
     model_validator,
     field_serializer,
     field_validator,
+    ValidationError,
 )
 from pydantic_core.core_schema import FieldSerializationInfo, ValidationInfo
-from redis.commands.search.index_definition import IndexDefinition, IndexType
-from redis.commands.search.query import Query
-from typing_extensions import deprecated
-
 from rapyer.config import RedisConfig
 from rapyer.context import _context_var, _context_xx_pipe
-from rapyer.errors.base import KeyNotFound, UnsupportedIndexedFieldError
+from rapyer.errors.base import (
+    KeyNotFound,
+    PersistentNoScriptError,
+    UnsupportedIndexedFieldError,
+    CantSerializeRedisValueError,
+)
 from rapyer.fields.expression import ExpressionField, AtomicField, Expression
 from rapyer.fields.index import IndexAnnotation
 from rapyer.fields.key import KeyAnnotation
+from rapyer.fields.safe_load import SafeLoadAnnotation
 from rapyer.links import REDIS_SUPPORTED_LINK
-from rapyer.types.base import RedisType, REDIS_DUMP_FLAG_NAME
+from rapyer.scripts import handle_noscript_error
+from rapyer.types.base import RedisType, REDIS_DUMP_FLAG_NAME, FAILED_FIELDS_KEY
 from rapyer.types.convert import RedisConverter
 from rapyer.typing_support import Self, Unpack
 from rapyer.utils.annotation import (
@@ -36,34 +41,61 @@ from rapyer.utils.annotation import (
     field_with_flag,
     DYNAMIC_CLASS_DOC,
 )
-from rapyer.utils.fields import get_all_pydantic_annotation, is_redis_field
+from rapyer.utils.fields import (
+    get_all_pydantic_annotation,
+    is_redis_field,
+    is_type_json_serializable,
+)
 from rapyer.utils.pythonic import safe_issubclass
 from rapyer.utils.redis import (
     acquire_lock,
     update_keys_in_pipeline,
     refresh_ttl_if_needed,
 )
+from redis.commands.search.index_definition import IndexDefinition, IndexType
+from redis.commands.search.query import Query
+from redis.exceptions import NoScriptError, ResponseError
+from typing_extensions import deprecated
+
+logger = logging.getLogger("rapyer")
 
 
-def make_pickle_field_serializer(field: str):
+def make_pickle_field_serializer(
+    field: str, safe_load: bool = False, can_json: bool = False
+):
     @field_serializer(field, when_used="json-unless-none")
-    def pickle_field_serializer(v, info: FieldSerializationInfo):
+    @classmethod
+    def pickle_field_serializer(cls, v, info: FieldSerializationInfo):
         ctx = info.context or {}
         should_serialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME, False)
-        if should_serialize_redis:
+        # Skip pickling if field CAN be JSON serialized AND user prefers JSON dump
+        field_can_be_json = can_json and cls.Meta.prefer_normal_json_dump
+        if should_serialize_redis and not field_can_be_json:
             return base64.b64encode(pickle.dumps(v)).decode("utf-8")
         return v
 
     pickle_field_serializer.__name__ = f"__serialize_{field}"
 
     @field_validator(field, mode="before")
-    def pickle_field_validator(v, info: ValidationInfo):
+    @classmethod
+    def pickle_field_validator(cls, v, info: ValidationInfo):
         if v is None:
             return v
         ctx = info.context or {}
         should_serialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME, False)
         if should_serialize_redis:
-            return pickle.loads(base64.b64decode(v))
+            try:
+                field_can_be_json = can_json and cls.Meta.prefer_normal_json_dump
+                if should_serialize_redis and not field_can_be_json:
+                    return pickle.loads(base64.b64decode(v))
+                return v
+            except Exception as e:
+                if safe_load:
+                    failed_fields = ctx.setdefault(FAILED_FIELDS_KEY, set())
+                    failed_fields.add(field)
+                    logger.warning("SafeLoad: Failed to deserialize field '%s'", field)
+                    return None
+                raise CantSerializeRedisValueError() from e
         return v
 
     pickle_field_validator.__name__ = f"__deserialize_{field}"
@@ -71,15 +103,40 @@ def make_pickle_field_serializer(field: str):
     return pickle_field_serializer, pickle_field_validator
 
 
+# TODO: Remove in next major version (2.0) - backward compatibility for pickled data
+# This validator handles loading old pickled data for fields that are now JSON-serializable.
+# In 2.0, remove this function and the validator registration in __init_subclass__.
+def make_backward_compat_validator(field: str):
+    @field_validator(field, mode="before")
+    def backward_compat_validator(v, info: ValidationInfo):
+        ctx = info.context or {}
+        should_deserialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME, False)
+        if should_deserialize_redis and isinstance(v, str):
+            try:
+                return pickle.loads(base64.b64decode(v))
+            except Exception:
+                pass
+        return v
+
+    backward_compat_validator.__name__ = f"__backward_compat_{field}"
+    return backward_compat_validator
+
+
 class AtomicRedisModel(BaseModel):
     _pk: str = PrivateAttr(default_factory=lambda: str(uuid.uuid4()))
     _base_model_link: Self | RedisType = PrivateAttr(default=None)
+    _failed_fields: set[str] = PrivateAttr(default_factory=set)
 
     Meta: ClassVar[RedisConfig] = RedisConfig()
     _key_field_name: ClassVar[str | None] = None
+    _safe_load_fields: ClassVar[set[str]] = set()
     _field_name: str = PrivateAttr(default="")
     model_config = ConfigDict(validate_assignment=True, validate_default=True)
 
+    @property
+    def failed_fields(self) -> set[str]:
+        return self._failed_fields
+
     @property
     def pk(self):
         if self._key_field_name:
@@ -112,6 +169,9 @@ class AtomicRedisModel(BaseModel):
         field_path = self.field_path
         return f"${field_path}" if field_path else "$"
 
+    def should_refresh(self):
+        return self.Meta.refresh_ttl and self.Meta.ttl is not None
+
     @classmethod
     def redis_schema(cls, redis_name: str = ""):
         fields = []
@@ -184,11 +244,13 @@ class AtomicRedisModel(BaseModel):
         self._pk = value.split(":", maxsplit=1)[-1]
 
     def __init_subclass__(cls, **kwargs):
-        # Find a field with KeyAnnotation and save its name
+        # Find fields with KeyAnnotation and SafeLoadAnnotation
+        cls._safe_load_fields = set()
         for field_name, annotation in cls.__annotations__.items():
             if has_annotation(annotation, KeyAnnotation):
                 cls._key_field_name = field_name
-                break
+            if has_annotation(annotation, SafeLoadAnnotation):
+                cls._safe_load_fields.add(field_name)
 
         # Redefine annotations to use redis types
         pydantic_annotation = get_all_pydantic_annotation(cls, AtomicRedisModel)
@@ -200,7 +262,13 @@ class AtomicRedisModel(BaseModel):
         original_annotations.update(new_annotation)
         new_annotations = {
             field_name: replace_to_redis_types_in_annotation(
-                annotation, RedisConverter(cls.Meta.redis_type, f".{field_name}")
+                annotation,
+                RedisConverter(
+                    cls.Meta.redis_type,
+                    f".{field_name}",
+                    safe_load=field_name in cls._safe_load_fields
+                    or cls.Meta.safe_load_all,
+                ),
             )
             for field_name, annotation in original_annotations.items()
             if is_redis_field(field_name, annotation)
@@ -216,9 +284,22 @@ class AtomicRedisModel(BaseModel):
             if not is_redis_field(attr_name, attr_type):
                 continue
             if original_annotations[attr_name] == attr_type:
-                serializer, validator = make_pickle_field_serializer(attr_name)
-                setattr(cls, serializer.__name__, serializer)
-                setattr(cls, validator.__name__, validator)
+                default_value = cls.__dict__.get(attr_name, None)
+                can_json = is_type_json_serializable(attr_type, default_value)
+                should_json_serialize = can_json and cls.Meta.prefer_normal_json_dump
+
+                if not should_json_serialize:
+                    is_field_marked_safe = attr_name in cls._safe_load_fields
+                    is_safe_load = is_field_marked_safe or cls.Meta.safe_load_all
+                    serializer, validator = make_pickle_field_serializer(
+                        attr_name, safe_load=is_safe_load, can_json=can_json
+                    )
+                    setattr(cls, serializer.__name__, serializer)
+                    setattr(cls, validator.__name__, validator)
+                else:
+                    # TODO: Remove in 2.0 - backward compatibility for old pickled data
+                    validator = make_backward_compat_validator(attr_name)
+                    setattr(cls, validator.__name__, validator)
                 continue
 
         # Update the redis model list for initialization
@@ -321,6 +402,11 @@ class AtomicRedisModel(BaseModel):
             self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
         )
 
+    async def aset_ttl(self, ttl: int) -> None:
+        if self.is_inner_model():
+            raise RuntimeError("Can only set TTL from top level model")
+        await self.Meta.redis.expire(self.key, ttl)
+
     @classmethod
     @deprecated(
         "get() classmethod is deprecated and will be removed in rapyer 1.2.0, use aget instead"
@@ -337,8 +423,10 @@ class AtomicRedisModel(BaseModel):
             raise KeyNotFound(f"{key} is missing in redis")
         model_dump = model_dump[0]
 
-        instance = cls.model_validate(model_dump, context={REDIS_DUMP_FLAG_NAME: True})
+        context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
+        instance = cls.model_validate(model_dump, context=context)
         instance.key = key
+        instance._failed_fields = context.get(FAILED_FIELDS_KEY, set())
         await refresh_ttl_if_needed(
             cls.Meta.redis, key, cls.Meta.ttl, cls.Meta.refresh_ttl
         )
@@ -355,56 +443,77 @@ class AtomicRedisModel(BaseModel):
         if not model_dump:
             raise KeyNotFound(f"{self.key} is missing in redis")
         model_dump = model_dump[0]
-        instance = self.__class__(**model_dump)
+        context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
+        instance = self.__class__.model_validate(model_dump, context=context)
         instance._pk = self._pk
         instance._base_model_link = self._base_model_link
+        instance._failed_fields = context.get(FAILED_FIELDS_KEY, set())
         await refresh_ttl_if_needed(
             self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
         )
         return instance
 
     @classmethod
-    async def afind(cls, *expressions):
-        # Original behavior when no expressions provided - return all
-        if not expressions:
-            keys = await cls.afind_keys()
-            if not keys:
-                return []
+    async def afind(cls, *args):
+        # Separate keys (str) from expressions (Expression)
+        provided_keys = [arg for arg in args if isinstance(arg, str)]
+        expressions = [arg for arg in args if isinstance(arg, Expression)]
+
+        if provided_keys and expressions:
+            logger.warning(
+                "afind called with both keys and expressions; expressions ignored"
+            )
 
-            models = await cls.Meta.redis.json().mget(keys=keys, path="$")
-        else:
-            # With expressions - use Redis Search
-            # Combine all expressions with & operator
+        if provided_keys:
+            # Case 1: Extract by keys
+            targeted_keys = [
+                k if ":" in k else f"{cls.class_key_initials()}:{k}"
+                for k in provided_keys
+            ]
+        elif expressions:
+            # Case 2: Extract by expressions
             combined_expression = functools.reduce(lambda a, b: a & b, expressions)
             query_string = combined_expression.create_filter()
-
-            # Create a Query object
             query = Query(query_string).no_content()
-
-            # Try to search using the index
             index_name = cls.index_name()
             search_result = await cls.Meta.redis.ft(index_name).search(query)
-
             if not search_result.docs:
                 return []
+            targeted_keys = [doc.id for doc in search_result.docs]
+        else:
+            # Case 3: Extract all
+            targeted_keys = await cls.afind_keys()
 
-            # Get the keys from search results
-            keys = [doc.id for doc in search_result.docs]
+        if not targeted_keys:
+            return []
 
-            # Fetch the actual documents
-            models = await cls.Meta.redis.json().mget(keys=keys, path="$")
+        # Fetch the actual documents
+        models = await cls.Meta.redis.json().mget(keys=targeted_keys, path="$")
+
+        instances = []
+        for model, key in zip(models, targeted_keys):
+            if model is None:
+                continue
+            context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
+            try:
+                model = cls.model_validate(model[0], context=context)
+            except ValidationError as exc:
+                logger.debug(
+                    "Skipping key %s due to validation error during afind: %s",
+                    key,
+                    exc,
+                )
+                continue
+            model.key = key
+            model._failed_fields = context.get(FAILED_FIELDS_KEY, set())
+            instances.append(model)
 
         if cls.Meta.ttl is not None and cls.Meta.refresh_ttl:
             async with cls.Meta.redis.pipeline() as pipe:
-                for key in keys:
-                    pipe.expire(key, cls.Meta.ttl)
+                for model in instances:
+                    pipe.expire(model.key, cls.Meta.ttl)
                 await pipe.execute()
 
-        instances = []
-        for model, key in zip(models, keys):
-            model = cls.model_validate(model[0], context={REDIS_DUMP_FLAG_NAME: True})
-            model.key = key
-            instances.append(model)
         return instances
 
     @classmethod
@@ -512,7 +621,7 @@ class AtomicRedisModel(BaseModel):
     async def apipeline(
         self, ignore_if_deleted: bool = False
     ) -> AbstractAsyncContextManager[Self]:
-        async with self.Meta.redis.pipeline() as pipe:
+        async with self.Meta.redis.pipeline(transaction=True) as pipe:
             try:
                 redis_model = await self.__class__.aget(self.key)
                 unset_fields = {
@@ -527,10 +636,49 @@ class AtomicRedisModel(BaseModel):
             _context_var.set(pipe)
             _context_xx_pipe.set(ignore_if_deleted)
             yield redis_model
-            await pipe.execute()
-            await refresh_ttl_if_needed(
-                self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
-            )
+            commands_backup = list(pipe.command_stack)
+            noscript_on_first_attempt = False
+            noscript_on_retry = False
+
+            try:
+                if self.should_refresh():
+                    pipe.expire(self.key, self.Meta.ttl)
+                await pipe.execute()
+            except NoScriptError:
+                noscript_on_first_attempt = True
+            except ResponseError as exc:
+                if ignore_if_deleted:
+                    logger.warning(
+                        "Swallowed ResponseError during pipeline.execute() with "
+                        "ignore_if_deleted=True for key %r: %s",
+                        getattr(self, "key", None),
+                        exc,
+                    )
+                else:
+                    raise
+
+            if noscript_on_first_attempt:
+                await handle_noscript_error(self.Meta.redis)
+                evalsha_commands = [
+                    (args, options)
+                    for args, options in commands_backup
+                    if args[0] == "EVALSHA"
+                ]
+                # Retry execute the pipeline actions
+                async with self.Meta.redis.pipeline(transaction=True) as retry_pipe:
+                    for args, options in evalsha_commands:
+                        retry_pipe.execute_command(*args, **options)
+                    try:
+                        await retry_pipe.execute()
+                    except NoScriptError:
+                        noscript_on_retry = True
+
+            if noscript_on_retry:
+                raise PersistentNoScriptError(
+                    "NOSCRIPT error persisted after re-registering scripts. "
+                    "This indicates a server-side problem with Redis."
+                )
+
             _context_var.set(None)
             _context_xx_pipe.set(False)
 

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/config.py

@@ -22,4 +22,9 @@ class RedisConfig:
     redis_type: dict[type, type] = dataclasses.field(default_factory=create_all_types)
     ttl: int | None = None
     init_with_rapyer: bool = True
-    refresh_ttl: bool = True  # Enable TTL refresh on read/write operations by default
+    # Enable TTL refresh on read/write operations by default
+    refresh_ttl: bool = True
+    # If True, all non-Redis-supported fields are treated as SafeLoad
+    safe_load_all: bool = False
+    # If True, use JSON serialization for fields that support it instead of pickle
+    prefer_normal_json_dump: bool = False

rapyer-1.1.6/rapyer/errors/__init__.py

@@ -0,0 +1,17 @@
+from rapyer.errors.base import (
+    BadFilterError,
+    FindError,
+    PersistentNoScriptError,
+    RapyerError,
+    ScriptsNotInitializedError,
+    UnsupportedIndexedFieldError,
+)
+
+__all__ = [
+    "BadFilterError",
+    "FindError",
+    "PersistentNoScriptError",
+    "RapyerError",
+    "ScriptsNotInitializedError",
+    "UnsupportedIndexedFieldError",
+]

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/errors/base.py

@@ -24,3 +24,15 @@ class BadFilterError(FindError):
 
 class UnsupportedIndexedFieldError(FindError):
     pass
+
+
+class CantSerializeRedisValueError(RapyerError):
+    pass
+
+
+class ScriptsNotInitializedError(RapyerError):
+    pass
+
+
+class PersistentNoScriptError(RapyerError):
+    pass

rapyer-1.1.6/rapyer/fields/__init__.py

@@ -0,0 +1,5 @@
+from rapyer.fields.index import Index
+from rapyer.fields.key import Key
+from rapyer.fields.safe_load import SafeLoad
+
+__all__ = ["Key", "Index", "SafeLoad"]

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/fields/expression.py

@@ -1,7 +1,6 @@
 from typing import Any
 
 from pydantic import TypeAdapter
-
 from rapyer.errors import BadFilterError
 from rapyer.types.base import REDIS_DUMP_FLAG_NAME
 from rapyer.typing_support import Unpack

rapyer-1.1.6/rapyer/fields/safe_load.py

@@ -0,0 +1,27 @@
+import dataclasses
+from typing import TYPE_CHECKING, Annotated, Any, Generic, TypeAlias, TypeVar
+
+
+@dataclasses.dataclass(frozen=True)
+class SafeLoadAnnotation:
+    pass
+
+
+T = TypeVar("T")
+
+
+class _SafeLoadType(Generic[T]):
+    def __new__(cls, typ: Any = None):
+        if typ is None:
+            return SafeLoadAnnotation()
+        return Annotated[typ, SafeLoadAnnotation()]
+
+    def __class_getitem__(cls, item):
+        return Annotated[item, SafeLoadAnnotation()]
+
+
+SafeLoad = _SafeLoadType
+
+
+if TYPE_CHECKING:
+    SafeLoad: TypeAlias = Annotated[T, SafeLoadAnnotation()]  # pragma: no cover

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/init.py

@@ -1,21 +1,39 @@
+import logging
+
 import redis.asyncio as redis_async
+from rapyer.base import REDIS_MODELS
+from rapyer.scripts import register_scripts
 from redis import ResponseError
 from redis.asyncio.client import Redis
 
-from rapyer.base import REDIS_MODELS
-
 
 async def init_rapyer(
-    redis: str | Redis = None, ttl: int = None, override_old_idx: bool = True
+    redis: str | Redis = None,
+    ttl: int = None,
+    override_old_idx: bool = True,
+    prefer_normal_json_dump: bool = None,
+    logger: logging.Logger = None,
 ):
+    if logger is not None:
+        rapyer_logger = logging.getLogger("rapyer")
+        rapyer_logger.setLevel(logger.level)
+        rapyer_logger.handlers.clear()
+        for handler in logger.handlers:
+            rapyer_logger.addHandler(handler)
+
     if isinstance(redis, str):
         redis = redis_async.from_url(redis, decode_responses=True, max_connections=20)
 
+    if redis is not None:
+        await register_scripts(redis)
+
     for model in REDIS_MODELS:
         if redis is not None:
             model.Meta.redis = redis
         if ttl is not None:
             model.Meta.ttl = ttl
+        if prefer_normal_json_dump is not None:
+            model.Meta.prefer_normal_json_dump = prefer_normal_json_dump
 
         # Initialize model fields
         model.init_class()

rapyer-1.1.6/rapyer/scripts.py

@@ -0,0 +1,86 @@
+from rapyer.errors import ScriptsNotInitializedError
+
+REMOVE_RANGE_SCRIPT_NAME = "remove_range"
+
+_REMOVE_RANGE_SCRIPT_TEMPLATE = """
+local key = KEYS[1]
+local path = ARGV[1]
+local start_idx = tonumber(ARGV[2])
+local end_idx = tonumber(ARGV[3])
+
+local arr_json = redis.call('JSON.GET', key, path)
+if not arr_json or arr_json == 'null' then
+    return nil
+end
+
+{extract_array}
+local n = #arr
+
+if start_idx < 0 then start_idx = n + start_idx end
+if end_idx < 0 then end_idx = n + end_idx end
+if start_idx < 0 then start_idx = 0 end
+if end_idx < 0 then end_idx = 0 end
+if end_idx > n then end_idx = n end
+if start_idx >= n or start_idx >= end_idx then return true end
+
+local new_arr = {{}}
+local j = 1
+
+for i = 1, start_idx do
+    new_arr[j] = arr[i]
+    j = j + 1
+end
+
+for i = end_idx + 1, n do
+    new_arr[j] = arr[i]
+    j = j + 1
+end
+
+local encoded = j == 1 and '[]' or cjson.encode(new_arr)
+redis.call('JSON.SET', key, path, encoded)
+return true
+"""
+
+_EXTRACT_ARRAY_REDIS = "local arr = cjson.decode(arr_json)[1]"
+_EXTRACT_ARRAY_FAKEREDIS = "local arr = cjson.decode(arr_json)"
+
+REMOVE_RANGE_SCRIPT = _REMOVE_RANGE_SCRIPT_TEMPLATE.format(
+    extract_array=_EXTRACT_ARRAY_REDIS
+)
+REMOVE_RANGE_SCRIPT_FAKEREDIS = _REMOVE_RANGE_SCRIPT_TEMPLATE.format(
+    extract_array=_EXTRACT_ARRAY_FAKEREDIS
+)
+
+SCRIPTS: dict[str, str] = {
+    REMOVE_RANGE_SCRIPT_NAME: REMOVE_RANGE_SCRIPT,
+}
+
+SCRIPTS_FAKEREDIS: dict[str, str] = {
+    REMOVE_RANGE_SCRIPT_NAME: REMOVE_RANGE_SCRIPT_FAKEREDIS,
+}
+
+_REGISTERED_SCRIPT_SHAS: dict[str, str] = {}
+
+
+def is_fakeredis(client) -> bool:
+    return "fakeredis" in type(client).__module__
+
+
+async def register_scripts(redis_client):
+    scripts = SCRIPTS_FAKEREDIS if is_fakeredis(redis_client) else SCRIPTS
+    for name, script_text in scripts.items():
+        sha = await redis_client.script_load(script_text)
+        _REGISTERED_SCRIPT_SHAS[name] = sha
+
+
+def run_sha(pipeline, script_name: str, keys: int, *args):
+    sha = _REGISTERED_SCRIPT_SHAS.get(script_name)
+    if sha is None:
+        raise ScriptsNotInitializedError(
+            f"Script '{script_name}' not loaded. Did you forget to call init_rapyer()?"
+        )
+    pipeline.evalsha(sha, keys, *args)
+
+
+async def handle_noscript_error(redis_client):
+    await register_scripts(redis_client)

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/types/base.py

@@ -1,5 +1,6 @@
 import abc
 import base64
+import logging
 import pickle
 from abc import ABC
 from typing import get_args, Any, TypeVar, Generic
@@ -7,14 +8,18 @@ from typing import get_args, Any, TypeVar, Generic
 from pydantic import GetCoreSchemaHandler, TypeAdapter
 from pydantic_core import core_schema
 from pydantic_core.core_schema import ValidationInfo, CoreSchema, SerializationInfo
-from redis.commands.search.field import TextField
-from typing_extensions import deprecated
-
 from rapyer.context import _context_var
+from rapyer.errors.base import CantSerializeRedisValueError
 from rapyer.typing_support import Self
 from rapyer.utils.redis import refresh_ttl_if_needed
+from redis.commands.search.field import TextField
+from typing_extensions import deprecated
+
+logger = logging.getLogger("rapyer")
 
 REDIS_DUMP_FLAG_NAME = "__rapyer_dumped__"
+FAILED_FIELDS_KEY = "__rapyer_failed_fields__"
+SKIP_SENTINEL = object()
 
 
 class RedisType(ABC):
@@ -129,6 +134,8 @@ T = TypeVar("T")
 
 
 class GenericRedisType(RedisType, Generic[T], ABC):
+    safe_load: bool = False
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         for key, val in self.iterate_items():
@@ -139,6 +146,18 @@ class GenericRedisType(RedisType, Generic[T], ABC):
         args = get_args(type_)
         return args[0] if args else Any
 
+    @classmethod
+    def try_deserialize_item(cls, item, identifier):
+        try:
+            return cls.deserialize_unknown(item)
+        except Exception as e:
+            if cls.safe_load:
+                logger.warning(
+                    "SafeLoad: Failed to deserialize item at '%s'.", identifier
+                )
+                return SKIP_SENTINEL
+            raise CantSerializeRedisValueError() from e
+
     @abc.abstractmethod
     def iterate_items(self):
         pass  # pragma: no cover

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/types/byte.py

@@ -2,7 +2,6 @@ from typing import TypeAlias, TYPE_CHECKING
 
 from pydantic_core import core_schema
 from pydantic_core.core_schema import ValidationInfo, SerializationInfo
-
 from rapyer.types.base import RedisType, REDIS_DUMP_FLAG_NAME
 
 

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/types/convert.py

@@ -1,16 +1,21 @@
-from typing import Any, get_origin
+from typing import get_origin
 
 from pydantic import BaseModel, PrivateAttr, TypeAdapter
-
 from rapyer.types.base import RedisType
 from rapyer.utils.annotation import TypeConverter, DYNAMIC_CLASS_DOC
 from rapyer.utils.pythonic import safe_issubclass
 
 
 class RedisConverter(TypeConverter):
-    def __init__(self, supported_types: dict[type, type], field_name: str):
+    def __init__(
+        self,
+        supported_types: dict[type, type],
+        field_name: str,
+        safe_load: bool = False,
+    ):
         self.supported_types = supported_types
         self.field_name = field_name
+        self.safe_load = safe_load
 
     def is_redis_type(self, type_to_check: type) -> bool:
         origin = get_origin(type_to_check) or type_to_check
@@ -62,6 +67,7 @@ class RedisConverter(TypeConverter):
             dict(
                 field_name=self.field_name,
                 original_type=original_type,
+                safe_load=self.safe_load,
                 __doc__=DYNAMIC_CLASS_DOC,
             ),
         )
@@ -86,6 +92,7 @@ class RedisConverter(TypeConverter):
             dict(
                 field_name=self.field_name,
                 original_type=original_type,
+                safe_load=self.safe_load,
                 __doc__=DYNAMIC_CLASS_DOC,
             ),
         )

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/types/datetime.py

@@ -3,9 +3,8 @@ from typing import TYPE_CHECKING
 
 from pydantic_core import core_schema
 from pydantic_core.core_schema import ValidationInfo, SerializationInfo
-from redis.commands.search.field import NumericField
-
 from rapyer.types.base import RedisType, REDIS_DUMP_FLAG_NAME
+from redis.commands.search.field import NumericField
 
 
 class RedisDatetime(datetime, RedisType):

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/types/dct.py

@@ -1,8 +1,12 @@
 from typing import TypeVar, Generic, get_args, Any, TypeAlias, TYPE_CHECKING
 
 from pydantic_core import core_schema
-
-from rapyer.types.base import GenericRedisType, RedisType, REDIS_DUMP_FLAG_NAME
+from rapyer.types.base import (
+    GenericRedisType,
+    RedisType,
+    REDIS_DUMP_FLAG_NAME,
+    SKIP_SENTINEL,
+)
 from rapyer.utils.redis import refresh_ttl_if_needed
 from rapyer.utils.redis import update_keys_in_pipeline
 
@@ -239,9 +243,15 @@ class RedisDict(dict[str, T], GenericRedisType, Generic[T]):
     def full_deserializer(cls, value: dict, info: core_schema.ValidationInfo):
         ctx = info.context or {}
         should_serialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME)
+
+        if not should_serialize_redis:
+            return value
+
         return {
-            key: cls.deserialize_unknown(item) if should_serialize_redis else item
+            key: deserialized
             for key, item in value.items()
+            if (deserialized := cls.try_deserialize_item(item, f"key '{key}'"))
+            is not SKIP_SENTINEL
         }
 
     @classmethod

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/types/float.py

@@ -1,9 +1,8 @@
 from typing import TypeAlias, TYPE_CHECKING
 
-from redis.commands.search.field import NumericField
-
 from rapyer.types.base import RedisType
 from rapyer.utils.redis import refresh_ttl_if_needed
+from redis.commands.search.field import NumericField
 
 
 class RedisFloat(float, RedisType):

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/types/integer.py

@@ -1,10 +1,9 @@
 from typing import TypeAlias, TYPE_CHECKING
 
-from redis.commands.search.field import NumericField
-from typing_extensions import deprecated
-
 from rapyer.types.base import RedisType
 from rapyer.utils.redis import refresh_ttl_if_needed
+from redis.commands.search.field import NumericField
+from typing_extensions import deprecated
 
 
 class RedisInt(int, RedisType):

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/types/lst.py

@@ -1,12 +1,20 @@
 import json
+import logging
 from typing import TypeVar, TYPE_CHECKING
 
 from pydantic_core import core_schema
 from pydantic_core.core_schema import ValidationInfo, SerializationInfo
+from rapyer.scripts import run_sha, REMOVE_RANGE_SCRIPT_NAME
+from rapyer.types.base import (
+    GenericRedisType,
+    RedisType,
+    REDIS_DUMP_FLAG_NAME,
+    SKIP_SENTINEL,
+)
+from rapyer.utils.redis import refresh_ttl_if_needed
 from typing_extensions import TypeAlias
 
-from rapyer.types.base import GenericRedisType, RedisType, REDIS_DUMP_FLAG_NAME
-from rapyer.utils.redis import refresh_ttl_if_needed
+logger = logging.getLogger("rapyer")
 
 T = TypeVar("T")
 
@@ -66,6 +74,24 @@ class RedisList(list, GenericRedisType[T]):
             self.pipeline.json().set(self.key, self.json_path, [])
         return super().clear()
 
+    def remove_range(self, start: int, end: int):
+        if self.pipeline:
+            run_sha(
+                self.pipeline,
+                REMOVE_RANGE_SCRIPT_NAME,
+                1,
+                self.key,
+                self.json_path,
+                start,
+                end,
+            )
+            del self[start:end]
+        else:
+            logger.warning(
+                "remove_range() called without a pipeline context. "
+                "No changes were made. Use 'async with model.apipeline():' to execute."
+            )
+
     async def aappend(self, __object):
         self.append(__object)
 
@@ -162,8 +188,14 @@ class RedisList(list, GenericRedisType[T]):
         ctx = info.context or {}
         is_redis_data = ctx.get(REDIS_DUMP_FLAG_NAME)
 
+        if not is_redis_data:
+            return value
+
         return [
-            cls.deserialize_unknown(item) if is_redis_data else item for item in value
+            deserialized
+            for idx, item in enumerate(value)
+            if (deserialized := cls.try_deserialize_item(item, f"index {idx}"))
+            is not SKIP_SENTINEL
         ]
 
     @classmethod

{rapyer-1.1.4 → rapyer-1.1.6}/rapyer/utils/fields.py

@@ -1,7 +1,8 @@
-from typing import get_origin, ClassVar
+from typing import get_origin, ClassVar, Any
 
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 from pydantic.fields import FieldInfo
+from pydantic_core import PydanticUndefined
 
 
 def _collect_annotations_recursive(
@@ -62,3 +63,25 @@ def is_redis_field(field_name, field_annotation):
         or field_name.endswith("_")
         or get_origin(field_annotation) is ClassVar
     )
+
+
+def is_field_default_has_value(field_default):
+    return field_default is not PydanticUndefined and field_default is not None
+
+
+def is_type_json_serializable(typ: type, test_value: Any) -> bool:
+    try:
+        adapter = TypeAdapter(typ)
+        if isinstance(test_value, FieldInfo):
+            if is_field_default_has_value(test_value.default):
+                test_value = test_value.default
+            elif is_field_default_has_value(test_value.default_factory):
+                test_value = test_value.default_factory()
+            else:
+                return False
+        if test_value is None:
+            return False
+        adapter.dump_python(test_value, mode="json")
+        return True
+    except Exception:
+        return False

rapyer-1.1.4/rapyer/errors/__init__.py

@@ -1,8 +0,0 @@
-from rapyer.errors.base import (
-    BadFilterError,
-    FindError,
-    RapyerError,
-    UnsupportedIndexedFieldError,
-)
-
-__all__ = ["BadFilterError", "FindError", "RapyerError", "UnsupportedIndexedFieldError"]

rapyer-1.1.4/rapyer/fields/__init__.py

@@ -1,4 +0,0 @@
-from rapyer.fields.index import Index
-from rapyer.fields.key import Key
-
-__all__ = ["Key", "Index"]