rapyer 1.1.4__tar.gz → 1.1.5__tar.gz

{rapyer-1.1.4 → rapyer-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rapyer
-Version: 1.1.4
+Version: 1.1.5
 Summary: Pydantic models with Redis as the backend
 License: MIT
 Keywords: redis,redis-json,pydantic,pydantic-v2,orm,database,async,nosql,cache,key-value,data-modeling,python,backend,storage,serialization,validation
@@ -23,7 +23,12 @@ Classifier: Topic :: Database :: Database Engines/Servers
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
 Classifier: Operating System :: OS Independent
+Provides-Extra: test
+Requires-Dist: fakeredis[json,lua] (>=2.20.0) ; extra == "test"
 Requires-Dist: pydantic (>=2.11.0,<2.13.0)
+Requires-Dist: pytest (>=8.4.2) ; extra == "test"
+Requires-Dist: pytest-asyncio (>=0.25.0) ; extra == "test"
+Requires-Dist: pytest-cov (>=6.0.0) ; extra == "test"
 Requires-Dist: redis[async] (>=6.0.0,<7.1.0)
 Project-URL: Bug Tracker, https://github.com/imaginary-cherry/rapyer/issues
 Project-URL: Changelog, https://github.com/imaginary-cherry/rapyer/releases

{rapyer-1.1.4 → rapyer-1.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "rapyer"
-version = "1.1.4"
+version = "1.1.5"
 description = "Pydantic models with Redis as the backend"
 authors = [{name = "YedidyaHKfir", email = "yedidyakfir@gmail.com"}]
 readme = "README.md"
@@ -51,6 +51,14 @@ dependencies = [
     "pydantic>=2.11.0, <2.13.0",
 ]
 
+[project.optional-dependencies]
+test = [
+    "pytest>=8.4.2",
+    "pytest-asyncio>=0.25.0",
+    "pytest-cov>=6.0.0",
+    "fakeredis[lua,json]>=2.20.0",
+]
+
 [project.urls]
 Homepage = "https://imaginary-cherry.github.io/rapyer/"
 Documentation = "https://imaginary-cherry.github.io/rapyer/"
@@ -67,11 +75,6 @@ packages = [{include = "rapyer"}]
 black = "^25.9.0"
 mypy = "^1.0.0"
 
-[tool.poetry.group.tests.dependencies]
-pytest = "^8.4.2"
-pytest-asyncio = "^0.25.0"
-pytest-cov = "^6.0.0"
-
 [tool.coverage.run]
 source = ["rapyer"]
 omit = ["*/tests/*", "*/test_*"]

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/base.py

@@ -2,6 +2,7 @@ import asyncio
 import base64
 import contextlib
 import functools
+import logging
 import pickle
 import uuid
 from contextlib import AbstractAsyncContextManager
@@ -18,16 +19,24 @@ from pydantic import (
 from pydantic_core.core_schema import FieldSerializationInfo, ValidationInfo
 from redis.commands.search.index_definition import IndexDefinition, IndexType
 from redis.commands.search.query import Query
+from redis.exceptions import NoScriptError
 from typing_extensions import deprecated
 
 from rapyer.config import RedisConfig
 from rapyer.context import _context_var, _context_xx_pipe
-from rapyer.errors.base import KeyNotFound, UnsupportedIndexedFieldError
+from rapyer.errors.base import (
+    KeyNotFound,
+    PersistentNoScriptError,
+    UnsupportedIndexedFieldError,
+    CantSerializeRedisValueError,
+)
 from rapyer.fields.expression import ExpressionField, AtomicField, Expression
 from rapyer.fields.index import IndexAnnotation
 from rapyer.fields.key import KeyAnnotation
+from rapyer.fields.safe_load import SafeLoadAnnotation
 from rapyer.links import REDIS_SUPPORTED_LINK
-from rapyer.types.base import RedisType, REDIS_DUMP_FLAG_NAME
+from rapyer.scripts import handle_noscript_error
+from rapyer.types.base import RedisType, REDIS_DUMP_FLAG_NAME, FAILED_FIELDS_KEY
 from rapyer.types.convert import RedisConverter
 from rapyer.typing_support import Self, Unpack
 from rapyer.utils.annotation import (
@@ -36,7 +45,11 @@ from rapyer.utils.annotation import (
     field_with_flag,
     DYNAMIC_CLASS_DOC,
 )
-from rapyer.utils.fields import get_all_pydantic_annotation, is_redis_field
+from rapyer.utils.fields import (
+    get_all_pydantic_annotation,
+    is_redis_field,
+    is_type_json_serializable,
+)
 from rapyer.utils.pythonic import safe_issubclass
 from rapyer.utils.redis import (
     acquire_lock,
@@ -44,26 +57,45 @@ from rapyer.utils.redis import (
     refresh_ttl_if_needed,
 )
 
+logger = logging.getLogger("rapyer")
 
-def make_pickle_field_serializer(field: str):
+
+def make_pickle_field_serializer(
+    field: str, safe_load: bool = False, can_json: bool = False
+):
     @field_serializer(field, when_used="json-unless-none")
-    def pickle_field_serializer(v, info: FieldSerializationInfo):
+    @classmethod
+    def pickle_field_serializer(cls, v, info: FieldSerializationInfo):
         ctx = info.context or {}
         should_serialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME, False)
-        if should_serialize_redis:
+        # Skip pickling if field CAN be JSON serialized AND user prefers JSON dump
+        field_can_be_json = can_json and cls.Meta.prefer_normal_json_dump
+        if should_serialize_redis and not field_can_be_json:
             return base64.b64encode(pickle.dumps(v)).decode("utf-8")
         return v
 
     pickle_field_serializer.__name__ = f"__serialize_{field}"
 
     @field_validator(field, mode="before")
-    def pickle_field_validator(v, info: ValidationInfo):
+    @classmethod
+    def pickle_field_validator(cls, v, info: ValidationInfo):
         if v is None:
             return v
         ctx = info.context or {}
         should_serialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME, False)
         if should_serialize_redis:
-            return pickle.loads(base64.b64decode(v))
+            try:
+                field_can_be_json = can_json and cls.Meta.prefer_normal_json_dump
+                if should_serialize_redis and not field_can_be_json:
+                    return pickle.loads(base64.b64decode(v))
+                return v
+            except Exception as e:
+                if safe_load:
+                    failed_fields = ctx.setdefault(FAILED_FIELDS_KEY, set())
+                    failed_fields.add(field)
+                    logger.warning("SafeLoad: Failed to deserialize field '%s'", field)
+                    return None
+                raise CantSerializeRedisValueError() from e
         return v
 
     pickle_field_validator.__name__ = f"__deserialize_{field}"
@@ -71,15 +103,40 @@ def make_pickle_field_serializer(field: str):
     return pickle_field_serializer, pickle_field_validator
 
 
+# TODO: Remove in next major version (2.0) - backward compatibility for pickled data
+# This validator handles loading old pickled data for fields that are now JSON-serializable.
+# In 2.0, remove this function and the validator registration in __init_subclass__.
+def make_backward_compat_validator(field: str):
+    @field_validator(field, mode="before")
+    def backward_compat_validator(v, info: ValidationInfo):
+        ctx = info.context or {}
+        should_deserialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME, False)
+        if should_deserialize_redis and isinstance(v, str):
+            try:
+                return pickle.loads(base64.b64decode(v))
+            except Exception:
+                pass
+        return v
+
+    backward_compat_validator.__name__ = f"__backward_compat_{field}"
+    return backward_compat_validator
+
+
 class AtomicRedisModel(BaseModel):
     _pk: str = PrivateAttr(default_factory=lambda: str(uuid.uuid4()))
     _base_model_link: Self | RedisType = PrivateAttr(default=None)
+    _failed_fields: set[str] = PrivateAttr(default_factory=set)
 
     Meta: ClassVar[RedisConfig] = RedisConfig()
     _key_field_name: ClassVar[str | None] = None
+    _safe_load_fields: ClassVar[set[str]] = set()
     _field_name: str = PrivateAttr(default="")
     model_config = ConfigDict(validate_assignment=True, validate_default=True)
 
+    @property
+    def failed_fields(self) -> set[str]:
+        return self._failed_fields
+
     @property
     def pk(self):
         if self._key_field_name:
@@ -184,11 +241,13 @@ class AtomicRedisModel(BaseModel):
         self._pk = value.split(":", maxsplit=1)[-1]
 
     def __init_subclass__(cls, **kwargs):
-        # Find a field with KeyAnnotation and save its name
+        # Find fields with KeyAnnotation and SafeLoadAnnotation
+        cls._safe_load_fields = set()
         for field_name, annotation in cls.__annotations__.items():
             if has_annotation(annotation, KeyAnnotation):
                 cls._key_field_name = field_name
-                break
+            if has_annotation(annotation, SafeLoadAnnotation):
+                cls._safe_load_fields.add(field_name)
 
         # Redefine annotations to use redis types
         pydantic_annotation = get_all_pydantic_annotation(cls, AtomicRedisModel)
@@ -200,7 +259,13 @@ class AtomicRedisModel(BaseModel):
         original_annotations.update(new_annotation)
         new_annotations = {
             field_name: replace_to_redis_types_in_annotation(
-                annotation, RedisConverter(cls.Meta.redis_type, f".{field_name}")
+                annotation,
+                RedisConverter(
+                    cls.Meta.redis_type,
+                    f".{field_name}",
+                    safe_load=field_name in cls._safe_load_fields
+                    or cls.Meta.safe_load_all,
+                ),
             )
             for field_name, annotation in original_annotations.items()
             if is_redis_field(field_name, annotation)
@@ -216,9 +281,22 @@ class AtomicRedisModel(BaseModel):
             if not is_redis_field(attr_name, attr_type):
                 continue
             if original_annotations[attr_name] == attr_type:
-                serializer, validator = make_pickle_field_serializer(attr_name)
-                setattr(cls, serializer.__name__, serializer)
-                setattr(cls, validator.__name__, validator)
+                default_value = cls.__dict__.get(attr_name, None)
+                can_json = is_type_json_serializable(attr_type, default_value)
+                should_json_serialize = can_json and cls.Meta.prefer_normal_json_dump
+
+                if not should_json_serialize:
+                    is_field_marked_safe = attr_name in cls._safe_load_fields
+                    is_safe_load = is_field_marked_safe or cls.Meta.safe_load_all
+                    serializer, validator = make_pickle_field_serializer(
+                        attr_name, safe_load=is_safe_load, can_json=can_json
+                    )
+                    setattr(cls, serializer.__name__, serializer)
+                    setattr(cls, validator.__name__, validator)
+                else:
+                    # TODO: Remove in 2.0 - backward compatibility for old pickled data
+                    validator = make_backward_compat_validator(attr_name)
+                    setattr(cls, validator.__name__, validator)
                 continue
 
         # Update the redis model list for initialization
@@ -337,8 +415,10 @@ class AtomicRedisModel(BaseModel):
             raise KeyNotFound(f"{key} is missing in redis")
         model_dump = model_dump[0]
 
-        instance = cls.model_validate(model_dump, context={REDIS_DUMP_FLAG_NAME: True})
+        context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
+        instance = cls.model_validate(model_dump, context=context)
         instance.key = key
+        instance._failed_fields = context.get(FAILED_FIELDS_KEY, set())
         await refresh_ttl_if_needed(
             cls.Meta.redis, key, cls.Meta.ttl, cls.Meta.refresh_ttl
         )
@@ -355,9 +435,11 @@ class AtomicRedisModel(BaseModel):
         if not model_dump:
             raise KeyNotFound(f"{self.key} is missing in redis")
         model_dump = model_dump[0]
-        instance = self.__class__(**model_dump)
+        context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
+        instance = self.__class__.model_validate(model_dump, context=context)
         instance._pk = self._pk
         instance._base_model_link = self._base_model_link
+        instance._failed_fields = context.get(FAILED_FIELDS_KEY, set())
         await refresh_ttl_if_needed(
             self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
         )
@@ -402,8 +484,10 @@ class AtomicRedisModel(BaseModel):
 
         instances = []
         for model, key in zip(models, keys):
-            model = cls.model_validate(model[0], context={REDIS_DUMP_FLAG_NAME: True})
+            context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
+            model = cls.model_validate(model[0], context=context)
             model.key = key
+            model._failed_fields = context.get(FAILED_FIELDS_KEY, set())
             instances.append(model)
         return instances
 
@@ -512,7 +596,7 @@ class AtomicRedisModel(BaseModel):
     async def apipeline(
         self, ignore_if_deleted: bool = False
     ) -> AbstractAsyncContextManager[Self]:
-        async with self.Meta.redis.pipeline() as pipe:
+        async with self.Meta.redis.pipeline(transaction=True) as pipe:
             try:
                 redis_model = await self.__class__.aget(self.key)
                 unset_fields = {
@@ -527,7 +611,37 @@ class AtomicRedisModel(BaseModel):
             _context_var.set(pipe)
             _context_xx_pipe.set(ignore_if_deleted)
             yield redis_model
-            await pipe.execute()
+            commands_backup = list(pipe.command_stack)
+            noscript_on_first_attempt = False
+            noscript_on_retry = False
+
+            try:
+                await pipe.execute()
+            except NoScriptError:
+                noscript_on_first_attempt = True
+
+            if noscript_on_first_attempt:
+                await handle_noscript_error(self.Meta.redis)
+                evalsha_commands = [
+                    (args, options)
+                    for args, options in commands_backup
+                    if args[0] == "EVALSHA"
+                ]
+                # Retry execute the pipeline actions
+                async with self.Meta.redis.pipeline(transaction=True) as retry_pipe:
+                    for args, options in evalsha_commands:
+                        retry_pipe.execute_command(*args, **options)
+                    try:
+                        await retry_pipe.execute()
+                    except NoScriptError:
+                        noscript_on_retry = True
+
+            if noscript_on_retry:
+                raise PersistentNoScriptError(
+                    "NOSCRIPT error persisted after re-registering scripts. "
+                    "This indicates a server-side problem with Redis."
+                )
+
             await refresh_ttl_if_needed(
                 self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
             )

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/config.py

@@ -22,4 +22,9 @@ class RedisConfig:
     redis_type: dict[type, type] = dataclasses.field(default_factory=create_all_types)
     ttl: int | None = None
     init_with_rapyer: bool = True
-    refresh_ttl: bool = True  # Enable TTL refresh on read/write operations by default
+    # Enable TTL refresh on read/write operations by default
+    refresh_ttl: bool = True
+    # If True, all non-Redis-supported fields are treated as SafeLoad
+    safe_load_all: bool = False
+    # If True, use JSON serialization for fields that support it instead of pickle
+    prefer_normal_json_dump: bool = False

rapyer-1.1.5/rapyer/errors/__init__.py

@@ -0,0 +1,17 @@
+from rapyer.errors.base import (
+    BadFilterError,
+    FindError,
+    PersistentNoScriptError,
+    RapyerError,
+    ScriptsNotInitializedError,
+    UnsupportedIndexedFieldError,
+)
+
+__all__ = [
+    "BadFilterError",
+    "FindError",
+    "PersistentNoScriptError",
+    "RapyerError",
+    "ScriptsNotInitializedError",
+    "UnsupportedIndexedFieldError",
+]

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/errors/base.py

@@ -24,3 +24,15 @@ class BadFilterError(FindError):
 
 class UnsupportedIndexedFieldError(FindError):
     pass
+
+
+class CantSerializeRedisValueError(RapyerError):
+    pass
+
+
+class ScriptsNotInitializedError(RapyerError):
+    pass
+
+
+class PersistentNoScriptError(RapyerError):
+    pass

rapyer-1.1.5/rapyer/fields/__init__.py

@@ -0,0 +1,5 @@
+from rapyer.fields.index import Index
+from rapyer.fields.key import Key
+from rapyer.fields.safe_load import SafeLoad
+
+__all__ = ["Key", "Index", "SafeLoad"]

rapyer-1.1.5/rapyer/fields/safe_load.py

@@ -0,0 +1,27 @@
+import dataclasses
+from typing import TYPE_CHECKING, Annotated, Any, Generic, TypeAlias, TypeVar
+
+
+@dataclasses.dataclass(frozen=True)
+class SafeLoadAnnotation:
+    pass
+
+
+T = TypeVar("T")
+
+
+class _SafeLoadType(Generic[T]):
+    def __new__(cls, typ: Any = None):
+        if typ is None:
+            return SafeLoadAnnotation()
+        return Annotated[typ, SafeLoadAnnotation()]
+
+    def __class_getitem__(cls, item):
+        return Annotated[item, SafeLoadAnnotation()]
+
+
+SafeLoad = _SafeLoadType
+
+
+if TYPE_CHECKING:
+    SafeLoad: TypeAlias = Annotated[T, SafeLoadAnnotation()]  # pragma: no cover

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/init.py

@@ -3,19 +3,28 @@ from redis import ResponseError
 from redis.asyncio.client import Redis
 
 from rapyer.base import REDIS_MODELS
+from rapyer.scripts import register_scripts
 
 
 async def init_rapyer(
-    redis: str | Redis = None, ttl: int = None, override_old_idx: bool = True
+    redis: str | Redis = None,
+    ttl: int = None,
+    override_old_idx: bool = True,
+    prefer_normal_json_dump: bool = None,
 ):
     if isinstance(redis, str):
         redis = redis_async.from_url(redis, decode_responses=True, max_connections=20)
 
+    if redis is not None:
+        await register_scripts(redis)
+
     for model in REDIS_MODELS:
         if redis is not None:
             model.Meta.redis = redis
         if ttl is not None:
             model.Meta.ttl = ttl
+        if prefer_normal_json_dump is not None:
+            model.Meta.prefer_normal_json_dump = prefer_normal_json_dump
 
         # Initialize model fields
         model.init_class()

rapyer-1.1.5/rapyer/scripts.py

@@ -0,0 +1,86 @@
+from rapyer.errors import ScriptsNotInitializedError
+
+REMOVE_RANGE_SCRIPT_NAME = "remove_range"
+
+_REMOVE_RANGE_SCRIPT_TEMPLATE = """
+local key = KEYS[1]
+local path = ARGV[1]
+local start_idx = tonumber(ARGV[2])
+local end_idx = tonumber(ARGV[3])
+
+local arr_json = redis.call('JSON.GET', key, path)
+if not arr_json or arr_json == 'null' then
+    return nil
+end
+
+{extract_array}
+local n = #arr
+
+if start_idx < 0 then start_idx = n + start_idx end
+if end_idx < 0 then end_idx = n + end_idx end
+if start_idx < 0 then start_idx = 0 end
+if end_idx < 0 then end_idx = 0 end
+if end_idx > n then end_idx = n end
+if start_idx >= n or start_idx >= end_idx then return true end
+
+local new_arr = {{}}
+local j = 1
+
+for i = 1, start_idx do
+    new_arr[j] = arr[i]
+    j = j + 1
+end
+
+for i = end_idx + 1, n do
+    new_arr[j] = arr[i]
+    j = j + 1
+end
+
+local encoded = j == 1 and '[]' or cjson.encode(new_arr)
+redis.call('JSON.SET', key, path, encoded)
+return true
+"""
+
+_EXTRACT_ARRAY_REDIS = "local arr = cjson.decode(arr_json)[1]"
+_EXTRACT_ARRAY_FAKEREDIS = "local arr = cjson.decode(arr_json)"
+
+REMOVE_RANGE_SCRIPT = _REMOVE_RANGE_SCRIPT_TEMPLATE.format(
+    extract_array=_EXTRACT_ARRAY_REDIS
+)
+REMOVE_RANGE_SCRIPT_FAKEREDIS = _REMOVE_RANGE_SCRIPT_TEMPLATE.format(
+    extract_array=_EXTRACT_ARRAY_FAKEREDIS
+)
+
+SCRIPTS: dict[str, str] = {
+    REMOVE_RANGE_SCRIPT_NAME: REMOVE_RANGE_SCRIPT,
+}
+
+SCRIPTS_FAKEREDIS: dict[str, str] = {
+    REMOVE_RANGE_SCRIPT_NAME: REMOVE_RANGE_SCRIPT_FAKEREDIS,
+}
+
+_REGISTERED_SCRIPT_SHAS: dict[str, str] = {}
+
+
+def is_fakeredis(client) -> bool:
+    return "fakeredis" in type(client).__module__
+
+
+async def register_scripts(redis_client):
+    scripts = SCRIPTS_FAKEREDIS if is_fakeredis(redis_client) else SCRIPTS
+    for name, script_text in scripts.items():
+        sha = await redis_client.script_load(script_text)
+        _REGISTERED_SCRIPT_SHAS[name] = sha
+
+
+def run_sha(pipeline, script_name: str, keys: int, *args):
+    sha = _REGISTERED_SCRIPT_SHAS.get(script_name)
+    if sha is None:
+        raise ScriptsNotInitializedError(
+            f"Script '{script_name}' not loaded. Did you forget to call init_rapyer()?"
+        )
+    pipeline.evalsha(sha, keys, *args)
+
+
+async def handle_noscript_error(redis_client):
+    await register_scripts(redis_client)

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/types/base.py

@@ -1,5 +1,6 @@
 import abc
 import base64
+import logging
 import pickle
 from abc import ABC
 from typing import get_args, Any, TypeVar, Generic
@@ -11,10 +12,15 @@ from redis.commands.search.field import TextField
 from typing_extensions import deprecated
 
 from rapyer.context import _context_var
+from rapyer.errors.base import CantSerializeRedisValueError
 from rapyer.typing_support import Self
 from rapyer.utils.redis import refresh_ttl_if_needed
 
+logger = logging.getLogger("rapyer")
+
 REDIS_DUMP_FLAG_NAME = "__rapyer_dumped__"
+FAILED_FIELDS_KEY = "__rapyer_failed_fields__"
+SKIP_SENTINEL = object()
 
 
 class RedisType(ABC):
@@ -129,6 +135,8 @@ T = TypeVar("T")
 
 
 class GenericRedisType(RedisType, Generic[T], ABC):
+    safe_load: bool = False
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         for key, val in self.iterate_items():
@@ -139,6 +147,18 @@ class GenericRedisType(RedisType, Generic[T], ABC):
         args = get_args(type_)
         return args[0] if args else Any
 
+    @classmethod
+    def try_deserialize_item(cls, item, identifier):
+        try:
+            return cls.deserialize_unknown(item)
+        except Exception as e:
+            if cls.safe_load:
+                logger.warning(
+                    "SafeLoad: Failed to deserialize item at '%s'.", identifier
+                )
+                return SKIP_SENTINEL
+            raise CantSerializeRedisValueError() from e
+
     @abc.abstractmethod
     def iterate_items(self):
         pass  # pragma: no cover

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/types/convert.py

@@ -1,4 +1,4 @@
-from typing import Any, get_origin
+from typing import get_origin
 
 from pydantic import BaseModel, PrivateAttr, TypeAdapter
 
@@ -8,9 +8,15 @@ from rapyer.utils.pythonic import safe_issubclass
 
 
 class RedisConverter(TypeConverter):
-    def __init__(self, supported_types: dict[type, type], field_name: str):
+    def __init__(
+        self,
+        supported_types: dict[type, type],
+        field_name: str,
+        safe_load: bool = False,
+    ):
         self.supported_types = supported_types
         self.field_name = field_name
+        self.safe_load = safe_load
 
     def is_redis_type(self, type_to_check: type) -> bool:
         origin = get_origin(type_to_check) or type_to_check
@@ -62,6 +68,7 @@ class RedisConverter(TypeConverter):
             dict(
                 field_name=self.field_name,
                 original_type=original_type,
+                safe_load=self.safe_load,
                 __doc__=DYNAMIC_CLASS_DOC,
             ),
         )
@@ -86,6 +93,7 @@ class RedisConverter(TypeConverter):
             dict(
                 field_name=self.field_name,
                 original_type=original_type,
+                safe_load=self.safe_load,
                 __doc__=DYNAMIC_CLASS_DOC,
             ),
         )

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/types/dct.py

@@ -2,7 +2,12 @@ from typing import TypeVar, Generic, get_args, Any, TypeAlias, TYPE_CHECKING
 
 from pydantic_core import core_schema
 
-from rapyer.types.base import GenericRedisType, RedisType, REDIS_DUMP_FLAG_NAME
+from rapyer.types.base import (
+    GenericRedisType,
+    RedisType,
+    REDIS_DUMP_FLAG_NAME,
+    SKIP_SENTINEL,
+)
 from rapyer.utils.redis import refresh_ttl_if_needed
 from rapyer.utils.redis import update_keys_in_pipeline
 
@@ -239,9 +244,15 @@ class RedisDict(dict[str, T], GenericRedisType, Generic[T]):
     def full_deserializer(cls, value: dict, info: core_schema.ValidationInfo):
         ctx = info.context or {}
         should_serialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME)
+
+        if not should_serialize_redis:
+            return value
+
         return {
-            key: cls.deserialize_unknown(item) if should_serialize_redis else item
+            key: deserialized
             for key, item in value.items()
+            if (deserialized := cls.try_deserialize_item(item, f"key '{key}'"))
+            is not SKIP_SENTINEL
         }
 
     @classmethod

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/types/lst.py

@@ -1,13 +1,22 @@
 import json
+import logging
 from typing import TypeVar, TYPE_CHECKING
 
 from pydantic_core import core_schema
 from pydantic_core.core_schema import ValidationInfo, SerializationInfo
 from typing_extensions import TypeAlias
 
-from rapyer.types.base import GenericRedisType, RedisType, REDIS_DUMP_FLAG_NAME
+from rapyer.scripts import run_sha, REMOVE_RANGE_SCRIPT_NAME
+from rapyer.types.base import (
+    GenericRedisType,
+    RedisType,
+    REDIS_DUMP_FLAG_NAME,
+    SKIP_SENTINEL,
+)
 from rapyer.utils.redis import refresh_ttl_if_needed
 
+logger = logging.getLogger("rapyer")
+
 T = TypeVar("T")
 
 
@@ -66,6 +75,24 @@ class RedisList(list, GenericRedisType[T]):
             self.pipeline.json().set(self.key, self.json_path, [])
         return super().clear()
 
+    def remove_range(self, start: int, end: int):
+        if self.pipeline:
+            run_sha(
+                self.pipeline,
+                REMOVE_RANGE_SCRIPT_NAME,
+                1,
+                self.key,
+                self.json_path,
+                start,
+                end,
+            )
+            del self[start:end]
+        else:
+            logger.warning(
+                "remove_range() called without a pipeline context. "
+                "No changes were made. Use 'async with model.apipeline():' to execute."
+            )
+
     async def aappend(self, __object):
         self.append(__object)
 
@@ -162,8 +189,14 @@ class RedisList(list, GenericRedisType[T]):
         ctx = info.context or {}
         is_redis_data = ctx.get(REDIS_DUMP_FLAG_NAME)
 
+        if not is_redis_data:
+            return value
+
         return [
-            cls.deserialize_unknown(item) if is_redis_data else item for item in value
+            deserialized
+            for idx, item in enumerate(value)
+            if (deserialized := cls.try_deserialize_item(item, f"index {idx}"))
+            is not SKIP_SENTINEL
         ]
 
     @classmethod

{rapyer-1.1.4 → rapyer-1.1.5}/rapyer/utils/fields.py

@@ -1,7 +1,8 @@
-from typing import get_origin, ClassVar
+from typing import get_origin, ClassVar, Any
 
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 from pydantic.fields import FieldInfo
+from pydantic_core import PydanticUndefined
 
 
 def _collect_annotations_recursive(
@@ -62,3 +63,25 @@ def is_redis_field(field_name, field_annotation):
         or field_name.endswith("_")
         or get_origin(field_annotation) is ClassVar
     )
+
+
+def is_field_default_has_value(field_default):
+    return field_default is not PydanticUndefined and field_default is not None
+
+
+def is_type_json_serializable(typ: type, test_value: Any) -> bool:
+    try:
+        adapter = TypeAdapter(typ)
+        if isinstance(test_value, FieldInfo):
+            if is_field_default_has_value(test_value.default):
+                test_value = test_value.default
+            elif is_field_default_has_value(test_value.default_factory):
+                test_value = test_value.default_factory()
+            else:
+                return False
+        if test_value is None:
+            return False
+        adapter.dump_python(test_value, mode="json")
+        return True
+    except Exception:
+        return False

rapyer-1.1.4/rapyer/errors/__init__.py

@@ -1,8 +0,0 @@
-from rapyer.errors.base import (
-    BadFilterError,
-    FindError,
-    RapyerError,
-    UnsupportedIndexedFieldError,
-)
-
-__all__ = ["BadFilterError", "FindError", "RapyerError", "UnsupportedIndexedFieldError"]

rapyer-1.1.4/rapyer/fields/__init__.py

@@ -1,4 +0,0 @@
-from rapyer.fields.index import Index
-from rapyer.fields.key import Key
-
-__all__ = ["Key", "Index"]