rapyer 1.1.6__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rapyer/__init__.py +2 -2
- rapyer/base.py +111 -148
- rapyer/config.py +2 -0
- rapyer/errors/__init__.py +6 -0
- rapyer/errors/base.py +8 -0
- rapyer/fields/expression.py +1 -0
- rapyer/init.py +10 -3
- rapyer/scripts/__init__.py +47 -0
- rapyer/scripts/constants.py +12 -0
- rapyer/scripts/loader.py +54 -0
- rapyer/scripts/lua/__init__.py +0 -0
- rapyer/scripts/lua/datetime/__init__.py +0 -0
- rapyer/scripts/lua/datetime/add.lua +80 -0
- rapyer/scripts/lua/dict/__init__.py +0 -0
- rapyer/scripts/lua/dict/pop.lua +13 -0
- rapyer/scripts/lua/dict/popitem.lua +29 -0
- rapyer/scripts/lua/list/__init__.py +0 -0
- rapyer/scripts/lua/list/remove_range.lua +36 -0
- rapyer/scripts/lua/numeric/__init__.py +0 -0
- rapyer/scripts/lua/numeric/floordiv.lua +13 -0
- rapyer/scripts/lua/numeric/mod.lua +13 -0
- rapyer/scripts/lua/numeric/mul.lua +13 -0
- rapyer/scripts/lua/numeric/pow.lua +13 -0
- rapyer/scripts/lua/numeric/pow_float.lua +13 -0
- rapyer/scripts/lua/numeric/truediv.lua +13 -0
- rapyer/scripts/lua/string/__init__.py +0 -0
- rapyer/scripts/lua/string/append.lua +13 -0
- rapyer/scripts/lua/string/mul.lua +13 -0
- rapyer/scripts/registry.py +79 -0
- rapyer/types/base.py +7 -18
- rapyer/types/byte.py +1 -0
- rapyer/types/convert.py +1 -0
- rapyer/types/datetime.py +58 -2
- rapyer/types/dct.py +25 -102
- rapyer/types/float.py +58 -9
- rapyer/types/integer.py +31 -20
- rapyer/types/lst.py +27 -21
- rapyer/types/string.py +22 -1
- rapyer/utils/redis.py +0 -7
- {rapyer-1.1.6.dist-info → rapyer-1.2.0.dist-info}/METADATA +1 -1
- rapyer-1.2.0.dist-info/RECORD +55 -0
- rapyer/scripts.py +0 -86
- rapyer-1.1.6.dist-info/RECORD +0 -34
- {rapyer-1.1.6.dist-info → rapyer-1.2.0.dist-info}/WHEEL +0 -0
rapyer/__init__.py
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
from rapyer.base import (
|
|
4
4
|
AtomicRedisModel,
|
|
5
5
|
aget,
|
|
6
|
+
afind,
|
|
6
7
|
find_redis_models,
|
|
7
8
|
ainsert,
|
|
8
|
-
get,
|
|
9
9
|
alock_from_key,
|
|
10
10
|
)
|
|
11
11
|
from rapyer.init import init_rapyer, teardown_rapyer
|
|
@@ -15,7 +15,7 @@ __all__ = [
|
|
|
15
15
|
"init_rapyer",
|
|
16
16
|
"teardown_rapyer",
|
|
17
17
|
"aget",
|
|
18
|
-
"
|
|
18
|
+
"afind",
|
|
19
19
|
"find_redis_models",
|
|
20
20
|
"ainsert",
|
|
21
21
|
"alock_from_key",
|
rapyer/base.py
CHANGED
|
@@ -6,7 +6,7 @@ import logging
|
|
|
6
6
|
import pickle
|
|
7
7
|
import uuid
|
|
8
8
|
from contextlib import AbstractAsyncContextManager
|
|
9
|
-
from typing import ClassVar, Any, get_origin
|
|
9
|
+
from typing import ClassVar, Any, get_origin, Optional
|
|
10
10
|
|
|
11
11
|
from pydantic import (
|
|
12
12
|
BaseModel,
|
|
@@ -18,6 +18,10 @@ from pydantic import (
|
|
|
18
18
|
ValidationError,
|
|
19
19
|
)
|
|
20
20
|
from pydantic_core.core_schema import FieldSerializationInfo, ValidationInfo
|
|
21
|
+
from redis.commands.search.index_definition import IndexDefinition, IndexType
|
|
22
|
+
from redis.commands.search.query import Query
|
|
23
|
+
from redis.exceptions import NoScriptError, ResponseError
|
|
24
|
+
|
|
21
25
|
from rapyer.config import RedisConfig
|
|
22
26
|
from rapyer.context import _context_var, _context_xx_pipe
|
|
23
27
|
from rapyer.errors.base import (
|
|
@@ -25,6 +29,7 @@ from rapyer.errors.base import (
|
|
|
25
29
|
PersistentNoScriptError,
|
|
26
30
|
UnsupportedIndexedFieldError,
|
|
27
31
|
CantSerializeRedisValueError,
|
|
32
|
+
RapyerModelDoesntExistError,
|
|
28
33
|
)
|
|
29
34
|
from rapyer.fields.expression import ExpressionField, AtomicField, Expression
|
|
30
35
|
from rapyer.fields.index import IndexAnnotation
|
|
@@ -50,12 +55,7 @@ from rapyer.utils.pythonic import safe_issubclass
|
|
|
50
55
|
from rapyer.utils.redis import (
|
|
51
56
|
acquire_lock,
|
|
52
57
|
update_keys_in_pipeline,
|
|
53
|
-
refresh_ttl_if_needed,
|
|
54
58
|
)
|
|
55
|
-
from redis.commands.search.index_definition import IndexDefinition, IndexType
|
|
56
|
-
from redis.commands.search.query import Query
|
|
57
|
-
from redis.exceptions import NoScriptError, ResponseError
|
|
58
|
-
from typing_extensions import deprecated
|
|
59
59
|
|
|
60
60
|
logger = logging.getLogger("rapyer")
|
|
61
61
|
|
|
@@ -103,25 +103,6 @@ def make_pickle_field_serializer(
|
|
|
103
103
|
return pickle_field_serializer, pickle_field_validator
|
|
104
104
|
|
|
105
105
|
|
|
106
|
-
# TODO: Remove in next major version (2.0) - backward compatibility for pickled data
|
|
107
|
-
# This validator handles loading old pickled data for fields that are now JSON-serializable.
|
|
108
|
-
# In 2.0, remove this function and the validator registration in __init_subclass__.
|
|
109
|
-
def make_backward_compat_validator(field: str):
|
|
110
|
-
@field_validator(field, mode="before")
|
|
111
|
-
def backward_compat_validator(v, info: ValidationInfo):
|
|
112
|
-
ctx = info.context or {}
|
|
113
|
-
should_deserialize_redis = ctx.get(REDIS_DUMP_FLAG_NAME, False)
|
|
114
|
-
if should_deserialize_redis and isinstance(v, str):
|
|
115
|
-
try:
|
|
116
|
-
return pickle.loads(base64.b64decode(v))
|
|
117
|
-
except Exception:
|
|
118
|
-
pass
|
|
119
|
-
return v
|
|
120
|
-
|
|
121
|
-
backward_compat_validator.__name__ = f"__backward_compat_{field}"
|
|
122
|
-
return backward_compat_validator
|
|
123
|
-
|
|
124
|
-
|
|
125
106
|
class AtomicRedisModel(BaseModel):
|
|
126
107
|
_pk: str = PrivateAttr(default_factory=lambda: str(uuid.uuid4()))
|
|
127
108
|
_base_model_link: Self | RedisType = PrivateAttr(default=None)
|
|
@@ -169,8 +150,13 @@ class AtomicRedisModel(BaseModel):
|
|
|
169
150
|
field_path = self.field_path
|
|
170
151
|
return f"${field_path}" if field_path else "$"
|
|
171
152
|
|
|
172
|
-
|
|
173
|
-
|
|
153
|
+
@classmethod
|
|
154
|
+
def should_refresh(cls):
|
|
155
|
+
return cls.Meta.refresh_ttl and cls.Meta.ttl is not None
|
|
156
|
+
|
|
157
|
+
async def refresh_ttl_if_needed(self):
|
|
158
|
+
if self.should_refresh():
|
|
159
|
+
await self.Meta.redis.expire(self.key, self.Meta.ttl)
|
|
174
160
|
|
|
175
161
|
@classmethod
|
|
176
162
|
def redis_schema(cls, redis_name: str = ""):
|
|
@@ -296,11 +282,6 @@ class AtomicRedisModel(BaseModel):
|
|
|
296
282
|
)
|
|
297
283
|
setattr(cls, serializer.__name__, serializer)
|
|
298
284
|
setattr(cls, validator.__name__, validator)
|
|
299
|
-
else:
|
|
300
|
-
# TODO: Remove in 2.0 - backward compatibility for old pickled data
|
|
301
|
-
validator = make_backward_compat_validator(attr_name)
|
|
302
|
-
setattr(cls, validator.__name__, validator)
|
|
303
|
-
continue
|
|
304
285
|
|
|
305
286
|
# Update the redis model list for initialization
|
|
306
287
|
# Skip dynamically created classes from type conversion
|
|
@@ -329,12 +310,6 @@ class AtomicRedisModel(BaseModel):
|
|
|
329
310
|
def is_inner_model(self) -> bool:
|
|
330
311
|
return bool(self.field_name)
|
|
331
312
|
|
|
332
|
-
@deprecated(
|
|
333
|
-
f"save function is deprecated and will become sync function in rapyer 1.2.0, use asave() instead"
|
|
334
|
-
)
|
|
335
|
-
async def save(self):
|
|
336
|
-
return await self.asave() # pragma: no cover
|
|
337
|
-
|
|
338
313
|
async def asave(self) -> Self:
|
|
339
314
|
model_dump = self.redis_dump()
|
|
340
315
|
await self.Meta.redis.json().set(self.key, self.json_path, model_dump)
|
|
@@ -349,12 +324,6 @@ class AtomicRedisModel(BaseModel):
|
|
|
349
324
|
def redis_dump_json(self):
|
|
350
325
|
return self.model_dump_json(context={REDIS_DUMP_FLAG_NAME: True})
|
|
351
326
|
|
|
352
|
-
@deprecated(
|
|
353
|
-
"duplicate function is deprecated and will be removed in rapyer 1.2.0, use aduplicate instead"
|
|
354
|
-
)
|
|
355
|
-
async def duplicate(self) -> Self:
|
|
356
|
-
return await self.aduplicate() # pragma: no cover
|
|
357
|
-
|
|
358
327
|
async def aduplicate(self) -> Self:
|
|
359
328
|
if self.is_inner_model():
|
|
360
329
|
raise RuntimeError("Can only duplicate from top level model")
|
|
@@ -363,12 +332,6 @@ class AtomicRedisModel(BaseModel):
|
|
|
363
332
|
await duplicated.asave()
|
|
364
333
|
return duplicated
|
|
365
334
|
|
|
366
|
-
@deprecated(
|
|
367
|
-
"duplicate_many function is deprecated and will be removed in rapyer 1.2.0, use aduplicate_many instead"
|
|
368
|
-
)
|
|
369
|
-
async def duplicate_many(self, num: int) -> list[Self]:
|
|
370
|
-
return await self.aduplicate_many(num) # pragma: no cover
|
|
371
|
-
|
|
372
335
|
async def aduplicate_many(self, num: int) -> list[Self]:
|
|
373
336
|
if self.is_inner_model():
|
|
374
337
|
raise RuntimeError("Can only duplicate from top level model")
|
|
@@ -395,24 +358,19 @@ class AtomicRedisModel(BaseModel):
|
|
|
395
358
|
for field_name in kwargs.keys()
|
|
396
359
|
}
|
|
397
360
|
|
|
398
|
-
async with self.Meta.redis.pipeline() as pipe:
|
|
361
|
+
async with self.Meta.redis.pipeline(transaction=True) as pipe:
|
|
399
362
|
update_keys_in_pipeline(pipe, self.key, **json_path_kwargs)
|
|
400
363
|
await pipe.execute()
|
|
401
|
-
await refresh_ttl_if_needed(
|
|
402
|
-
self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
|
|
403
|
-
)
|
|
364
|
+
await self.refresh_ttl_if_needed()
|
|
404
365
|
|
|
405
366
|
async def aset_ttl(self, ttl: int) -> None:
|
|
406
367
|
if self.is_inner_model():
|
|
407
368
|
raise RuntimeError("Can only set TTL from top level model")
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
)
|
|
414
|
-
async def get(cls, key: str) -> Self:
|
|
415
|
-
return await cls.aget(key) # pragma: no cover
|
|
369
|
+
pipeline = _context_var.get()
|
|
370
|
+
if pipeline is not None:
|
|
371
|
+
pipeline.expire(self.key, ttl)
|
|
372
|
+
else:
|
|
373
|
+
await self.Meta.redis.expire(self.key, ttl)
|
|
416
374
|
|
|
417
375
|
@classmethod
|
|
418
376
|
async def aget(cls, key: str) -> Self:
|
|
@@ -427,17 +385,10 @@ class AtomicRedisModel(BaseModel):
|
|
|
427
385
|
instance = cls.model_validate(model_dump, context=context)
|
|
428
386
|
instance.key = key
|
|
429
387
|
instance._failed_fields = context.get(FAILED_FIELDS_KEY, set())
|
|
430
|
-
|
|
431
|
-
cls.Meta.redis
|
|
432
|
-
)
|
|
388
|
+
if cls.should_refresh():
|
|
389
|
+
await cls.Meta.redis.expire(key, cls.Meta.ttl)
|
|
433
390
|
return instance
|
|
434
391
|
|
|
435
|
-
@deprecated(
|
|
436
|
-
"load function is deprecated and will be removed in rapyer 1.2.0, use aload() instead"
|
|
437
|
-
)
|
|
438
|
-
async def load(self):
|
|
439
|
-
return await self.aload() # pragma: no cover
|
|
440
|
-
|
|
441
392
|
async def aload(self) -> Self:
|
|
442
393
|
model_dump = await self.Meta.redis.json().get(self.key, self.json_path)
|
|
443
394
|
if not model_dump:
|
|
@@ -448,16 +399,32 @@ class AtomicRedisModel(BaseModel):
|
|
|
448
399
|
instance._pk = self._pk
|
|
449
400
|
instance._base_model_link = self._base_model_link
|
|
450
401
|
instance._failed_fields = context.get(FAILED_FIELDS_KEY, set())
|
|
451
|
-
await refresh_ttl_if_needed(
|
|
452
|
-
self.Meta.redis, self.key, self.Meta.ttl, self.Meta.refresh_ttl
|
|
453
|
-
)
|
|
402
|
+
await self.refresh_ttl_if_needed()
|
|
454
403
|
return instance
|
|
455
404
|
|
|
405
|
+
@classmethod
|
|
406
|
+
def create_redis_model(cls, model_dump: dict, key: str) -> Optional[Self]:
|
|
407
|
+
context = {REDIS_DUMP_FLAG_NAME: True, FAILED_FIELDS_KEY: set()}
|
|
408
|
+
try:
|
|
409
|
+
model = cls.model_validate(model_dump, context=context)
|
|
410
|
+
model.key = key
|
|
411
|
+
except ValidationError as exc:
|
|
412
|
+
logger.debug(
|
|
413
|
+
"Skipping key %s due to validation error during afind: %s",
|
|
414
|
+
key,
|
|
415
|
+
exc,
|
|
416
|
+
)
|
|
417
|
+
return None
|
|
418
|
+
model.key = key
|
|
419
|
+
model._failed_fields = context.get(FAILED_FIELDS_KEY, set())
|
|
420
|
+
return model
|
|
421
|
+
|
|
456
422
|
@classmethod
|
|
457
423
|
async def afind(cls, *args):
|
|
458
424
|
# Separate keys (str) from expressions (Expression)
|
|
459
425
|
provided_keys = [arg for arg in args if isinstance(arg, str)]
|
|
460
426
|
expressions = [arg for arg in args if isinstance(arg, Expression)]
|
|
427
|
+
raise_on_missing = bool(provided_keys)
|
|
461
428
|
|
|
462
429
|
if provided_keys and expressions:
|
|
463
430
|
logger.warning(
|
|
@@ -493,22 +460,17 @@ class AtomicRedisModel(BaseModel):
|
|
|
493
460
|
instances = []
|
|
494
461
|
for model, key in zip(models, targeted_keys):
|
|
495
462
|
if model is None:
|
|
463
|
+
if raise_on_missing:
|
|
464
|
+
raise KeyNotFound(f"{key} is missing in redis")
|
|
496
465
|
continue
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
logger.debug(
|
|
502
|
-
"Skipping key %s due to validation error during afind: %s",
|
|
503
|
-
key,
|
|
504
|
-
exc,
|
|
505
|
-
)
|
|
466
|
+
if not cls.Meta.is_fake_redis:
|
|
467
|
+
model = model[0]
|
|
468
|
+
model = cls.create_redis_model(model, key)
|
|
469
|
+
if model is None:
|
|
506
470
|
continue
|
|
507
|
-
model.key = key
|
|
508
|
-
model._failed_fields = context.get(FAILED_FIELDS_KEY, set())
|
|
509
471
|
instances.append(model)
|
|
510
472
|
|
|
511
|
-
if cls.
|
|
473
|
+
if cls.should_refresh():
|
|
512
474
|
async with cls.Meta.redis.pipeline() as pipe:
|
|
513
475
|
for model in instances:
|
|
514
476
|
pipe.expire(model.key, cls.Meta.ttl)
|
|
@@ -529,24 +491,11 @@ class AtomicRedisModel(BaseModel):
|
|
|
529
491
|
pipe.expire(model.key, cls.Meta.ttl)
|
|
530
492
|
await pipe.execute()
|
|
531
493
|
|
|
532
|
-
@classmethod
|
|
533
|
-
@deprecated(
|
|
534
|
-
"function delete is deprecated and will be removed in rapyer 1.2.0, use adelete instead"
|
|
535
|
-
)
|
|
536
|
-
async def delete_by_key(cls, key: str) -> bool:
|
|
537
|
-
return await cls.adelete_by_key(key) # pragma: no cover
|
|
538
|
-
|
|
539
494
|
@classmethod
|
|
540
495
|
async def adelete_by_key(cls, key: str) -> bool:
|
|
541
496
|
client = _context_var.get() or cls.Meta.redis
|
|
542
497
|
return await client.delete(key) == 1
|
|
543
498
|
|
|
544
|
-
@deprecated(
|
|
545
|
-
"function delete is deprecated and will be removed in rapyer 1.2.0, use adelete instead"
|
|
546
|
-
)
|
|
547
|
-
async def delete(self):
|
|
548
|
-
return await self.adelete() # pragma: no cover
|
|
549
|
-
|
|
550
499
|
async def adelete(self):
|
|
551
500
|
if self.is_inner_model():
|
|
552
501
|
raise RuntimeError("Can only delete from inner model")
|
|
@@ -558,19 +507,6 @@ class AtomicRedisModel(BaseModel):
|
|
|
558
507
|
*[model if isinstance(model, str) else model.key for model in args]
|
|
559
508
|
)
|
|
560
509
|
|
|
561
|
-
@classmethod
|
|
562
|
-
@contextlib.asynccontextmanager
|
|
563
|
-
@deprecated(
|
|
564
|
-
"lock_from_key function is deprecated and will be removed in rapyer 1.2.0, use alock_from_key instead"
|
|
565
|
-
)
|
|
566
|
-
async def lock_from_key(
|
|
567
|
-
cls, key: str, action: str = "default", save_at_end: bool = False
|
|
568
|
-
) -> AbstractAsyncContextManager[Self]:
|
|
569
|
-
async with cls.alock_from_key( # pragma: no cover
|
|
570
|
-
key, action, save_at_end # pragma: no cover
|
|
571
|
-
) as redis_model: # pragma: no cover
|
|
572
|
-
yield redis_model # pragma: no cover
|
|
573
|
-
|
|
574
510
|
@classmethod
|
|
575
511
|
@contextlib.asynccontextmanager
|
|
576
512
|
async def alock_from_key(
|
|
@@ -582,18 +518,6 @@ class AtomicRedisModel(BaseModel):
|
|
|
582
518
|
if save_at_end:
|
|
583
519
|
await redis_model.asave()
|
|
584
520
|
|
|
585
|
-
@contextlib.asynccontextmanager
|
|
586
|
-
@deprecated(
|
|
587
|
-
"lock function is deprecated and will be removed in rapyer 1.2.0, use alock instead"
|
|
588
|
-
)
|
|
589
|
-
async def lock(
|
|
590
|
-
self, action: str = "default", save_at_end: bool = False
|
|
591
|
-
) -> AbstractAsyncContextManager[Self]:
|
|
592
|
-
async with self.alock_from_key( # pragma: no cover
|
|
593
|
-
self.key, action, save_at_end # pragma: no cover
|
|
594
|
-
) as redis_model: # pragma: no cover
|
|
595
|
-
yield redis_model # pragma: no cover
|
|
596
|
-
|
|
597
521
|
@contextlib.asynccontextmanager
|
|
598
522
|
async def alock(
|
|
599
523
|
self, action: str = "default", save_at_end: bool = False
|
|
@@ -605,21 +529,9 @@ class AtomicRedisModel(BaseModel):
|
|
|
605
529
|
self.__dict__.update(unset_fields)
|
|
606
530
|
yield redis_model
|
|
607
531
|
|
|
608
|
-
@contextlib.asynccontextmanager
|
|
609
|
-
@deprecated(
|
|
610
|
-
"pipeline function is deprecated and will be removed in rapyer 1.2.0, use apipeline instead"
|
|
611
|
-
)
|
|
612
|
-
async def pipeline(
|
|
613
|
-
self, ignore_if_deleted: bool = False
|
|
614
|
-
) -> AbstractAsyncContextManager[Self]:
|
|
615
|
-
async with self.apipeline( # pragma: no cover
|
|
616
|
-
ignore_if_deleted=ignore_if_deleted # pragma: no cover
|
|
617
|
-
) as redis_model: # pragma: no cover
|
|
618
|
-
yield redis_model # pragma: no cover
|
|
619
|
-
|
|
620
532
|
@contextlib.asynccontextmanager
|
|
621
533
|
async def apipeline(
|
|
622
|
-
self,
|
|
534
|
+
self, ignore_redis_error: bool = False
|
|
623
535
|
) -> AbstractAsyncContextManager[Self]:
|
|
624
536
|
async with self.Meta.redis.pipeline(transaction=True) as pipe:
|
|
625
537
|
try:
|
|
@@ -629,12 +541,12 @@ class AtomicRedisModel(BaseModel):
|
|
|
629
541
|
}
|
|
630
542
|
self.__dict__.update(unset_fields)
|
|
631
543
|
except (TypeError, KeyNotFound):
|
|
632
|
-
if
|
|
544
|
+
if ignore_redis_error:
|
|
633
545
|
redis_model = self
|
|
634
546
|
else:
|
|
635
547
|
raise
|
|
636
548
|
_context_var.set(pipe)
|
|
637
|
-
_context_xx_pipe.set(
|
|
549
|
+
_context_xx_pipe.set(ignore_redis_error)
|
|
638
550
|
yield redis_model
|
|
639
551
|
commands_backup = list(pipe.command_stack)
|
|
640
552
|
noscript_on_first_attempt = False
|
|
@@ -647,10 +559,10 @@ class AtomicRedisModel(BaseModel):
|
|
|
647
559
|
except NoScriptError:
|
|
648
560
|
noscript_on_first_attempt = True
|
|
649
561
|
except ResponseError as exc:
|
|
650
|
-
if
|
|
562
|
+
if ignore_redis_error:
|
|
651
563
|
logger.warning(
|
|
652
564
|
"Swallowed ResponseError during pipeline.execute() with "
|
|
653
|
-
"
|
|
565
|
+
"ignore_redis_error=True for key %r: %s",
|
|
654
566
|
getattr(self, "key", None),
|
|
655
567
|
exc,
|
|
656
568
|
)
|
|
@@ -693,6 +605,16 @@ class AtomicRedisModel(BaseModel):
|
|
|
693
605
|
if isinstance(attr, RedisType):
|
|
694
606
|
attr._base_model_link = self
|
|
695
607
|
|
|
608
|
+
pipeline = _context_var.get()
|
|
609
|
+
if pipeline is not None:
|
|
610
|
+
serialized = self.model_dump(
|
|
611
|
+
mode="json",
|
|
612
|
+
context={REDIS_DUMP_FLAG_NAME: True},
|
|
613
|
+
include={name},
|
|
614
|
+
)
|
|
615
|
+
json_path = f"{self.json_path}.{name}"
|
|
616
|
+
pipeline.json().set(self.key, json_path, serialized[name])
|
|
617
|
+
|
|
696
618
|
def __eq__(self, other):
|
|
697
619
|
if not isinstance(other, BaseModel):
|
|
698
620
|
return False
|
|
@@ -720,13 +642,6 @@ class AtomicRedisModel(BaseModel):
|
|
|
720
642
|
REDIS_MODELS: list[type[AtomicRedisModel]] = []
|
|
721
643
|
|
|
722
644
|
|
|
723
|
-
@deprecated(
|
|
724
|
-
"get function is deprecated and will be removed in rapyer 1.2.0, use aget instead"
|
|
725
|
-
)
|
|
726
|
-
async def get(redis_key: str) -> AtomicRedisModel:
|
|
727
|
-
return await aget(redis_key) # pragma: no cover
|
|
728
|
-
|
|
729
|
-
|
|
730
645
|
async def aget(redis_key: str) -> AtomicRedisModel:
|
|
731
646
|
redis_model_mapping = {klass.__name__: klass for klass in REDIS_MODELS}
|
|
732
647
|
class_name = redis_key.split(":")[0]
|
|
@@ -736,6 +651,52 @@ async def aget(redis_key: str) -> AtomicRedisModel:
|
|
|
736
651
|
return await klass.aget(redis_key)
|
|
737
652
|
|
|
738
653
|
|
|
654
|
+
async def afind(*redis_keys: str, skip_missing: bool = False) -> list[AtomicRedisModel]:
|
|
655
|
+
if not redis_keys:
|
|
656
|
+
return []
|
|
657
|
+
|
|
658
|
+
redis_model_mapping = {klass.__name__: klass for klass in REDIS_MODELS}
|
|
659
|
+
|
|
660
|
+
key_to_class: dict[str, type[AtomicRedisModel]] = {}
|
|
661
|
+
for key in redis_keys:
|
|
662
|
+
class_name = key.split(":")[0]
|
|
663
|
+
if class_name not in redis_model_mapping:
|
|
664
|
+
raise RapyerModelDoesntExistError(
|
|
665
|
+
class_name, f"Unknown model class: {class_name}"
|
|
666
|
+
)
|
|
667
|
+
key_to_class[key] = redis_model_mapping[class_name]
|
|
668
|
+
|
|
669
|
+
models_data = await AtomicRedisModel.Meta.redis.json().mget(
|
|
670
|
+
keys=redis_keys, path="$"
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
instances = []
|
|
674
|
+
instances_by_class: dict[type[AtomicRedisModel], list[AtomicRedisModel]] = {}
|
|
675
|
+
|
|
676
|
+
for data, key in zip(models_data, redis_keys):
|
|
677
|
+
if data is None:
|
|
678
|
+
if not skip_missing:
|
|
679
|
+
raise KeyNotFound(f"{key} is missing in redis")
|
|
680
|
+
continue
|
|
681
|
+
klass = key_to_class[key]
|
|
682
|
+
if not klass.Meta.is_fake_redis:
|
|
683
|
+
data = data[0]
|
|
684
|
+
model = klass.create_redis_model(data, key)
|
|
685
|
+
if model is None:
|
|
686
|
+
continue
|
|
687
|
+
instances.append(model)
|
|
688
|
+
instances_by_class.setdefault(klass, []).append(model)
|
|
689
|
+
|
|
690
|
+
async with AtomicRedisModel.Meta.redis.pipeline() as pipe:
|
|
691
|
+
for klass, class_instances in instances_by_class.items():
|
|
692
|
+
if klass.should_refresh():
|
|
693
|
+
for model in class_instances:
|
|
694
|
+
pipe.expire(model.key, klass.Meta.ttl)
|
|
695
|
+
await pipe.execute()
|
|
696
|
+
|
|
697
|
+
return instances
|
|
698
|
+
|
|
699
|
+
|
|
739
700
|
def find_redis_models() -> list[type[AtomicRedisModel]]:
|
|
740
701
|
return REDIS_MODELS
|
|
741
702
|
|
|
@@ -744,6 +705,8 @@ async def ainsert(*models: Unpack[AtomicRedisModel]) -> list[AtomicRedisModel]:
|
|
|
744
705
|
async with AtomicRedisModel.Meta.redis.pipeline() as pipe:
|
|
745
706
|
for model in models:
|
|
746
707
|
pipe.json().set(model.key, model.json_path, model.redis_dump())
|
|
708
|
+
if model.Meta.ttl is not None:
|
|
709
|
+
pipe.expire(model.key, model.Meta.ttl)
|
|
747
710
|
await pipe.execute()
|
|
748
711
|
return models
|
|
749
712
|
|
rapyer/config.py
CHANGED
|
@@ -28,3 +28,5 @@ class RedisConfig:
|
|
|
28
28
|
safe_load_all: bool = False
|
|
29
29
|
# If True, use JSON serialization for fields that support it instead of pickle
|
|
30
30
|
prefer_normal_json_dump: bool = False
|
|
31
|
+
# Set to True when using FakeRedis to normalize JSON responses
|
|
32
|
+
is_fake_redis: bool = False
|
rapyer/errors/__init__.py
CHANGED
|
@@ -5,6 +5,9 @@ from rapyer.errors.base import (
|
|
|
5
5
|
RapyerError,
|
|
6
6
|
ScriptsNotInitializedError,
|
|
7
7
|
UnsupportedIndexedFieldError,
|
|
8
|
+
RapyerModelDoesntExistError,
|
|
9
|
+
CantSerializeRedisValueError,
|
|
10
|
+
KeyNotFound,
|
|
8
11
|
)
|
|
9
12
|
|
|
10
13
|
__all__ = [
|
|
@@ -12,6 +15,9 @@ __all__ = [
|
|
|
12
15
|
"FindError",
|
|
13
16
|
"PersistentNoScriptError",
|
|
14
17
|
"RapyerError",
|
|
18
|
+
"KeyNotFound",
|
|
15
19
|
"ScriptsNotInitializedError",
|
|
16
20
|
"UnsupportedIndexedFieldError",
|
|
21
|
+
"RapyerModelDoesntExistError",
|
|
22
|
+
"CantSerializeRedisValueError",
|
|
17
23
|
]
|
rapyer/errors/base.py
CHANGED
|
@@ -10,6 +10,14 @@ class KeyNotFound(RapyerError):
|
|
|
10
10
|
pass
|
|
11
11
|
|
|
12
12
|
|
|
13
|
+
class RapyerModelDoesntExistError(RapyerError):
|
|
14
|
+
"""Raised when a model doesn't exist."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, model_name: str, *args):
|
|
17
|
+
super().__init__(*args)
|
|
18
|
+
self.model_name = model_name
|
|
19
|
+
|
|
20
|
+
|
|
13
21
|
class FindError(RapyerError):
|
|
14
22
|
"""Raised when a model cannot be found."""
|
|
15
23
|
|
rapyer/fields/expression.py
CHANGED
rapyer/init.py
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
3
|
import redis.asyncio as redis_async
|
|
4
|
-
from rapyer.base import REDIS_MODELS
|
|
5
|
-
from rapyer.scripts import register_scripts
|
|
6
4
|
from redis import ResponseError
|
|
7
5
|
from redis.asyncio.client import Redis
|
|
8
6
|
|
|
7
|
+
from rapyer.base import REDIS_MODELS
|
|
8
|
+
from rapyer.scripts import register_scripts
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def is_fakeredis(client) -> bool:
|
|
12
|
+
return "fakeredis" in type(client).__module__
|
|
13
|
+
|
|
9
14
|
|
|
10
15
|
async def init_rapyer(
|
|
11
16
|
redis: str | Redis = None,
|
|
@@ -24,12 +29,14 @@ async def init_rapyer(
|
|
|
24
29
|
if isinstance(redis, str):
|
|
25
30
|
redis = redis_async.from_url(redis, decode_responses=True, max_connections=20)
|
|
26
31
|
|
|
32
|
+
is_fake_redis = is_fakeredis(redis)
|
|
27
33
|
if redis is not None:
|
|
28
|
-
await register_scripts(redis)
|
|
34
|
+
await register_scripts(redis, is_fake_redis)
|
|
29
35
|
|
|
30
36
|
for model in REDIS_MODELS:
|
|
31
37
|
if redis is not None:
|
|
32
38
|
model.Meta.redis = redis
|
|
39
|
+
model.Meta.is_fake_redis = is_fake_redis
|
|
33
40
|
if ttl is not None:
|
|
34
41
|
model.Meta.ttl = ttl
|
|
35
42
|
if prefer_normal_json_dump is not None:
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from rapyer.scripts.constants import (
|
|
2
|
+
DATETIME_ADD_SCRIPT_NAME,
|
|
3
|
+
DICT_POP_SCRIPT_NAME,
|
|
4
|
+
DICT_POPITEM_SCRIPT_NAME,
|
|
5
|
+
NUM_FLOORDIV_SCRIPT_NAME,
|
|
6
|
+
NUM_MOD_SCRIPT_NAME,
|
|
7
|
+
NUM_MUL_SCRIPT_NAME,
|
|
8
|
+
NUM_POW_FLOAT_SCRIPT_NAME,
|
|
9
|
+
NUM_POW_SCRIPT_NAME,
|
|
10
|
+
NUM_TRUEDIV_SCRIPT_NAME,
|
|
11
|
+
REMOVE_RANGE_SCRIPT_NAME,
|
|
12
|
+
STR_APPEND_SCRIPT_NAME,
|
|
13
|
+
STR_MUL_SCRIPT_NAME,
|
|
14
|
+
)
|
|
15
|
+
from rapyer.scripts.registry import (
|
|
16
|
+
_REGISTERED_SCRIPT_SHAS,
|
|
17
|
+
arun_sha,
|
|
18
|
+
get_scripts,
|
|
19
|
+
get_scripts_fakeredis,
|
|
20
|
+
handle_noscript_error,
|
|
21
|
+
register_scripts,
|
|
22
|
+
run_sha,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
SCRIPTS = get_scripts()
|
|
26
|
+
SCRIPTS_FAKEREDIS = get_scripts_fakeredis()
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"DATETIME_ADD_SCRIPT_NAME",
|
|
30
|
+
"DICT_POP_SCRIPT_NAME",
|
|
31
|
+
"DICT_POPITEM_SCRIPT_NAME",
|
|
32
|
+
"NUM_FLOORDIV_SCRIPT_NAME",
|
|
33
|
+
"NUM_MOD_SCRIPT_NAME",
|
|
34
|
+
"NUM_MUL_SCRIPT_NAME",
|
|
35
|
+
"NUM_POW_FLOAT_SCRIPT_NAME",
|
|
36
|
+
"NUM_POW_SCRIPT_NAME",
|
|
37
|
+
"NUM_TRUEDIV_SCRIPT_NAME",
|
|
38
|
+
"REMOVE_RANGE_SCRIPT_NAME",
|
|
39
|
+
"SCRIPTS",
|
|
40
|
+
"SCRIPTS_FAKEREDIS",
|
|
41
|
+
"STR_APPEND_SCRIPT_NAME",
|
|
42
|
+
"STR_MUL_SCRIPT_NAME",
|
|
43
|
+
"arun_sha",
|
|
44
|
+
"handle_noscript_error",
|
|
45
|
+
"register_scripts",
|
|
46
|
+
"run_sha",
|
|
47
|
+
]
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
REMOVE_RANGE_SCRIPT_NAME = "remove_range"
|
|
2
|
+
NUM_MUL_SCRIPT_NAME = "num_mul"
|
|
3
|
+
NUM_FLOORDIV_SCRIPT_NAME = "num_floordiv"
|
|
4
|
+
NUM_MOD_SCRIPT_NAME = "num_mod"
|
|
5
|
+
NUM_POW_SCRIPT_NAME = "num_pow"
|
|
6
|
+
NUM_POW_FLOAT_SCRIPT_NAME = "num_pow_float"
|
|
7
|
+
NUM_TRUEDIV_SCRIPT_NAME = "num_truediv"
|
|
8
|
+
STR_APPEND_SCRIPT_NAME = "str_append"
|
|
9
|
+
STR_MUL_SCRIPT_NAME = "str_mul"
|
|
10
|
+
DATETIME_ADD_SCRIPT_NAME = "datetime_add"
|
|
11
|
+
DICT_POP_SCRIPT_NAME = "dict_pop"
|
|
12
|
+
DICT_POPITEM_SCRIPT_NAME = "dict_popitem"
|
rapyer/scripts/loader.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from functools import lru_cache
|
|
2
|
+
from importlib import resources
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
VARIANTS = {
|
|
6
|
+
"redis": {
|
|
7
|
+
"EXTRACT_ARRAY": "local arr = cjson.decode(arr_json)[1]",
|
|
8
|
+
"EXTRACT_VALUE": "local value = tonumber(cjson.decode(current_json)[1])",
|
|
9
|
+
"EXTRACT_STR": "local value = cjson.decode(current_json)[1]",
|
|
10
|
+
"EXTRACT_DATETIME": "local value = cjson.decode(current_json)[1]",
|
|
11
|
+
"DICT_EXTRACT_VALUE": "local extracted = cjson.decode(value)[1]",
|
|
12
|
+
"DICT_EXTRACT_POPITEM": """local parsed = cjson.decode(value)
|
|
13
|
+
if type(parsed) == 'table' then
|
|
14
|
+
for _, v in pairs(parsed) do
|
|
15
|
+
extracted = v
|
|
16
|
+
break
|
|
17
|
+
end
|
|
18
|
+
else
|
|
19
|
+
extracted = parsed
|
|
20
|
+
end""",
|
|
21
|
+
},
|
|
22
|
+
"fakeredis": {
|
|
23
|
+
"EXTRACT_ARRAY": "local arr = cjson.decode(arr_json)",
|
|
24
|
+
"EXTRACT_VALUE": "local value = tonumber(cjson.decode(current_json)[1])",
|
|
25
|
+
"EXTRACT_STR": "local value = cjson.decode(current_json)[1]",
|
|
26
|
+
"EXTRACT_DATETIME": "local value = cjson.decode(current_json)[1]",
|
|
27
|
+
"DICT_EXTRACT_VALUE": "local extracted = cjson.decode(value)[1]",
|
|
28
|
+
"DICT_EXTRACT_POPITEM": """local parsed = cjson.decode(value)
|
|
29
|
+
if type(parsed) == 'table' then
|
|
30
|
+
for _, v in pairs(parsed) do
|
|
31
|
+
extracted = v
|
|
32
|
+
break
|
|
33
|
+
end
|
|
34
|
+
else
|
|
35
|
+
extracted = parsed
|
|
36
|
+
end""",
|
|
37
|
+
},
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@lru_cache(maxsize=None)
|
|
42
|
+
def _load_template(category: str, name: str) -> str:
|
|
43
|
+
package = f"rapyer.scripts.lua.{category}"
|
|
44
|
+
filename = f"{name}.lua"
|
|
45
|
+
return resources.files(package).joinpath(filename).read_text()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def load_script(category: str, name: str, variant: str = "redis") -> str:
|
|
49
|
+
template = _load_template(category, name)
|
|
50
|
+
replacements = VARIANTS[variant]
|
|
51
|
+
result = template
|
|
52
|
+
for placeholder, value in replacements.items():
|
|
53
|
+
result = result.replace(f"--[[{placeholder}]]", value)
|
|
54
|
+
return result
|
|
File without changes
|
|
File without changes
|