cledar-sdk 1.2.1__py3-none-any.whl → 1.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cledar_sdk-1.2.1.dist-info → cledar_sdk-1.4.0.dist-info}/METADATA +1 -1
- {cledar_sdk-1.2.1.dist-info → cledar_sdk-1.4.0.dist-info}/RECORD +15 -12
- {cledar_sdk-1.2.1.dist-info → cledar_sdk-1.4.0.dist-info}/WHEEL +1 -1
- kafka_service/__init__.py +8 -1
- kafka_service/clients/consumer.py +1 -10
- kafka_service/clients/producer.py +1 -8
- kafka_service/config/schemas.py +72 -0
- kafka_service/tests/unit/test_config_validation.py +85 -1
- redis_service/README.md +156 -16
- redis_service/__init__.py +15 -0
- redis_service/async_example.py +111 -0
- redis_service/redis.py +247 -0
- redis_service/tests/test_async_integration_redis.py +162 -0
- redis_service/tests/test_async_redis_service.py +384 -0
- {cledar_sdk-1.2.1.dist-info → cledar_sdk-1.4.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -3,13 +3,13 @@ common_logging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
3
3
|
common_logging/universal_plaintext_formatter.py,sha256=ExuXwfZmc_4tp3RIsoe_ByYpnTN0iQBOi101FB8hksc,3309
|
|
4
4
|
common_logging/tests/test_universal_plaintext_formatter.py,sha256=bY6vvpra81_YXN2MLVjNuA8Iu7kOaZH0lmQMsohjRPI,8560
|
|
5
5
|
kafka_service/README.md,sha256=3lKulsfEUs6qaKExlpPkxT_X1NYBAHraiqQRdvkQ48I,7021
|
|
6
|
-
kafka_service/__init__.py,sha256=
|
|
6
|
+
kafka_service/__init__.py,sha256=YejBYb9sS5d0VLuYg48vqKWTWSTwlEnsGF_2Z3guXAQ,1131
|
|
7
7
|
kafka_service/exceptions.py,sha256=LJ-mUYCwoQsDM-bL4ms9ZCiWIOw5kt1KStbN04Mavl0,501
|
|
8
8
|
kafka_service/logger.py,sha256=jdZSJnuPpUXEwG5eLKlQMLkVxaKwuma_o9udbOAjXo0,60
|
|
9
9
|
kafka_service/clients/base.py,sha256=LhiCF9vNxn-kCWld4Xfzy1lYCUj7PbhJhNURE7wIx70,3692
|
|
10
|
-
kafka_service/clients/consumer.py,sha256=
|
|
11
|
-
kafka_service/clients/producer.py,sha256=
|
|
12
|
-
kafka_service/config/schemas.py,sha256=
|
|
10
|
+
kafka_service/clients/consumer.py,sha256=m0u-Q2HXLk7pD4z2ucwaQxavW3VATbvVD1fDTSAi0tA,3590
|
|
11
|
+
kafka_service/clients/producer.py,sha256=oJMZ9zg7n-DFWrySu-xveQ70ZXZEQkuXE0ewMeLLKoo,2605
|
|
12
|
+
kafka_service/config/schemas.py,sha256=ToAv2d3F7mMpuePBWsC12jugdMWLPQMoNCXub0VIbgE,6550
|
|
13
13
|
kafka_service/handlers/dead_letter.py,sha256=Y4krNONrMayaYsvFZrXh-pRK8kfyWibEYrUyDLy-XH8,2552
|
|
14
14
|
kafka_service/handlers/parser.py,sha256=BsEUFI5bDBQzf9f9s7XClk_TfEXveOG5yCTT4sOibPI,1475
|
|
15
15
|
kafka_service/models/input.py,sha256=VEf9DDbXew4tJSHP7cEbXWW3k6s_1gN8QGcqs5LmxuU,254
|
|
@@ -27,7 +27,7 @@ kafka_service/tests/integration/test_producer_consumer_interaction.py,sha256=7Z6
|
|
|
27
27
|
kafka_service/tests/integration/test_producer_integration.py,sha256=0NXNB5gN0hPSXj5j2oP9FlhAXrNVpNg5N_1DkGOHUy0,6438
|
|
28
28
|
kafka_service/tests/unit/__init__.py,sha256=uiWL_3JORErRzHi12hW0-kFU36zdkl3MBg4_qtH6TM0,36
|
|
29
29
|
kafka_service/tests/unit/test_base_kafka_client.py,sha256=6O31wIuoKkb-EEqD-a5PjB-zbM0h5vTFz1VGr9V3KVQ,13321
|
|
30
|
-
kafka_service/tests/unit/test_config_validation.py,sha256=
|
|
30
|
+
kafka_service/tests/unit/test_config_validation.py,sha256=KpMAI_5s-XWOW_bNT9rTie61eoet4kkB7YTFd9W2hOE,19971
|
|
31
31
|
kafka_service/tests/unit/test_dead_letter_handler.py,sha256=qsJZKXm_fBZUnaDwBAF7Bj9aS-y7Uxc-e7u6pPQLTgc,13905
|
|
32
32
|
kafka_service/tests/unit/test_error_handling.py,sha256=h91hwG02DwGEZrziSv0mLshQFK98p-P38g4gWPHuyyY,22828
|
|
33
33
|
kafka_service/tests/unit/test_input_parser.py,sha256=_MIWkvhYRZR_7lg9Jo_MD3ckCTGrMrSyeR_7F5wrHSw,11556
|
|
@@ -52,14 +52,17 @@ nonce_service/__init__.py,sha256=Rh6JWML_ncfb_t_mVl7PIKOXpELRdfunMsFWAtbt5EE,68
|
|
|
52
52
|
nonce_service/nonce_service.py,sha256=6RhA2eEztH_pNdjkBI5eqmRh14I96NjgVYy2v_z7vdM,1100
|
|
53
53
|
nonce_service/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
54
54
|
nonce_service/tests/test_nonce_service.py,sha256=sWzCBXghZNDOnSo8NQB6tyJJ95g2tZGRPlThFqD2P7g,4047
|
|
55
|
-
redis_service/README.md,sha256=
|
|
56
|
-
redis_service/__init__.py,sha256=
|
|
55
|
+
redis_service/README.md,sha256=rsIieAu4TwjoeIbHG30x3t3TyR9P9Ulcy69gdfG_jRg,14632
|
|
56
|
+
redis_service/__init__.py,sha256=SgTAyteZ3pPV8R608-EQ84RmytUfSQaPGl0UJxLnGL0,250
|
|
57
|
+
redis_service/async_example.py,sha256=m8dkSspJK8yti9U_Q3_Sc9XW1aq3RYL4EzBdLx5_u84,2949
|
|
57
58
|
redis_service/example.py,sha256=OApl0JU59viBUZORLxlYBhVMdCc5QvOBdOjscxbm_aQ,1091
|
|
58
59
|
redis_service/exceptions.py,sha256=vvD7SO0xHutSLvUf0ttMo7C6OeaVi8f3bxMservsSVI,737
|
|
59
60
|
redis_service/logger.py,sha256=OBOTx6zk_6wkpB2N_FRV7gXR3xy4dpy4iX2B0oFfZ90,60
|
|
60
61
|
redis_service/model.py,sha256=ykW_KHygNHhfHPvP2RyJj0g_WENXxJEP2HvGy7Yb4uo,174
|
|
61
|
-
redis_service/redis.py,sha256=
|
|
62
|
+
redis_service/redis.py,sha256=DuWaXMU5RhAW2Rf9T-Hg_1FIrQqkPin1vTXQMJJ6Sno,19817
|
|
62
63
|
redis_service/redis_config_store.py,sha256=EFhyvg_Eklrh2tc5dtFpe6nnjMDkTqYT_egclhf0KaI,8919
|
|
64
|
+
redis_service/tests/test_async_integration_redis.py,sha256=I-rWvOm9Cnpl8AZcrPb58tTbkq_YKh3WR2bwXB-EJRM,5005
|
|
65
|
+
redis_service/tests/test_async_redis_service.py,sha256=DgzBZscXu_FiV6QIZT2AHQ0rk_Zwzu2HmsifeDhArRU,12501
|
|
63
66
|
redis_service/tests/test_integration_redis.py,sha256=O3z5EgkeB6Sd9C1S3rRMW916bybDCAHb-DgTFZGLTqI,3345
|
|
64
67
|
redis_service/tests/test_redis_service.py,sha256=n3FIYBk9SGzpDkRWnPj-Mi3NKYtDEpbwgjpjb_JwjF4,10252
|
|
65
68
|
storage_service/README.md,sha256=Rkew1joVijHVEfyXS9C7dDw55rQ4HYRbHkRYlZiREak,15655
|
|
@@ -74,7 +77,7 @@ storage_service/tests/test_integration_filesystem.py,sha256=-H3Skc_geYIjXW1si-8u
|
|
|
74
77
|
storage_service/tests/test_integration_s3.py,sha256=Ivg_52LXibqVGMS-53z4zda_Yh4u6FO8WplUUu5WWBc,13614
|
|
75
78
|
storage_service/tests/test_local.py,sha256=3CgtxQ_lBBaPR4t9Ip0i7T98scrTOCdkmHYMVansNCc,11256
|
|
76
79
|
storage_service/tests/test_s3.py,sha256=zAppsvVCeLx_NN1tQfqHo57mONEjeFDmiwyecrS3ZgQ,16355
|
|
77
|
-
cledar_sdk-1.
|
|
78
|
-
cledar_sdk-1.
|
|
79
|
-
cledar_sdk-1.
|
|
80
|
-
cledar_sdk-1.
|
|
80
|
+
cledar_sdk-1.4.0.dist-info/METADATA,sha256=KY0mpTdaySyoD8xe8RDSEM0IZRFrF2rLCwNAy1SBcjg,6752
|
|
81
|
+
cledar_sdk-1.4.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
82
|
+
cledar_sdk-1.4.0.dist-info/licenses/LICENSE,sha256=Pz2eACSxkhsGfW9_iN60pgy-enjnbGTj8df8O3ebnQQ,16726
|
|
83
|
+
cledar_sdk-1.4.0.dist-info/RECORD,,
|
kafka_service/__init__.py
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
from .clients.base import BaseKafkaClient
|
|
2
2
|
from .clients.consumer import KafkaConsumer
|
|
3
3
|
from .clients.producer import KafkaProducer
|
|
4
|
-
from .config.schemas import
|
|
4
|
+
from .config.schemas import (
|
|
5
|
+
KafkaConsumerConfig,
|
|
6
|
+
KafkaProducerConfig,
|
|
7
|
+
KafkaSaslMechanism,
|
|
8
|
+
KafkaSecurityProtocol,
|
|
9
|
+
)
|
|
5
10
|
from .exceptions import (
|
|
6
11
|
KafkaConnectionError,
|
|
7
12
|
KafkaConsumerError,
|
|
@@ -26,6 +31,8 @@ __all__ = [
|
|
|
26
31
|
"KafkaMessage",
|
|
27
32
|
"KafkaProducerConfig",
|
|
28
33
|
"KafkaConsumerConfig",
|
|
34
|
+
"KafkaSecurityProtocol",
|
|
35
|
+
"KafkaSaslMechanism",
|
|
29
36
|
"KafkaConnectionError",
|
|
30
37
|
"KafkaConsumerNotConnectedError",
|
|
31
38
|
"KafkaProducerNotConnectedError",
|
|
@@ -20,16 +20,7 @@ class KafkaConsumer(BaseKafkaClient):
|
|
|
20
20
|
client: Consumer | None = None
|
|
21
21
|
|
|
22
22
|
def connect(self) -> None:
|
|
23
|
-
self.client = Consumer(
|
|
24
|
-
{
|
|
25
|
-
"bootstrap.servers": self.config.kafka_servers,
|
|
26
|
-
"enable.auto.commit": False,
|
|
27
|
-
"enable.partition.eof": False,
|
|
28
|
-
"auto.commit.interval.ms": self.config.kafka_auto_commit_interval_ms,
|
|
29
|
-
"auto.offset.reset": self.config.kafka_offset,
|
|
30
|
-
"group.id": self.config.kafka_group_id,
|
|
31
|
-
}
|
|
32
|
-
)
|
|
23
|
+
self.client = Consumer(self.config.to_kafka_config())
|
|
33
24
|
self.check_connection()
|
|
34
25
|
logger.info(
|
|
35
26
|
"Connected KafkaConsumer to Kafka servers.",
|
|
@@ -17,14 +17,7 @@ class KafkaProducer(BaseKafkaClient):
|
|
|
17
17
|
client: Producer | None = None
|
|
18
18
|
|
|
19
19
|
def connect(self) -> None:
|
|
20
|
-
self.client = Producer(
|
|
21
|
-
{
|
|
22
|
-
"bootstrap.servers": self.config.kafka_servers,
|
|
23
|
-
"client.id": self.config.kafka_group_id,
|
|
24
|
-
"compression.type": self.config.compression_type,
|
|
25
|
-
"partitioner": self.config.kafka_partitioner,
|
|
26
|
-
}
|
|
27
|
-
)
|
|
20
|
+
self.client = Producer(self.config.to_kafka_config())
|
|
28
21
|
self.check_connection()
|
|
29
22
|
logger.info(
|
|
30
23
|
"Connected Producer to Kafka servers.",
|
kafka_service/config/schemas.py
CHANGED
|
@@ -1,7 +1,27 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
|
|
1
3
|
from pydantic import field_validator
|
|
2
4
|
from pydantic.dataclasses import dataclass
|
|
3
5
|
|
|
4
6
|
|
|
7
|
+
class KafkaSecurityProtocol(str, Enum):
|
|
8
|
+
"""Supported Kafka security protocols."""
|
|
9
|
+
|
|
10
|
+
PLAINTEXT = "PLAINTEXT"
|
|
11
|
+
SSL = "SSL"
|
|
12
|
+
SASL_PLAINTEXT = "SASL_PLAINTEXT"
|
|
13
|
+
SASL_SSL = "SASL_SSL"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class KafkaSaslMechanism(str, Enum):
|
|
17
|
+
"""Supported Kafka SASL mechanisms."""
|
|
18
|
+
|
|
19
|
+
PLAIN = "PLAIN"
|
|
20
|
+
SCRAM_SHA_256 = "SCRAM-SHA-256"
|
|
21
|
+
SCRAM_SHA_512 = "SCRAM-SHA-512"
|
|
22
|
+
GSSAPI = "GSSAPI"
|
|
23
|
+
|
|
24
|
+
|
|
5
25
|
def _validate_kafka_servers(v: list[str] | str) -> list[str] | str:
|
|
6
26
|
"""Validate kafka_servers is not empty."""
|
|
7
27
|
if isinstance(v, str) and v.strip() == "":
|
|
@@ -36,6 +56,10 @@ class KafkaProducerConfig:
|
|
|
36
56
|
|
|
37
57
|
kafka_servers: list[str] | str
|
|
38
58
|
kafka_group_id: str
|
|
59
|
+
kafka_security_protocol: KafkaSecurityProtocol | None = None
|
|
60
|
+
kafka_sasl_mechanism: KafkaSaslMechanism | None = None
|
|
61
|
+
kafka_sasl_username: str | None = None
|
|
62
|
+
kafka_sasl_password: str | None = None
|
|
39
63
|
kafka_topic_prefix: str | None = None
|
|
40
64
|
kafka_block_buffer_time_sec: int = 10
|
|
41
65
|
kafka_connection_check_timeout_sec: int = 5
|
|
@@ -57,6 +81,27 @@ class KafkaProducerConfig:
|
|
|
57
81
|
def validate_positive_timeouts(cls, v: int) -> int:
|
|
58
82
|
return _validate_non_negative(v)
|
|
59
83
|
|
|
84
|
+
def to_kafka_config(self) -> dict[str, list[str] | str | None]:
|
|
85
|
+
"""Build Kafka producer configuration dictionary."""
|
|
86
|
+
config = {
|
|
87
|
+
"bootstrap.servers": self.kafka_servers,
|
|
88
|
+
"client.id": self.kafka_group_id,
|
|
89
|
+
"compression.type": self.compression_type,
|
|
90
|
+
"partitioner": self.kafka_partitioner,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
# Add SASL configuration if specified
|
|
94
|
+
if self.kafka_security_protocol:
|
|
95
|
+
config["security.protocol"] = self.kafka_security_protocol.value
|
|
96
|
+
if self.kafka_sasl_mechanism:
|
|
97
|
+
config["sasl.mechanism"] = self.kafka_sasl_mechanism.value
|
|
98
|
+
if self.kafka_sasl_username:
|
|
99
|
+
config["sasl.username"] = self.kafka_sasl_username
|
|
100
|
+
if self.kafka_sasl_password:
|
|
101
|
+
config["sasl.password"] = self.kafka_sasl_password
|
|
102
|
+
|
|
103
|
+
return config
|
|
104
|
+
|
|
60
105
|
|
|
61
106
|
@dataclass(frozen=True)
|
|
62
107
|
class KafkaConsumerConfig:
|
|
@@ -76,6 +121,10 @@ class KafkaConsumerConfig:
|
|
|
76
121
|
|
|
77
122
|
kafka_servers: list[str] | str
|
|
78
123
|
kafka_group_id: str
|
|
124
|
+
kafka_security_protocol: KafkaSecurityProtocol | None = None
|
|
125
|
+
kafka_sasl_mechanism: KafkaSaslMechanism | None = None
|
|
126
|
+
kafka_sasl_username: str | None = None
|
|
127
|
+
kafka_sasl_password: str | None = None
|
|
79
128
|
kafka_offset: str = "latest"
|
|
80
129
|
kafka_topic_prefix: str | None = None
|
|
81
130
|
kafka_block_consumer_time_sec: int = 2
|
|
@@ -104,3 +153,26 @@ class KafkaConsumerConfig:
|
|
|
104
153
|
@classmethod
|
|
105
154
|
def validate_positive_timeouts(cls, v: int) -> int:
|
|
106
155
|
return _validate_non_negative(v)
|
|
156
|
+
|
|
157
|
+
def to_kafka_config(self) -> dict[str, int | list[str] | str]:
|
|
158
|
+
"""Build Kafka consumer configuration dictionary."""
|
|
159
|
+
config = {
|
|
160
|
+
"bootstrap.servers": self.kafka_servers,
|
|
161
|
+
"enable.auto.commit": False,
|
|
162
|
+
"enable.partition.eof": False,
|
|
163
|
+
"auto.commit.interval.ms": self.kafka_auto_commit_interval_ms,
|
|
164
|
+
"auto.offset.reset": self.kafka_offset,
|
|
165
|
+
"group.id": self.kafka_group_id,
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
# Add SASL configuration if specified
|
|
169
|
+
if self.kafka_security_protocol:
|
|
170
|
+
config["security.protocol"] = self.kafka_security_protocol.value
|
|
171
|
+
if self.kafka_sasl_mechanism:
|
|
172
|
+
config["sasl.mechanism"] = self.kafka_sasl_mechanism.value
|
|
173
|
+
if self.kafka_sasl_username:
|
|
174
|
+
config["sasl.username"] = self.kafka_sasl_username
|
|
175
|
+
if self.kafka_sasl_password:
|
|
176
|
+
config["sasl.password"] = self.kafka_sasl_password
|
|
177
|
+
|
|
178
|
+
return config
|
|
@@ -8,7 +8,12 @@ from dataclasses import FrozenInstanceError
|
|
|
8
8
|
import pytest
|
|
9
9
|
from pydantic import ValidationError
|
|
10
10
|
|
|
11
|
-
from kafka_service.config.schemas import
|
|
11
|
+
from kafka_service.config.schemas import (
|
|
12
|
+
KafkaConsumerConfig,
|
|
13
|
+
KafkaProducerConfig,
|
|
14
|
+
KafkaSaslMechanism,
|
|
15
|
+
KafkaSecurityProtocol,
|
|
16
|
+
)
|
|
12
17
|
from kafka_service.models.message import KafkaMessage
|
|
13
18
|
|
|
14
19
|
|
|
@@ -523,3 +528,82 @@ def test_config_inequality() -> None:
|
|
|
523
528
|
)
|
|
524
529
|
|
|
525
530
|
assert config1 != config2
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
# SASL Configuration Tests
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def test_config_with_sasl_authentication() -> None:
|
|
537
|
+
"""Test creating configs with SASL authentication."""
|
|
538
|
+
producer_config = KafkaProducerConfig(
|
|
539
|
+
kafka_servers="localhost:9092",
|
|
540
|
+
kafka_group_id="test-group",
|
|
541
|
+
kafka_security_protocol=KafkaSecurityProtocol.SASL_PLAINTEXT,
|
|
542
|
+
kafka_sasl_mechanism=KafkaSaslMechanism.PLAIN,
|
|
543
|
+
kafka_sasl_username="test-user",
|
|
544
|
+
kafka_sasl_password="test-password",
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
consumer_config = KafkaConsumerConfig(
|
|
548
|
+
kafka_servers="localhost:9092",
|
|
549
|
+
kafka_group_id="test-group",
|
|
550
|
+
kafka_security_protocol=KafkaSecurityProtocol.SASL_SSL,
|
|
551
|
+
kafka_sasl_mechanism=KafkaSaslMechanism.SCRAM_SHA_256,
|
|
552
|
+
kafka_sasl_username="test-user",
|
|
553
|
+
kafka_sasl_password="test-password",
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
assert (
|
|
557
|
+
producer_config.kafka_security_protocol == KafkaSecurityProtocol.SASL_PLAINTEXT
|
|
558
|
+
)
|
|
559
|
+
assert producer_config.kafka_sasl_mechanism == KafkaSaslMechanism.PLAIN
|
|
560
|
+
assert consumer_config.kafka_security_protocol == KafkaSecurityProtocol.SASL_SSL
|
|
561
|
+
assert consumer_config.kafka_sasl_mechanism == KafkaSaslMechanism.SCRAM_SHA_256
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def test_config_invalid_sasl_values() -> None:
|
|
565
|
+
"""Test that invalid SASL values raise ValidationError."""
|
|
566
|
+
with pytest.raises(ValidationError):
|
|
567
|
+
KafkaProducerConfig(
|
|
568
|
+
kafka_servers="localhost:9092",
|
|
569
|
+
kafka_group_id="test-group",
|
|
570
|
+
kafka_security_protocol="INVALID", # type: ignore[arg-type]
|
|
571
|
+
)
|
|
572
|
+
|
|
573
|
+
with pytest.raises(ValidationError):
|
|
574
|
+
KafkaConsumerConfig(
|
|
575
|
+
kafka_servers="localhost:9092",
|
|
576
|
+
kafka_group_id="test-group",
|
|
577
|
+
kafka_sasl_mechanism="INVALID", # type: ignore[arg-type]
|
|
578
|
+
)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def test_to_kafka_config_with_and_without_sasl() -> None:
|
|
582
|
+
"""Test that to_kafka_config() correctly includes/excludes SASL parameters."""
|
|
583
|
+
# Without SASL
|
|
584
|
+
config_no_sasl = KafkaProducerConfig(
|
|
585
|
+
kafka_servers="localhost:9092",
|
|
586
|
+
kafka_group_id="test-group",
|
|
587
|
+
)
|
|
588
|
+
kafka_config_no_sasl = config_no_sasl.to_kafka_config()
|
|
589
|
+
|
|
590
|
+
assert "security.protocol" not in kafka_config_no_sasl
|
|
591
|
+
assert "sasl.mechanism" not in kafka_config_no_sasl
|
|
592
|
+
assert "sasl.username" not in kafka_config_no_sasl
|
|
593
|
+
assert "sasl.password" not in kafka_config_no_sasl
|
|
594
|
+
|
|
595
|
+
# With SASL
|
|
596
|
+
config_with_sasl = KafkaConsumerConfig(
|
|
597
|
+
kafka_servers="localhost:9092",
|
|
598
|
+
kafka_group_id="test-group",
|
|
599
|
+
kafka_security_protocol=KafkaSecurityProtocol.SASL_SSL,
|
|
600
|
+
kafka_sasl_mechanism=KafkaSaslMechanism.SCRAM_SHA_256,
|
|
601
|
+
kafka_sasl_username="user",
|
|
602
|
+
kafka_sasl_password="pass",
|
|
603
|
+
)
|
|
604
|
+
kafka_config_with_sasl = config_with_sasl.to_kafka_config()
|
|
605
|
+
|
|
606
|
+
assert kafka_config_with_sasl["security.protocol"] == "SASL_SSL"
|
|
607
|
+
assert kafka_config_with_sasl["sasl.mechanism"] == "SCRAM-SHA-256"
|
|
608
|
+
assert kafka_config_with_sasl["sasl.username"] == "user"
|
|
609
|
+
assert kafka_config_with_sasl["sasl.password"] == "pass"
|
redis_service/README.md
CHANGED
|
@@ -7,6 +7,7 @@ The `redis_service` package provides a typed, high-level interface over Redis fo
|
|
|
7
7
|
### Key Features
|
|
8
8
|
|
|
9
9
|
- **Typed API with Pydantic**: Validate JSON payloads into Pydantic models on read
|
|
10
|
+
- **Async/Sync Support**: Both `AsyncRedisService` (async/await) and `RedisService` (synchronous) available
|
|
10
11
|
- **Safe Serialization**: Custom JSON encoder for `Enum` (to lowercase names) and `datetime` (ISO 8601)
|
|
11
12
|
- **Ergonomic Helpers**: `get`, `get_raw`, `set`, `list_keys`, `mget`, `delete`
|
|
12
13
|
- **Bulk Reads**: `mget` returns a list with typed results, `None`, or `FailedValue` for per-key errors
|
|
@@ -21,6 +22,7 @@ The `redis_service` package provides a typed, high-level interface over Redis fo
|
|
|
21
22
|
- Reading and writing typed configuration objects
|
|
22
23
|
- Bulk retrieval of many keys while tolerating per-key failures
|
|
23
24
|
- Observing and reacting to configuration changes in near real-time
|
|
25
|
+
- Asynchronous I/O for high-performance applications (FastAPI, aiohttp, etc.)
|
|
24
26
|
|
|
25
27
|
## Installation
|
|
26
28
|
|
|
@@ -34,7 +36,9 @@ uv sync --all-groups
|
|
|
34
36
|
pip install -e .
|
|
35
37
|
```
|
|
36
38
|
|
|
37
|
-
## Usage
|
|
39
|
+
## Usage Examples
|
|
40
|
+
|
|
41
|
+
### Synchronous Usage
|
|
38
42
|
|
|
39
43
|
```python
|
|
40
44
|
from pydantic import BaseModel
|
|
@@ -78,6 +82,122 @@ bulk = service.mget(keys, UserModel)
|
|
|
78
82
|
service.delete("greeting")
|
|
79
83
|
```
|
|
80
84
|
|
|
85
|
+
### Asynchronous Usage
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
import asyncio
|
|
89
|
+
from pydantic import BaseModel
|
|
90
|
+
from redis_service.redis import AsyncRedisService, RedisServiceConfig
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class UserModel(BaseModel):
|
|
94
|
+
user_id: int
|
|
95
|
+
name: str
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def main():
|
|
99
|
+
# Configure and create async service
|
|
100
|
+
config = RedisServiceConfig(
|
|
101
|
+
redis_host="localhost",
|
|
102
|
+
redis_port=6379,
|
|
103
|
+
redis_db=0,
|
|
104
|
+
)
|
|
105
|
+
service = AsyncRedisService(config)
|
|
106
|
+
await service.connect()
|
|
107
|
+
|
|
108
|
+
try:
|
|
109
|
+
# Health check
|
|
110
|
+
assert await service.is_alive() is True
|
|
111
|
+
|
|
112
|
+
# Write a typed value (automatically serialized to JSON)
|
|
113
|
+
user = UserModel(user_id=1, name="Alice")
|
|
114
|
+
await service.set("user:1", user)
|
|
115
|
+
|
|
116
|
+
# Read and validate back into the model
|
|
117
|
+
loaded = await service.get("user:1", UserModel)
|
|
118
|
+
print(loaded) # UserModel(user_id=1, name='Alice')
|
|
119
|
+
|
|
120
|
+
# Raw access (no validation/decoding beyond Redis decode_responses)
|
|
121
|
+
await service.set("greeting", "hello")
|
|
122
|
+
print(await service.get_raw("greeting")) # "hello"
|
|
123
|
+
|
|
124
|
+
# List keys by pattern and bulk-fetch
|
|
125
|
+
keys = await service.list_keys("user:*")
|
|
126
|
+
bulk = await service.mget(keys, UserModel)
|
|
127
|
+
# bulk is a list of UserModel | None | FailedValue
|
|
128
|
+
|
|
129
|
+
# Delete
|
|
130
|
+
await service.delete("greeting")
|
|
131
|
+
|
|
132
|
+
finally:
|
|
133
|
+
# Always close the connection
|
|
134
|
+
await service.close()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
if __name__ == "__main__":
|
|
138
|
+
asyncio.run(main())
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
### FastAPI Integration Example
|
|
142
|
+
|
|
143
|
+
```python
|
|
144
|
+
from contextlib import asynccontextmanager
|
|
145
|
+
from fastapi import FastAPI, Depends
|
|
146
|
+
from pydantic import BaseModel
|
|
147
|
+
from redis_service.redis import AsyncRedisService, RedisServiceConfig
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class UserModel(BaseModel):
|
|
151
|
+
user_id: int
|
|
152
|
+
name: str
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# Global service instance
|
|
156
|
+
redis_service: AsyncRedisService | None = None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@asynccontextmanager
|
|
160
|
+
async def lifespan(app: FastAPI):
|
|
161
|
+
# Startup: initialize Redis service
|
|
162
|
+
global redis_service
|
|
163
|
+
config = RedisServiceConfig(
|
|
164
|
+
redis_host="localhost",
|
|
165
|
+
redis_port=6379,
|
|
166
|
+
redis_db=0,
|
|
167
|
+
)
|
|
168
|
+
redis_service = AsyncRedisService(config)
|
|
169
|
+
await redis_service.connect()
|
|
170
|
+
|
|
171
|
+
yield
|
|
172
|
+
|
|
173
|
+
# Shutdown: close Redis connection
|
|
174
|
+
if redis_service:
|
|
175
|
+
await redis_service.close()
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
app = FastAPI(lifespan=lifespan)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def get_redis() -> AsyncRedisService:
|
|
182
|
+
if redis_service is None:
|
|
183
|
+
raise RuntimeError("Redis service not initialized")
|
|
184
|
+
return redis_service
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@app.get("/users/{user_id}")
|
|
188
|
+
async def get_user(user_id: int, redis: AsyncRedisService = Depends(get_redis)):
|
|
189
|
+
user = await redis.get(f"user:{user_id}", UserModel)
|
|
190
|
+
if user is None:
|
|
191
|
+
return {"error": "User not found"}
|
|
192
|
+
return user
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@app.post("/users")
|
|
196
|
+
async def create_user(user: UserModel, redis: AsyncRedisService = Depends(get_redis)):
|
|
197
|
+
await redis.set(f"user:{user.user_id}", user)
|
|
198
|
+
return {"status": "created", "user": user}
|
|
199
|
+
```
|
|
200
|
+
|
|
81
201
|
## Development
|
|
82
202
|
|
|
83
203
|
### Project Structure
|
|
@@ -85,16 +205,18 @@ service.delete("greeting")
|
|
|
85
205
|
```
|
|
86
206
|
redis_service/
|
|
87
207
|
├── __init__.py
|
|
88
|
-
├── exceptions.py
|
|
89
|
-
├── logger.py
|
|
90
|
-
├── model.py
|
|
91
|
-
├── redis.py
|
|
92
|
-
├── redis_config_store.py
|
|
93
|
-
├── example.py
|
|
208
|
+
├── exceptions.py # Custom exceptions
|
|
209
|
+
├── logger.py # Module logger
|
|
210
|
+
├── model.py # Base config type for RedisConfigStore
|
|
211
|
+
├── redis.py # RedisService and AsyncRedisService
|
|
212
|
+
├── redis_config_store.py # Config store with caching and watchers
|
|
213
|
+
├── example.py # Small example of using RedisConfigStore
|
|
94
214
|
├── tests/
|
|
95
|
-
│ ├── test_redis_service.py
|
|
96
|
-
│
|
|
97
|
-
|
|
215
|
+
│ ├── test_redis_service.py # Sync unit tests (mocked Redis)
|
|
216
|
+
│ ├── test_async_redis_service.py # Async unit tests (mocked Redis)
|
|
217
|
+
│ ├── test_integration_redis.py # Sync integration tests with testcontainers
|
|
218
|
+
│ └── test_async_integration_redis.py # Async integration tests with testcontainers
|
|
219
|
+
└── README.md # This file
|
|
98
220
|
```
|
|
99
221
|
|
|
100
222
|
## Running Linters
|
|
@@ -158,9 +280,9 @@ PYTHONPATH=$PWD uv run pytest redis_service/tests/test_redis_service.py::test_se
|
|
|
158
280
|
|
|
159
281
|
### Unit Test Details
|
|
160
282
|
|
|
161
|
-
- **Test Framework**: pytest
|
|
162
|
-
- **Mocking**: unittest.mock
|
|
163
|
-
- **Test Count**:
|
|
283
|
+
- **Test Framework**: pytest, pytest-asyncio
|
|
284
|
+
- **Mocking**: unittest.mock (sync), AsyncMock (async)
|
|
285
|
+
- **Test Count**: 60 unit tests (30 sync + 30 async)
|
|
164
286
|
|
|
165
287
|
## Running Integration Tests
|
|
166
288
|
|
|
@@ -183,10 +305,10 @@ PYTHONPATH=$PWD uv run pytest redis_service/tests/test_integration_redis.py -v
|
|
|
183
305
|
|
|
184
306
|
### Integration Test Details
|
|
185
307
|
|
|
186
|
-
- **Test Framework**: pytest + testcontainers
|
|
308
|
+
- **Test Framework**: pytest, pytest-asyncio + testcontainers
|
|
187
309
|
- **Container**: Redis
|
|
188
310
|
- **Image**: `redis:7.2-alpine`
|
|
189
|
-
- **Test Count**:
|
|
311
|
+
- **Test Count**: 17 integration tests (8 sync + 9 async)
|
|
190
312
|
|
|
191
313
|
### Run All Tests (Unit + Integration)
|
|
192
314
|
|
|
@@ -284,7 +406,7 @@ class RedisServiceConfig:
|
|
|
284
406
|
redis_password: str | None = None
|
|
285
407
|
```
|
|
286
408
|
|
|
287
|
-
### RedisService
|
|
409
|
+
### RedisService (Synchronous)
|
|
288
410
|
|
|
289
411
|
High-level service over `redis.Redis` with JSON handling and typed reads.
|
|
290
412
|
|
|
@@ -298,6 +420,24 @@ High-level service over `redis.Redis` with JSON handling and typed reads.
|
|
|
298
420
|
- `mget(keys: list[str], model: type[T]) -> list[T | None | FailedValue]` — Bulk read with per-key error details
|
|
299
421
|
- `delete(key: str) -> bool` — Delete a key; returns True if a key was removed
|
|
300
422
|
|
|
423
|
+
### AsyncRedisService (Asynchronous)
|
|
424
|
+
|
|
425
|
+
High-level async service over `redis.asyncio.Redis` with JSON handling and typed reads.
|
|
426
|
+
|
|
427
|
+
#### Methods
|
|
428
|
+
|
|
429
|
+
All methods are async (use `await`):
|
|
430
|
+
|
|
431
|
+
- `connect() -> None` — Establish connection to Redis (must be called before using other methods)
|
|
432
|
+
- `close() -> None` — Close the Redis connection
|
|
433
|
+
- `is_alive() -> bool` — Ping Redis to check connectivity
|
|
434
|
+
- `set(key: str, value: Any) -> bool` — Serialize and store a value; supports dict/list, Pydantic models, primitives
|
|
435
|
+
- `get(key: str, model: type[T]) -> T | None` — Read and validate JSON into the given Pydantic model
|
|
436
|
+
- `get_raw(key: str) -> Any | None` — Read raw value (usually string) without validation
|
|
437
|
+
- `list_keys(pattern: str) -> list[str]` — List keys matching a glob-like pattern
|
|
438
|
+
- `mget(keys: list[str], model: type[T]) -> list[T | None | FailedValue]` — Bulk read with per-key error details
|
|
439
|
+
- `delete(key: str) -> bool` — Delete a key; returns True if a key was removed
|
|
440
|
+
|
|
301
441
|
#### Exceptions
|
|
302
442
|
|
|
303
443
|
- `RedisConnectionError` — Connection/transport errors
|
redis_service/__init__.py
CHANGED