cledar-sdk 2.0.2__py3-none-any.whl → 2.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cledar/__init__.py +0 -0
- cledar/kafka/README.md +239 -0
- cledar/kafka/__init__.py +40 -0
- cledar/kafka/clients/base.py +98 -0
- cledar/kafka/clients/consumer.py +110 -0
- cledar/kafka/clients/producer.py +80 -0
- cledar/kafka/config/schemas.py +178 -0
- cledar/kafka/exceptions.py +22 -0
- cledar/kafka/handlers/dead_letter.py +82 -0
- cledar/kafka/handlers/parser.py +49 -0
- cledar/kafka/logger.py +3 -0
- cledar/kafka/models/input.py +13 -0
- cledar/kafka/models/message.py +10 -0
- cledar/kafka/models/output.py +8 -0
- cledar/kafka/tests/.env.test.kafka +3 -0
- cledar/kafka/tests/README.md +216 -0
- cledar/kafka/tests/conftest.py +104 -0
- cledar/kafka/tests/integration/__init__.py +1 -0
- cledar/kafka/tests/integration/conftest.py +78 -0
- cledar/kafka/tests/integration/helpers.py +47 -0
- cledar/kafka/tests/integration/test_consumer_integration.py +375 -0
- cledar/kafka/tests/integration/test_integration.py +394 -0
- cledar/kafka/tests/integration/test_producer_consumer_interaction.py +388 -0
- cledar/kafka/tests/integration/test_producer_integration.py +217 -0
- cledar/kafka/tests/unit/__init__.py +1 -0
- cledar/kafka/tests/unit/test_base_kafka_client.py +391 -0
- cledar/kafka/tests/unit/test_config_validation.py +609 -0
- cledar/kafka/tests/unit/test_dead_letter_handler.py +443 -0
- cledar/kafka/tests/unit/test_error_handling.py +674 -0
- cledar/kafka/tests/unit/test_input_parser.py +310 -0
- cledar/kafka/tests/unit/test_input_parser_comprehensive.py +489 -0
- cledar/kafka/tests/unit/test_utils.py +25 -0
- cledar/kafka/tests/unit/test_utils_comprehensive.py +408 -0
- cledar/kafka/utils/callbacks.py +19 -0
- cledar/kafka/utils/messages.py +28 -0
- cledar/kafka/utils/topics.py +2 -0
- cledar/kserve/README.md +352 -0
- cledar/kserve/__init__.py +3 -0
- cledar/kserve/tests/__init__.py +0 -0
- cledar/kserve/tests/test_utils.py +64 -0
- cledar/kserve/utils.py +27 -0
- cledar/logging/README.md +53 -0
- cledar/logging/__init__.py +3 -0
- cledar/logging/tests/test_universal_plaintext_formatter.py +249 -0
- cledar/logging/universal_plaintext_formatter.py +94 -0
- cledar/monitoring/README.md +71 -0
- cledar/monitoring/__init__.py +3 -0
- cledar/monitoring/monitoring_server.py +112 -0
- cledar/monitoring/tests/integration/test_monitoring_server_int.py +162 -0
- cledar/monitoring/tests/test_monitoring_server.py +59 -0
- cledar/nonce/README.md +99 -0
- cledar/nonce/__init__.py +3 -0
- cledar/nonce/nonce_service.py +36 -0
- cledar/nonce/tests/__init__.py +0 -0
- cledar/nonce/tests/test_nonce_service.py +136 -0
- cledar/redis/README.md +536 -0
- cledar/redis/__init__.py +15 -0
- cledar/redis/async_example.py +111 -0
- cledar/redis/example.py +37 -0
- cledar/redis/exceptions.py +22 -0
- cledar/redis/logger.py +3 -0
- cledar/redis/model.py +10 -0
- cledar/redis/redis.py +525 -0
- cledar/redis/redis_config_store.py +252 -0
- cledar/redis/tests/test_async_integration_redis.py +158 -0
- cledar/redis/tests/test_async_redis_service.py +380 -0
- cledar/redis/tests/test_integration_redis.py +119 -0
- cledar/redis/tests/test_redis_service.py +319 -0
- cledar/storage/README.md +529 -0
- cledar/storage/__init__.py +4 -0
- cledar/storage/constants.py +3 -0
- cledar/storage/exceptions.py +50 -0
- cledar/storage/models.py +19 -0
- cledar/storage/object_storage.py +955 -0
- cledar/storage/tests/conftest.py +18 -0
- cledar/storage/tests/test_abfs.py +164 -0
- cledar/storage/tests/test_integration_filesystem.py +359 -0
- cledar/storage/tests/test_integration_s3.py +453 -0
- cledar/storage/tests/test_local.py +384 -0
- cledar/storage/tests/test_s3.py +521 -0
- {cledar_sdk-2.0.2.dist-info → cledar_sdk-2.0.3.dist-info}/METADATA +1 -1
- cledar_sdk-2.0.3.dist-info/RECORD +84 -0
- cledar_sdk-2.0.2.dist-info/RECORD +0 -4
- {cledar_sdk-2.0.2.dist-info → cledar_sdk-2.0.3.dist-info}/WHEEL +0 -0
- {cledar_sdk-2.0.2.dist-info → cledar_sdk-2.0.3.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from fastapi import FastAPI
|
|
3
|
+
from fastapi.testclient import TestClient
|
|
4
|
+
|
|
5
|
+
from cledar.monitoring import MonitoringServer, MonitoringServerConfig
|
|
6
|
+
|
|
7
|
+
HOST = "localhost"
|
|
8
|
+
PORT = 9999
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ReadinessFlag:
|
|
12
|
+
def __init__(self) -> None:
|
|
13
|
+
self.is_ready = False
|
|
14
|
+
|
|
15
|
+
def mark_ready(self) -> None:
|
|
16
|
+
self.is_ready = True
|
|
17
|
+
|
|
18
|
+
def check_if_ready(self) -> bool:
|
|
19
|
+
return self.is_ready
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.fixture
|
|
23
|
+
def readiness_flag() -> ReadinessFlag:
|
|
24
|
+
_readiness_flag = ReadinessFlag()
|
|
25
|
+
return _readiness_flag
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@pytest.fixture
|
|
29
|
+
def app(readiness_flag: ReadinessFlag) -> FastAPI:
|
|
30
|
+
_app = FastAPI()
|
|
31
|
+
default_readiness_checks = dict({"is_ready": readiness_flag.check_if_ready})
|
|
32
|
+
config = MonitoringServerConfig(default_readiness_checks)
|
|
33
|
+
monitoring_server = MonitoringServer(HOST, PORT, config)
|
|
34
|
+
monitoring_server.add_paths(_app)
|
|
35
|
+
return _app
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@pytest.fixture
|
|
39
|
+
def client(app: FastAPI) -> TestClient:
|
|
40
|
+
_client = TestClient(app)
|
|
41
|
+
return _client
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_liveness(client: TestClient) -> None:
|
|
45
|
+
response = client.get("/healthz/liveness")
|
|
46
|
+
assert response.status_code == 200
|
|
47
|
+
assert response.text == '{"status": "ok", "checks": {}}'
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_readiness(client: TestClient, readiness_flag: ReadinessFlag) -> None:
|
|
51
|
+
response = client.get("/healthz/readiness")
|
|
52
|
+
assert response.status_code == 503
|
|
53
|
+
assert response.text == '{"status": "error", "checks": {"is_ready": false}}'
|
|
54
|
+
|
|
55
|
+
readiness_flag.mark_ready()
|
|
56
|
+
|
|
57
|
+
response = client.get("/healthz/readiness")
|
|
58
|
+
assert response.status_code == 200
|
|
59
|
+
assert response.text == '{"status": "ok", "checks": {"is_ready": true}}'
|
cledar/nonce/README.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# NonceService
|
|
2
|
+
|
|
3
|
+
Simple nonce service for preventing duplicate request processing using Redis with TTL-based automatic cleanup.
|
|
4
|
+
|
|
5
|
+
It is useful for implementing idempotency in APIs, background workers, and event processing pipelines where the same request/message may arrive more than once.
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
- Fast duplicate detection backed by Redis SET NX
|
|
9
|
+
- Per-endpoint scoping (the same nonce can be used across different endpoints)
|
|
10
|
+
- Automatic expiry via TTL (default 1 hour)
|
|
11
|
+
- Tiny API surface and easy integration
|
|
12
|
+
|
|
13
|
+
## Requirements
|
|
14
|
+
- Redis server accessible from your application
|
|
15
|
+
- Python dependencies:
|
|
16
|
+
- `redis` (redis-py)
|
|
17
|
+
- This repository/module (imports shown below)
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
Make sure you have `redis` installed in your environment:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
uv add redis
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
This module lives inside this repository. Import it directly from your code as shown in the examples below.
|
|
27
|
+
|
|
28
|
+
## Quickstart
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
import asyncio
|
|
32
|
+
from cledar.redis.redis import RedisService, RedisServiceConfig
|
|
33
|
+
from cledar.nonce import NonceService
|
|
34
|
+
|
|
35
|
+
# 1) Create a Redis client
|
|
36
|
+
config = RedisServiceConfig(redis_host="localhost", redis_port=6379, redis_db=0)
|
|
37
|
+
redis_client = RedisService(config)
|
|
38
|
+
|
|
39
|
+
# 2) Create the NonceService
|
|
40
|
+
nonce_service = NonceService(redis_client)
|
|
41
|
+
|
|
42
|
+
# Optional: override default TTL (seconds)
|
|
43
|
+
nonce_service.default_ttl = 7200 # 2 hours
|
|
44
|
+
|
|
45
|
+
async def main():
|
|
46
|
+
nonce = "request-id-123"
|
|
47
|
+
endpoint = "/api/payment" # any string identifying the endpoint or operation
|
|
48
|
+
|
|
49
|
+
# First time: not a duplicate -> returns False
|
|
50
|
+
first = await nonce_service.is_duplicate(nonce, endpoint)
|
|
51
|
+
print(first) # False
|
|
52
|
+
|
|
53
|
+
# Second time (same nonce + same endpoint): duplicate -> returns True
|
|
54
|
+
second = await nonce_service.is_duplicate(nonce, endpoint)
|
|
55
|
+
print(second) # True
|
|
56
|
+
|
|
57
|
+
# Same nonce but a different endpoint is treated independently
|
|
58
|
+
third = await nonce_service.is_duplicate(nonce, "/api/other-endpoint")
|
|
59
|
+
print(third) # False
|
|
60
|
+
|
|
61
|
+
asyncio.run(main())
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## How it works
|
|
65
|
+
Under the hood, `NonceService.is_duplicate(nonce, endpoint)` performs a Redis `SET` with the flags `NX` (only set if not exists) and `EX=<TTL>`. If Redis returns that the key was set, this is the first time the nonce is seen for that endpoint (not a duplicate -> returns `False`). If the key already exists, Redis returns `None`, which the service treats as a duplicate -> returns `True`.
|
|
66
|
+
|
|
67
|
+
- Redis key format: `nonce:{endpoint}:{nonce}`
|
|
68
|
+
- Default TTL: `3600` seconds (1 hour); can be changed via `nonce_service.default_ttl`
|
|
69
|
+
- Endpoint scoping: the same nonce value can be used across different endpoints without being considered a duplicate between them
|
|
70
|
+
|
|
71
|
+
## API
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from cledar.redis.redis import RedisService
|
|
75
|
+
|
|
76
|
+
class NonceService:
|
|
77
|
+
def __init__(self, redis_client: RedisService):
|
|
78
|
+
"""Simple service for managing nonces to prevent duplicate requests."""
|
|
79
|
+
...
|
|
80
|
+
|
|
81
|
+
async def is_duplicate(self, nonce: str, endpoint: str) -> bool:
|
|
82
|
+
"""Return True if (nonce, endpoint) was already used within TTL."""
|
|
83
|
+
...
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### Errors
|
|
87
|
+
- `RuntimeError("Redis client is not initialized")` — raised when `redis_client._client` is `None`.
|
|
88
|
+
|
|
89
|
+
## Testing
|
|
90
|
+
This module includes unit tests. To run them:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
uv run pytest nonce_service/tests -q
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Best practices
|
|
97
|
+
- Use a stable nonce that uniquely identifies an operation (e.g., request ID, message ID)
|
|
98
|
+
- Choose a TTL that matches the maximum time window in which duplicates must be rejected
|
|
99
|
+
- Scope by a meaningful `endpoint` string to separate distinct operations
|
cledar/nonce/__init__.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Simple nonce service for preventing duplicate request processing.
|
|
3
|
+
Uses Redis with TTL for automatic cleanup.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from cledar.redis.redis import RedisService
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class NonceService:
|
|
10
|
+
"""Simple service for managing nonces to prevent duplicate requests."""
|
|
11
|
+
|
|
12
|
+
def __init__(self, redis_client: RedisService):
|
|
13
|
+
self.redis_client = redis_client
|
|
14
|
+
self.nonce_prefix = "nonce"
|
|
15
|
+
self.default_ttl = 3600 # 1 hour
|
|
16
|
+
|
|
17
|
+
def _get_nonce_key(self, nonce: str, endpoint: str) -> str:
|
|
18
|
+
"""Generate Redis key for nonce"""
|
|
19
|
+
return f"{self.nonce_prefix}:{endpoint}:{nonce}"
|
|
20
|
+
|
|
21
|
+
async def is_duplicate(self, nonce: str, endpoint: str) -> bool:
|
|
22
|
+
"""Check if nonce was already used (returns True if duplicate)"""
|
|
23
|
+
|
|
24
|
+
nonce_key = self._get_nonce_key(nonce, endpoint)
|
|
25
|
+
|
|
26
|
+
if self.redis_client._client is None:
|
|
27
|
+
raise RuntimeError("Redis client is not initialized")
|
|
28
|
+
|
|
29
|
+
result = self.redis_client._client.set(
|
|
30
|
+
nonce_key,
|
|
31
|
+
"used",
|
|
32
|
+
nx=True,
|
|
33
|
+
ex=self.default_ttl,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
return result is None
|
|
File without changes
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# mypy: disable-error-code=no-untyped-def
|
|
2
|
+
from unittest.mock import MagicMock
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from cledar.nonce import NonceService
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@pytest.fixture(name="redis_client")
|
|
10
|
+
def fixture_redis_client():
|
|
11
|
+
"""Mock Redis client"""
|
|
12
|
+
mock_redis = MagicMock()
|
|
13
|
+
mock_redis._client = MagicMock()
|
|
14
|
+
return mock_redis
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture(name="nonce_service")
|
|
18
|
+
def fixture_nonce_service(redis_client):
|
|
19
|
+
"""Create NonceService with a mocked Redis client"""
|
|
20
|
+
return NonceService(redis_client)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_init(redis_client):
|
|
24
|
+
"""Test NonceService initialization"""
|
|
25
|
+
service = NonceService(redis_client)
|
|
26
|
+
|
|
27
|
+
assert service.redis_client == redis_client
|
|
28
|
+
assert service.nonce_prefix == "nonce"
|
|
29
|
+
assert service.default_ttl == 3600
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_get_nonce_key(nonce_service):
|
|
33
|
+
"""Test nonce key generation"""
|
|
34
|
+
nonce = "test-nonce-123"
|
|
35
|
+
endpoint = "/api/payment"
|
|
36
|
+
|
|
37
|
+
key = nonce_service._get_nonce_key(nonce, endpoint)
|
|
38
|
+
|
|
39
|
+
assert key == "nonce:/api/payment:test-nonce-123"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.mark.asyncio
|
|
43
|
+
async def test_is_duplicate_first_time(nonce_service):
|
|
44
|
+
"""Test that the first use of nonce returns False (not duplicate)"""
|
|
45
|
+
nonce = "unique-nonce-456"
|
|
46
|
+
endpoint = "/api/transaction"
|
|
47
|
+
|
|
48
|
+
# Mock Redis set to return True (key was set successfully)
|
|
49
|
+
nonce_service.redis_client._client.set.return_value = True
|
|
50
|
+
|
|
51
|
+
result = await nonce_service.is_duplicate(nonce, endpoint)
|
|
52
|
+
|
|
53
|
+
assert result is False
|
|
54
|
+
nonce_service.redis_client._client.set.assert_called_once_with(
|
|
55
|
+
"nonce:/api/transaction:unique-nonce-456",
|
|
56
|
+
"used",
|
|
57
|
+
nx=True,
|
|
58
|
+
ex=3600,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@pytest.mark.asyncio
|
|
63
|
+
async def test_is_duplicate_second_time(nonce_service):
|
|
64
|
+
"""Test that second use of same nonce returns True (duplicate)"""
|
|
65
|
+
nonce = "duplicate-nonce-789"
|
|
66
|
+
endpoint = "/api/refund"
|
|
67
|
+
|
|
68
|
+
# Mock Redis set to return None (key already exists)
|
|
69
|
+
nonce_service.redis_client._client.set.return_value = None
|
|
70
|
+
|
|
71
|
+
result = await nonce_service.is_duplicate(nonce, endpoint)
|
|
72
|
+
|
|
73
|
+
assert result is True
|
|
74
|
+
nonce_service.redis_client._client.set.assert_called_once_with(
|
|
75
|
+
"nonce:/api/refund:duplicate-nonce-789",
|
|
76
|
+
"used",
|
|
77
|
+
nx=True,
|
|
78
|
+
ex=3600,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@pytest.mark.asyncio
|
|
83
|
+
async def test_is_duplicate_redis_not_initialized(nonce_service):
|
|
84
|
+
"""Test that RuntimeError is raised when a Redis client is not initialized"""
|
|
85
|
+
nonce = "test-nonce"
|
|
86
|
+
endpoint = "/api/test"
|
|
87
|
+
|
|
88
|
+
# Simulate uninitialized Redis client
|
|
89
|
+
nonce_service.redis_client._client = None
|
|
90
|
+
|
|
91
|
+
with pytest.raises(RuntimeError, match="Redis client is not initialized"):
|
|
92
|
+
await nonce_service.is_duplicate(nonce, endpoint)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@pytest.mark.asyncio
|
|
96
|
+
async def test_is_duplicate_different_endpoints(nonce_service):
|
|
97
|
+
"""Test that the same nonce is independent across different endpoints"""
|
|
98
|
+
nonce = "same-nonce"
|
|
99
|
+
endpoint1 = "/api/endpoint1"
|
|
100
|
+
endpoint2 = "/api/endpoint2"
|
|
101
|
+
|
|
102
|
+
# Both calls should treat the nonce as new (not duplicate)
|
|
103
|
+
nonce_service.redis_client._client.set.return_value = True
|
|
104
|
+
|
|
105
|
+
result1 = await nonce_service.is_duplicate(nonce, endpoint1)
|
|
106
|
+
result2 = await nonce_service.is_duplicate(nonce, endpoint2)
|
|
107
|
+
|
|
108
|
+
assert result1 is False
|
|
109
|
+
assert result2 is False
|
|
110
|
+
|
|
111
|
+
# Verify both keys were set with different endpoint prefixes
|
|
112
|
+
assert nonce_service.redis_client._client.set.call_count == 2
|
|
113
|
+
calls = nonce_service.redis_client._client.set.call_args_list
|
|
114
|
+
assert calls[0][0][0] == "nonce:/api/endpoint1:same-nonce"
|
|
115
|
+
assert calls[1][0][0] == "nonce:/api/endpoint2:same-nonce"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@pytest.mark.asyncio
|
|
119
|
+
async def test_is_duplicate_custom_ttl(redis_client):
|
|
120
|
+
"""Test that custom TTL is used correctly"""
|
|
121
|
+
service = NonceService(redis_client)
|
|
122
|
+
service.default_ttl = 7200 # 2 hours
|
|
123
|
+
|
|
124
|
+
nonce = "test-nonce"
|
|
125
|
+
endpoint = "/api/test"
|
|
126
|
+
|
|
127
|
+
redis_client._client.set.return_value = True
|
|
128
|
+
|
|
129
|
+
await service.is_duplicate(nonce, endpoint)
|
|
130
|
+
|
|
131
|
+
redis_client._client.set.assert_called_once_with(
|
|
132
|
+
"nonce:/api/test:test-nonce",
|
|
133
|
+
"used",
|
|
134
|
+
nx=True,
|
|
135
|
+
ex=7200,
|
|
136
|
+
)
|