cledar-sdk 2.0.3__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cledar/__init__.py +1 -0
- cledar/kafka/__init__.py +2 -0
- cledar/kafka/clients/base.py +24 -5
- cledar/kafka/clients/consumer.py +28 -0
- cledar/kafka/clients/producer.py +17 -0
- cledar/kafka/config/schemas.py +91 -7
- cledar/kafka/exceptions.py +7 -12
- cledar/kafka/handlers/dead_letter.py +26 -20
- cledar/kafka/handlers/parser.py +36 -2
- cledar/kafka/logger.py +2 -0
- cledar/kafka/models/input.py +4 -0
- cledar/kafka/models/message.py +4 -0
- cledar/kafka/models/output.py +4 -0
- cledar/kafka/utils/callbacks.py +9 -0
- cledar/kafka/utils/messages.py +11 -0
- cledar/kafka/utils/topics.py +13 -0
- cledar/kserve/__init__.py +2 -0
- cledar/kserve/utils.py +3 -0
- cledar/logging/__init__.py +2 -0
- cledar/logging/universal_plaintext_formatter.py +17 -12
- cledar/monitoring/__init__.py +2 -0
- cledar/monitoring/monitoring_server.py +45 -1
- cledar/nonce/__init__.py +2 -0
- cledar/nonce/nonce_service.py +30 -4
- cledar/redis/__init__.py +2 -0
- cledar/redis/async_example.py +4 -3
- cledar/redis/example.py +30 -0
- cledar/redis/exceptions.py +3 -0
- cledar/redis/logger.py +2 -0
- cledar/redis/model.py +4 -0
- cledar/redis/redis.py +252 -13
- cledar/redis/redis_config_store.py +81 -0
- cledar/storage/__init__.py +2 -0
- cledar/storage/constants.py +2 -0
- cledar/storage/exceptions.py +29 -0
- cledar/storage/models.py +22 -0
- cledar/storage/object_storage.py +342 -23
- {cledar_sdk-2.0.3.dist-info → cledar_sdk-2.1.0.dist-info}/METADATA +1 -1
- {cledar_sdk-2.0.3.dist-info → cledar_sdk-2.1.0.dist-info}/RECORD +41 -41
- {cledar_sdk-2.0.3.dist-info → cledar_sdk-2.1.0.dist-info}/WHEEL +0 -0
- {cledar_sdk-2.0.3.dist-info → cledar_sdk-2.1.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,11 +1,16 @@
|
|
|
1
|
+
"""Custom logging formatter for plaintext output with extra attributes.
|
|
2
|
+
|
|
3
|
+
This module provides the UniversalPlaintextFormatter class which allows
|
|
4
|
+
for easy logging of extra attributes in a human-readable format.
|
|
5
|
+
"""
|
|
6
|
+
|
|
1
7
|
import configparser
|
|
2
8
|
import logging
|
|
3
9
|
from typing import Any
|
|
4
10
|
|
|
5
11
|
|
|
6
12
|
class UniversalPlaintextFormatter(logging.Formatter):
|
|
7
|
-
"""
|
|
8
|
-
A custom formatter for logging that extends the standard logging.Formatter.
|
|
13
|
+
"""A custom formatter for logging that extends the standard logging.Formatter.
|
|
9
14
|
|
|
10
15
|
This formatter adds the ability to include extra attributes from log records while
|
|
11
16
|
excluding standard attributes and configurable keys.
|
|
@@ -15,23 +20,23 @@ class UniversalPlaintextFormatter(logging.Formatter):
|
|
|
15
20
|
DEFAULT_EXCLUDE_KEYS = {"message", "asctime"}
|
|
16
21
|
|
|
17
22
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
18
|
-
"""
|
|
19
|
-
Initialize the formatter with standard formatter parameters.
|
|
23
|
+
"""Initialize the formatter with standard formatter parameters.
|
|
20
24
|
|
|
21
25
|
Args:
|
|
22
26
|
*args: Variable length argument list for the parent class.
|
|
23
27
|
**kwargs: Arbitrary keyword arguments for the parent class.
|
|
28
|
+
|
|
24
29
|
"""
|
|
25
30
|
super().__init__(*args, **kwargs)
|
|
26
31
|
self._standard_attrs: set[str] | None = None
|
|
27
32
|
self._config_exclude_keys = self._load_exclude_keys_from_config()
|
|
28
33
|
|
|
29
34
|
def _load_exclude_keys_from_config(self) -> set[str]:
|
|
30
|
-
"""
|
|
31
|
-
Load additional keys to exclude from the configuration file.
|
|
35
|
+
"""Load additional keys to exclude from the configuration file.
|
|
32
36
|
|
|
33
37
|
Returns:
|
|
34
|
-
set: A set of keys to exclude from log records.
|
|
38
|
+
set[str]: A set of keys to exclude from log records.
|
|
39
|
+
|
|
35
40
|
"""
|
|
36
41
|
try:
|
|
37
42
|
config = configparser.ConfigParser()
|
|
@@ -44,14 +49,14 @@ class UniversalPlaintextFormatter(logging.Formatter):
|
|
|
44
49
|
return set()
|
|
45
50
|
|
|
46
51
|
def _get_standard_attrs(self) -> set[str]:
|
|
47
|
-
"""
|
|
48
|
-
Get the set of standard attributes to exclude from log records.
|
|
52
|
+
"""Get the set of standard attributes to exclude from log records.
|
|
49
53
|
|
|
50
54
|
This includes standard LogRecord attributes, predefined exclusions,
|
|
51
55
|
and exclusions from configuration.
|
|
52
56
|
|
|
53
57
|
Returns:
|
|
54
|
-
set: A set of attribute names to exclude.
|
|
58
|
+
set[str]: A set of attribute names to exclude.
|
|
59
|
+
|
|
55
60
|
"""
|
|
56
61
|
if self._standard_attrs is None:
|
|
57
62
|
dummy_record = logging.LogRecord(
|
|
@@ -73,14 +78,14 @@ class UniversalPlaintextFormatter(logging.Formatter):
|
|
|
73
78
|
return self._standard_attrs
|
|
74
79
|
|
|
75
80
|
def format(self, record: logging.LogRecord) -> str:
|
|
76
|
-
"""
|
|
77
|
-
Format the log record, adding any extra attributes not in the standard set.
|
|
81
|
+
"""Format the log record, adding any extra attributes not in the standard set.
|
|
78
82
|
|
|
79
83
|
Args:
|
|
80
84
|
record: The log record to format.
|
|
81
85
|
|
|
82
86
|
Returns:
|
|
83
87
|
str: The formatted log message with extra attributes appended.
|
|
88
|
+
|
|
84
89
|
"""
|
|
85
90
|
base = super().format(record)
|
|
86
91
|
extras = {
|
cledar/monitoring/__init__.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
"""Prometheus monitoring and health checks server implementation."""
|
|
2
|
+
|
|
1
3
|
import json
|
|
2
4
|
import logging
|
|
3
|
-
import logging.config
|
|
4
5
|
import threading
|
|
5
6
|
from collections.abc import Callable
|
|
6
7
|
|
|
@@ -28,22 +29,49 @@ def _run_monitoring_server(host: str, port: int, app: FastAPI) -> None:
|
|
|
28
29
|
|
|
29
30
|
@dataclass
|
|
30
31
|
class MonitoringServerConfig:
|
|
32
|
+
"""Configuration for the MonitoringServer.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
readiness_checks: A dictionary of name to callable for readiness checks.
|
|
36
|
+
liveness_checks: An optional dictionary for liveness checks.
|
|
37
|
+
|
|
38
|
+
"""
|
|
39
|
+
|
|
31
40
|
readiness_checks: dict[str, Callable[[], bool]]
|
|
32
41
|
liveness_checks: dict[str, Callable[[], bool]] | None = None
|
|
33
42
|
|
|
34
43
|
|
|
35
44
|
class EndpointFilter(logging.Filter):
|
|
45
|
+
"""Filter for logging that excludes certain paths."""
|
|
46
|
+
|
|
36
47
|
def __init__(self, paths_excluded_for_logging: list[str]):
|
|
48
|
+
"""Initialize the EndpointFilter.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
paths_excluded_for_logging: List of paths to exclude from logs.
|
|
52
|
+
|
|
53
|
+
"""
|
|
37
54
|
super().__init__()
|
|
38
55
|
self.paths_excluded_for_logging = paths_excluded_for_logging
|
|
39
56
|
|
|
40
57
|
def filter(self, record: logging.LogRecord) -> bool:
|
|
58
|
+
"""Filter log records based on path exclusions.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
record: The log record to check.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
bool: True if record should be logged, False otherwise.
|
|
65
|
+
|
|
66
|
+
"""
|
|
41
67
|
return not any(
|
|
42
68
|
path in record.getMessage() for path in self.paths_excluded_for_logging
|
|
43
69
|
)
|
|
44
70
|
|
|
45
71
|
|
|
46
72
|
class MonitoringServer:
|
|
73
|
+
"""A server that exposes Prometheus metrics and health check endpoints."""
|
|
74
|
+
|
|
47
75
|
PATHS_EXCLUDED_FOR_LOGGING = ["/healthz/readiness", "/healthz/liveness"]
|
|
48
76
|
|
|
49
77
|
def __init__(
|
|
@@ -52,6 +80,14 @@ class MonitoringServer:
|
|
|
52
80
|
port: int,
|
|
53
81
|
config: MonitoringServerConfig,
|
|
54
82
|
):
|
|
83
|
+
"""Initialize the MonitoringServer.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
host: The host to bind the server to.
|
|
87
|
+
port: The port to bind the server to.
|
|
88
|
+
config: The server configuration.
|
|
89
|
+
|
|
90
|
+
"""
|
|
55
91
|
self.config = config
|
|
56
92
|
self.host = host
|
|
57
93
|
self.port = port
|
|
@@ -60,6 +96,13 @@ class MonitoringServer:
|
|
|
60
96
|
)
|
|
61
97
|
|
|
62
98
|
def add_paths(self, app: FastAPI) -> None:
|
|
99
|
+
"""Add monitoring and health check endpoints to the FastAPI application.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
app: The FastAPI application to add routes to.
|
|
103
|
+
|
|
104
|
+
"""
|
|
105
|
+
|
|
63
106
|
@app.get("/metrics")
|
|
64
107
|
async def get_metrics() -> Response:
|
|
65
108
|
return Response(
|
|
@@ -101,6 +144,7 @@ class MonitoringServer:
|
|
|
101
144
|
return Response(content=data_json, status_code=503)
|
|
102
145
|
|
|
103
146
|
def start_monitoring_server(self) -> None:
|
|
147
|
+
"""Start the monitoring server in a background thread."""
|
|
104
148
|
local_app = _create_app()
|
|
105
149
|
self.add_paths(local_app)
|
|
106
150
|
server_thread = threading.Thread(
|
cledar/nonce/__init__.py
CHANGED
cledar/nonce/nonce_service.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
1
|
+
"""Simple nonce service for preventing duplicate request processing.
|
|
2
|
+
|
|
3
3
|
Uses Redis with TTL for automatic cleanup.
|
|
4
4
|
"""
|
|
5
5
|
|
|
@@ -10,17 +10,43 @@ class NonceService:
|
|
|
10
10
|
"""Simple service for managing nonces to prevent duplicate requests."""
|
|
11
11
|
|
|
12
12
|
def __init__(self, redis_client: RedisService):
|
|
13
|
+
"""Initialize the nonce service.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
redis_client: The Redis client service used for storage.
|
|
17
|
+
|
|
18
|
+
"""
|
|
13
19
|
self.redis_client = redis_client
|
|
14
20
|
self.nonce_prefix = "nonce"
|
|
15
21
|
self.default_ttl = 3600 # 1 hour
|
|
16
22
|
|
|
17
23
|
def _get_nonce_key(self, nonce: str, endpoint: str) -> str:
|
|
18
|
-
"""Generate Redis key for nonce
|
|
24
|
+
"""Generate Redis key for nonce.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
nonce: The unique identifier for the request.
|
|
28
|
+
endpoint: The endpoint the request is directed at.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
str: The formatted Redis key.
|
|
32
|
+
|
|
33
|
+
"""
|
|
19
34
|
return f"{self.nonce_prefix}:{endpoint}:{nonce}"
|
|
20
35
|
|
|
21
36
|
async def is_duplicate(self, nonce: str, endpoint: str) -> bool:
|
|
22
|
-
"""Check if nonce was already used (returns True if duplicate)
|
|
37
|
+
"""Check if nonce was already used (returns True if duplicate).
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
nonce: The unique identifier for the request.
|
|
41
|
+
endpoint: The endpoint the request is directed at.
|
|
42
|
+
|
|
43
|
+
Returns:
|
|
44
|
+
bool: True if the nonce is a duplicate, False otherwise.
|
|
45
|
+
|
|
46
|
+
Raises:
|
|
47
|
+
RuntimeError: If the Redis client is not initialized.
|
|
23
48
|
|
|
49
|
+
"""
|
|
24
50
|
nonce_key = self._get_nonce_key(nonce, endpoint)
|
|
25
51
|
|
|
26
52
|
if self.redis_client._client is None:
|
cledar/redis/__init__.py
CHANGED
cledar/redis/async_example.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Example usage of AsyncRedisService with async/await.
|
|
1
|
+
"""Example usage of AsyncRedisService with async/await.
|
|
3
2
|
|
|
4
3
|
This example demonstrates:
|
|
5
4
|
- Connecting to Redis asynchronously
|
|
@@ -16,13 +15,15 @@ from cledar.redis import AsyncRedisService, RedisServiceConfig
|
|
|
16
15
|
|
|
17
16
|
|
|
18
17
|
class UserModel(BaseModel):
|
|
18
|
+
"""Simple user model for demonstration."""
|
|
19
|
+
|
|
19
20
|
user_id: int
|
|
20
21
|
name: str
|
|
21
22
|
email: str
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
async def basic_usage_example() -> None:
|
|
25
|
-
"""
|
|
26
|
+
"""Demonstrate basic async Redis operations."""
|
|
26
27
|
print("=== Basic Async Usage ===")
|
|
27
28
|
|
|
28
29
|
# Configure service
|
cledar/redis/example.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
"""Example usage of RedisConfigStore."""
|
|
2
|
+
|
|
1
3
|
from dataclasses import dataclass
|
|
2
4
|
from typing import Any
|
|
3
5
|
|
|
@@ -7,6 +9,8 @@ from .redis_config_store import RedisConfigStore
|
|
|
7
9
|
|
|
8
10
|
@dataclass
|
|
9
11
|
class ExampleConfig(BaseConfigClass):
|
|
12
|
+
"""Example configuration model."""
|
|
13
|
+
|
|
10
14
|
name: str
|
|
11
15
|
index: int
|
|
12
16
|
data: dict[str, Any]
|
|
@@ -17,20 +21,46 @@ CONFIG_KEY = "example_config"
|
|
|
17
21
|
|
|
18
22
|
|
|
19
23
|
class ConfigProvider:
|
|
24
|
+
"""Example provider for configuration using RedisConfigStore."""
|
|
25
|
+
|
|
20
26
|
def __init__(self, redis_config_store: RedisConfigStore) -> None:
|
|
27
|
+
"""Initialize the ConfigProvider.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
redis_config_store: The Redis config store instance.
|
|
31
|
+
|
|
32
|
+
"""
|
|
21
33
|
self.redis_config_store = redis_config_store
|
|
22
34
|
if self.redis_config_store.fetch(ExampleConfig, CONFIG_KEY) is None:
|
|
23
35
|
self.redis_config_store[CONFIG_KEY] = DEFAULT_CONFIG
|
|
24
36
|
|
|
25
37
|
def get_example_config(self) -> ExampleConfig:
|
|
38
|
+
"""Get the example configuration.
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
ExampleConfig: The current configuration.
|
|
42
|
+
|
|
43
|
+
"""
|
|
26
44
|
return (
|
|
27
45
|
self.redis_config_store.fetch(ExampleConfig, CONFIG_KEY) or DEFAULT_CONFIG
|
|
28
46
|
)
|
|
29
47
|
|
|
30
48
|
def get_example_config_version(self) -> int:
|
|
49
|
+
"""Get the version of the example configuration.
|
|
50
|
+
|
|
51
|
+
Returns:
|
|
52
|
+
int: The configuration version.
|
|
53
|
+
|
|
54
|
+
"""
|
|
31
55
|
return self.redis_config_store.cached_version(CONFIG_KEY) or -1
|
|
32
56
|
|
|
33
57
|
def set_example_config(self, config: ExampleConfig | None) -> None:
|
|
58
|
+
"""Set the example configuration.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
config: The new configuration to set.
|
|
62
|
+
|
|
63
|
+
"""
|
|
34
64
|
if config is None:
|
|
35
65
|
return
|
|
36
66
|
|
cledar/redis/exceptions.py
CHANGED
cledar/redis/logger.py
CHANGED
cledar/redis/model.py
CHANGED