muscles-data-redis 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ from __future__ import annotations
2
+
3
+ from .adapter import (
4
+ RedisAdapterError,
5
+ RedisClientMissingError,
6
+ RedisConfigError,
7
+ RedisConnectionError,
8
+ RedisDataAdapter,
9
+ RedisDataFactory,
10
+ )
11
+
12
+
13
+ __all__ = [
14
+ "RedisAdapterError",
15
+ "RedisClientMissingError",
16
+ "RedisConfigError",
17
+ "RedisConnectionError",
18
+ "RedisDataAdapter",
19
+ "RedisDataFactory",
20
+ ]
@@ -0,0 +1,419 @@
1
+ from __future__ import annotations
2
+
3
+ import importlib
4
+ import json
5
+ import threading
6
+ import time
7
+ import uuid
8
+ from collections.abc import Callable
9
+ from dataclasses import asdict
10
+ from typing import Any, Mapping
11
+ from urllib.parse import urlsplit
12
+
13
+ from muscles_data.config import DataResourceConfig
14
+ from muscles_data.errors import DataConfigurationError, DataConnectionError, DataError
15
+ from muscles_data.models import DataCapability, HealthResult, InspectResult, LockHandle, StreamReadResult, WriteResult
16
+
17
+
18
+ _CLIENT_UNSET = object()
19
+ _ALLOWED_OPTIONS = {
20
+ "url",
21
+ "url_env",
22
+ "namespace",
23
+ "decode_responses",
24
+ "timeout",
25
+ "socket_timeout",
26
+ "socket_connect_timeout",
27
+ "stream_group",
28
+ "consumer",
29
+ "native_client",
30
+ }
31
+ _LOCK_RELEASE_SCRIPT = """
32
+ local token = redis.call('get', KEYS[1])
33
+ if not token or token ~= ARGV[1] then
34
+ return 0
35
+ end
36
+ redis.call('del', KEYS[1])
37
+ return 1
38
+ """
39
+
40
+
41
+ class RedisAdapterError(DataError):
42
+ """Base error for Redis data adapter failures."""
43
+
44
+
45
+ class RedisConfigError(ValueError, DataConfigurationError, RedisAdapterError):
46
+ """Raised when a Redis resource config cannot be mapped safely."""
47
+
48
+
49
+ class RedisClientMissingError(DataConfigurationError, RedisAdapterError):
50
+ """Raised when Redis adapter is used without an available client."""
51
+
52
+
53
+ class RedisConnectionError(DataConnectionError, RedisAdapterError):
54
+ """Raised when a Redis operation cannot reach or use the backend."""
55
+
56
+
57
+ class RedisDataAdapter:
58
+ resource_type = "redis"
59
+
60
+ def __init__(
61
+ self,
62
+ config: DataResourceConfig,
63
+ *,
64
+ client_factory: Callable[[DataResourceConfig], Any] | None = None,
65
+ ) -> None:
66
+ self.config = config
67
+ self._client_factory = client_factory
68
+ self._client: Any = _CLIENT_UNSET
69
+ self._lock = threading.RLock()
70
+ self.closed = False
71
+
72
+ def get(self, key: str) -> bytes | None:
73
+ try:
74
+ value = self._client_instance().get(self._key(key))
75
+ except (RedisClientMissingError, RedisConfigError):
76
+ raise
77
+ except Exception as exc:
78
+ raise RedisConnectionError(self._safe_error(exc)) from exc
79
+ return _bytes_or_none(value)
80
+
81
+ def set(self, key: str, value: bytes, ttl_seconds: float | None = None) -> WriteResult:
82
+ kwargs: dict[str, Any] = {}
83
+ if ttl_seconds is not None:
84
+ kwargs["px"] = _ttl_ms(ttl_seconds, "Redis key ttl_seconds")
85
+ try:
86
+ written = bool(self._client_instance().set(self._key(key), bytes(value), **kwargs))
87
+ except (RedisClientMissingError, RedisConfigError):
88
+ raise
89
+ except Exception as exc:
90
+ raise RedisConnectionError(self._safe_error(exc)) from exc
91
+ return WriteResult(written=1 if written else 0, matched=1 if written else 0)
92
+
93
+ def delete(self, key: str) -> WriteResult:
94
+ try:
95
+ deleted = int(self._client_instance().delete(self._key(key)) or 0)
96
+ except (RedisClientMissingError, RedisConfigError):
97
+ raise
98
+ except Exception as exc:
99
+ raise RedisConnectionError(self._safe_error(exc)) from exc
100
+ return WriteResult(deleted=deleted, matched=deleted)
101
+
102
+ def exists(self, key: str) -> bool:
103
+ try:
104
+ return bool(self._client_instance().exists(self._key(key)))
105
+ except (RedisClientMissingError, RedisConfigError):
106
+ raise
107
+ except Exception as exc:
108
+ raise RedisConnectionError(self._safe_error(exc)) from exc
109
+
110
+ def acquire_lock(self, name: str, ttl_seconds: float):
111
+ ttl_ms = _ttl_ms(ttl_seconds, "Redis lock ttl_seconds")
112
+ token = uuid.uuid4().hex
113
+ lock_key = self._lock_key(name)
114
+ try:
115
+ acquired = bool(self._client_instance().set(lock_key, token, nx=True, px=ttl_ms))
116
+ except (RedisClientMissingError, RedisConfigError):
117
+ raise
118
+ except Exception as exc:
119
+ raise RedisConnectionError(self._safe_error(exc)) from exc
120
+ if not acquired:
121
+ return None
122
+ return LockHandle(name=str(name), token=token, expires_at=time.monotonic() + float(ttl_seconds))
123
+
124
+ def release_lock(self, handle) -> WriteResult:
125
+ if handle is None:
126
+ return WriteResult()
127
+ name = getattr(handle, "name", None)
128
+ token = getattr(handle, "token", None)
129
+ if not name or not token:
130
+ raise RedisConfigError("Redis lock handle requires name and token")
131
+ try:
132
+ deleted = int(self._client_instance().eval(_LOCK_RELEASE_SCRIPT, 1, self._lock_key(str(name)), str(token)) or 0)
133
+ except (RedisClientMissingError, RedisConfigError):
134
+ raise
135
+ except Exception as exc:
136
+ raise RedisConnectionError(self._safe_error(exc)) from exc
137
+ matched = 1 if deleted else 0
138
+ return WriteResult(deleted=matched, matched=matched)
139
+
140
+ def publish(self, stream: str, message: Mapping[str, Any]) -> WriteResult:
141
+ try:
142
+ message_id = self._client_instance().xadd(self._stream_key(stream), _stream_fields(message))
143
+ except (RedisClientMissingError, RedisConfigError):
144
+ raise
145
+ except Exception as exc:
146
+ raise RedisConnectionError(self._safe_error(exc)) from exc
147
+ return WriteResult(written=1, matched=1, message_id=_text(message_id))
148
+
149
+ def read(self, stream: str, cursor: str | None = None, limit: int = 100) -> StreamReadResult:
150
+ if limit <= 0:
151
+ return StreamReadResult(messages=[], cursor=cursor)
152
+ stream_name = str(stream)
153
+ stream_key = self._stream_key(stream_name)
154
+ try:
155
+ self._ensure_group(stream_key)
156
+ response = self._client_instance().xreadgroup(
157
+ self.stream_group(),
158
+ self.consumer_name(),
159
+ {stream_key: cursor or ">"},
160
+ count=max(0, int(limit)),
161
+ )
162
+ except (RedisClientMissingError, RedisConfigError):
163
+ raise
164
+ except Exception as exc:
165
+ raise RedisConnectionError(self._safe_error(exc)) from exc
166
+
167
+ messages: list[dict[str, Any]] = []
168
+ next_cursor = cursor
169
+ for _raw_name, entries in response or []:
170
+ for message_id, fields in entries:
171
+ normalized_id = _text(message_id)
172
+ messages.append(
173
+ {
174
+ "stream": stream_name,
175
+ "id": normalized_id,
176
+ **_decode_envelope(fields),
177
+ }
178
+ )
179
+ next_cursor = normalized_id
180
+ return StreamReadResult(messages=messages, cursor=next_cursor)
181
+
182
+ def ack(self, stream: str, message_id: str) -> WriteResult:
183
+ group = self.stream_group()
184
+ try:
185
+ matched = int(self._client_instance().xack(self._stream_key(stream), group, str(message_id)) or 0)
186
+ except (RedisClientMissingError, RedisConfigError):
187
+ raise
188
+ except Exception as exc:
189
+ raise RedisConnectionError(self._safe_error(exc)) from exc
190
+ return WriteResult(matched=matched)
191
+
192
+ def inspect(self) -> dict[str, Any]:
193
+ return asdict(
194
+ InspectResult(
195
+ name=self.config.name,
196
+ type=self.config.type,
197
+ capabilities=[],
198
+ initialized=self._client is not _CLIENT_UNSET,
199
+ status="ok",
200
+ options=self.config.safe_options(),
201
+ details={
202
+ "backend": "redis",
203
+ "namespace_enabled": bool(self.namespace()),
204
+ "stream_group": self.stream_group(),
205
+ },
206
+ )
207
+ )
208
+
209
+ def health(self) -> HealthResult:
210
+ try:
211
+ ping = bool(self._client_instance().ping())
212
+ except DataConfigurationError as exc:
213
+ return HealthResult(status="failed", code="configuration_error", message=str(exc))
214
+ except Exception as exc:
215
+ return HealthResult(status="failed", code="connection_failed", message=self._safe_error(exc))
216
+ if not ping:
217
+ return HealthResult(status="failed", code="connection_failed", message="Redis ping failed")
218
+ return HealthResult(status="ok", code="ok", message="Redis connection is available")
219
+
220
+ def close(self) -> None:
221
+ if self._client is _CLIENT_UNSET:
222
+ self.closed = True
223
+ return
224
+ close = getattr(self._client, "close", None)
225
+ if callable(close):
226
+ close()
227
+ self.closed = True
228
+
229
+ def native_client(self):
230
+ return self._client_instance()
231
+
232
+ def namespace(self) -> str:
233
+ return str(self.config.options.get("namespace", "")).strip(":")
234
+
235
+ def stream_group(self) -> str:
236
+ group = str(self.config.options.get("stream_group", "default"))
237
+ if not group:
238
+ raise RedisConfigError("Redis stream_group must not be empty")
239
+ return group
240
+
241
+ def consumer_name(self) -> str:
242
+ consumer = str(self.config.options.get("consumer", "muscles-worker"))
243
+ if not consumer:
244
+ raise RedisConfigError("Redis consumer must not be empty")
245
+ return consumer
246
+
247
+ def _ensure_group(self, stream_key: str) -> None:
248
+ create = getattr(self._client_instance(), "xgroup_create", None)
249
+ if not callable(create):
250
+ raise RedisConfigError("Redis client does not support consumer groups")
251
+ try:
252
+ create(stream_key, self.stream_group(), id="0-0", mkstream=True)
253
+ except Exception as exc:
254
+ if "busygroup" not in str(exc).lower() and "already exists" not in str(exc).lower():
255
+ raise RedisConnectionError(self._safe_error(exc)) from exc
256
+
257
+ def _key(self, key: str) -> str:
258
+ normalized = _name(key, "Redis key")
259
+ namespace = self.namespace()
260
+ return f"{namespace}:{normalized}" if namespace else normalized
261
+
262
+ def _lock_key(self, name: str) -> str:
263
+ return self._key(f"lock:{_name(name, 'Redis lock name')}")
264
+
265
+ def _stream_key(self, stream: str) -> str:
266
+ return self._key(f"stream:{_name(stream, 'Redis stream name')}")
267
+
268
+ def _client_instance(self):
269
+ if self._client is _CLIENT_UNSET:
270
+ with self._lock:
271
+ if self._client is _CLIENT_UNSET:
272
+ self._validate_options()
273
+ client = (
274
+ self._client_factory(self.config)
275
+ if self._client_factory
276
+ else _default_redis_client(self.config)
277
+ )
278
+ if client is None:
279
+ raise RedisClientMissingError("Redis client is not available")
280
+ self._client = client
281
+ return self._client
282
+
283
+ def _validate_options(self) -> None:
284
+ unknown = sorted(set(self.config.options) - _ALLOWED_OPTIONS)
285
+ if unknown:
286
+ names = ", ".join(unknown)
287
+ raise RedisConfigError(f"Unsupported Redis resource options: {names}")
288
+ if not self.config.options.get("url") and not self.config.options.get("url_env"):
289
+ raise RedisConfigError("Redis resource requires url_env")
290
+ for option in ("timeout", "socket_timeout", "socket_connect_timeout"):
291
+ if option in self.config.options and float(self.config.options[option]) <= 0:
292
+ raise RedisConfigError(f"Redis {option} must be positive")
293
+
294
+ def _safe_error(self, exc: Exception) -> str:
295
+ message = str(exc)
296
+ try:
297
+ url = str(self.config.resolved_options().get("url", ""))
298
+ except DataConfigurationError:
299
+ url = str(self.config.options.get("url", ""))
300
+ sensitive_values = {url}
301
+ try:
302
+ parsed = urlsplit(url)
303
+ if parsed.username:
304
+ sensitive_values.add(parsed.username)
305
+ if parsed.password:
306
+ sensitive_values.add(parsed.password)
307
+ except Exception: # pragma: no cover
308
+ pass
309
+ for value in sorted((item for item in sensitive_values if item), key=len, reverse=True):
310
+ message = message.replace(value, "***")
311
+ return message
312
+
313
+
314
+ class RedisDataFactory:
315
+ resource_type = "redis"
316
+
317
+ def __init__(self, *, client_factory: Callable[[DataResourceConfig], Any] | None = None) -> None:
318
+ self._client_factory = client_factory
319
+
320
+ def capabilities(self, config: DataResourceConfig) -> set[DataCapability]:
321
+ native = {DataCapability.NATIVE_CLIENT} if bool(config.options.get("native_client")) else set()
322
+ return {
323
+ DataCapability.KEY_VALUE,
324
+ DataCapability.CACHE,
325
+ DataCapability.LOCK,
326
+ DataCapability.STREAM,
327
+ DataCapability.HEALTHCHECK,
328
+ } | native
329
+
330
+ def create(self, config: DataResourceConfig) -> RedisDataAdapter:
331
+ return RedisDataAdapter(config, client_factory=self._client_factory)
332
+
333
+
334
+ def _default_redis_client(config: DataResourceConfig):
335
+ try:
336
+ redis_module = importlib.import_module("redis")
337
+ except ImportError as exc:
338
+ raise RedisClientMissingError("redis package is not installed; install muscles-data-redis") from exc
339
+ redis_cls = getattr(redis_module, "Redis", None)
340
+ from_url = getattr(redis_cls, "from_url", None) if redis_cls is not None else None
341
+ if from_url is None:
342
+ from_url = getattr(redis_module, "from_url", None)
343
+ if from_url is None:
344
+ raise RedisClientMissingError("redis package does not expose Redis.from_url")
345
+
346
+ options = config.resolved_options()
347
+ kwargs: dict[str, Any] = {}
348
+ kwargs["decode_responses"] = bool(options.get("decode_responses", False))
349
+ timeout = options.get("timeout")
350
+ if timeout is not None:
351
+ kwargs["socket_timeout"] = float(timeout)
352
+ kwargs["socket_connect_timeout"] = float(timeout)
353
+ for option in ("socket_timeout", "socket_connect_timeout"):
354
+ if option in options:
355
+ kwargs[option] = float(options[option])
356
+ return from_url(str(options["url"]), **kwargs)
357
+
358
+
359
+ def _ttl_ms(value: float, label: str) -> int:
360
+ seconds = float(value)
361
+ if seconds <= 0:
362
+ raise RedisConfigError(f"{label} must be positive")
363
+ return max(1, int(seconds * 1000))
364
+
365
+
366
+ def _name(value: str, label: str) -> str:
367
+ normalized = str(value).strip()
368
+ if not normalized:
369
+ raise RedisConfigError(f"{label} must not be empty")
370
+ return normalized.strip(":")
371
+
372
+
373
+ def _bytes_or_none(value: Any) -> bytes | None:
374
+ if value is None:
375
+ return None
376
+ if isinstance(value, bytes):
377
+ return value
378
+ if isinstance(value, bytearray):
379
+ return bytes(value)
380
+ if isinstance(value, memoryview):
381
+ return value.tobytes()
382
+ return str(value).encode("utf-8")
383
+
384
+
385
+ def _text(value: Any) -> str:
386
+ if isinstance(value, bytes):
387
+ return value.decode("utf-8")
388
+ return str(value)
389
+
390
+
391
+ def _stream_fields(message: Mapping[str, Any]) -> dict[str, Any]:
392
+ return {
393
+ "_schema": "muscles.data.message.v1",
394
+ "_payload": json.dumps(dict(message), ensure_ascii=False, sort_keys=True),
395
+ }
396
+
397
+
398
+ def _read_fields(fields: Mapping[Any, Any]) -> dict[str, Any]:
399
+ return {_text(key): _decode_stream_value(value) for key, value in dict(fields).items()}
400
+
401
+
402
+ def _decode_stream_value(value: Any) -> Any:
403
+ if isinstance(value, bytes):
404
+ return value.decode("utf-8")
405
+ return value
406
+
407
+
408
+ def _decode_envelope(fields: Mapping[Any, Any]) -> dict[str, Any]:
409
+ normalized = _read_fields(fields)
410
+ try:
411
+ payload = json.loads(str(normalized.get("_payload", "{}")))
412
+ except (TypeError, ValueError) as exc:
413
+ raise RedisConnectionError("Redis stream message has invalid envelope payload") from exc
414
+ if not isinstance(payload, dict):
415
+ raise RedisConnectionError("Redis stream message envelope payload must be an object")
416
+ return {
417
+ "fields": payload,
418
+ "envelope": {"version": 1, "schema": normalized.get("_schema")},
419
+ }
@@ -0,0 +1,81 @@
1
+ Metadata-Version: 2.4
2
+ Name: muscles-data-redis
3
+ Version: 0.1.0
4
+ Summary: Redis KeyValuePort, LockPort and StreamPort adapter for muscles-data
5
+ Author-email: "Denis B." <denis@butko.info>
6
+ Requires-Python: >=3.10
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: muscles-data<1.0.0,>=0.1.0
9
+ Requires-Dist: redis<9.0,>=5.0
10
+ Provides-Extra: dev
11
+ Requires-Dist: pytest<9.0,>=8.0; extra == "dev"
12
+
13
+ # muscles-data-redis
14
+
15
+ Redis adapter package for `muscles-data`.
16
+
17
+ This package is intentionally separate from `muscles-data`: the core package
18
+ owns typed ports, resource runtime and diagnostics, while this package owns
19
+ Redis-backed `KeyValuePort`, `LockPort` and `StreamPort` implementations.
20
+
21
+ ## Related packages
22
+
23
+ - Core runtime and port contracts:
24
+ [`muscles-data`](https://github.com/butkoden/muscles-data)
25
+ - Elasticsearch search adapter:
26
+ [`muscles-data-elasticsearch`](https://github.com/butkoden/muscles-data-elasticsearch)
27
+ - OpenSearch search adapter:
28
+ [`muscles-data-opensearch`](https://github.com/butkoden/muscles-data-opensearch)
29
+ - Qdrant vector adapter:
30
+ [`muscles-data-qdrant`](https://github.com/butkoden/muscles-data-qdrant)
31
+ - MongoDB document-store adapter:
32
+ [`muscles-data-mongodb`](https://github.com/butkoden/muscles-data-mongodb)
33
+ - S3 object-store adapter:
34
+ [`muscles-data-s3`](https://github.com/butkoden/muscles-data-s3)
35
+ - SQLAlchemy direct SQL resource adapter:
36
+ [`muscles-data-sqlalchemy`](https://github.com/butkoden/muscles-data-sqlalchemy)
37
+ - Executable example:
38
+ [`example_data_redis_1`](https://github.com/butkoden/muscular-example/tree/master/example_data_redis_1)
39
+
40
+ ## Usage
41
+
42
+ Register the factory in the project composition root:
43
+
44
+ ```python
45
+ from muscles_data.catalog import DataAdapterCatalog
46
+ from muscles_data.ports import KeyValuePort
47
+ from muscles_data.runtime import DataRuntime
48
+ from muscles_data_redis import RedisDataFactory
49
+
50
+ catalog = DataAdapterCatalog.with_defaults()
51
+ catalog.register(RedisDataFactory())
52
+
53
+ runtime = DataRuntime(config=config, catalog=catalog)
54
+ cache = runtime.require_port("cache.redis", KeyValuePort)
55
+ ```
56
+
57
+ Resource config stays in the project:
58
+
59
+ ```yaml
60
+ data:
61
+ resources:
62
+ cache.redis:
63
+ type: redis
64
+ url_env: REDIS_URL
65
+ namespace: app
66
+ timeout: 3
67
+ stream_group: workers
68
+ consumer: index-worker
69
+ ```
70
+
71
+ The adapter creates the Redis client lazily by key-value, lock, stream, explicit
72
+ native access or `data.doctor` operations. Application code should use
73
+ `KeyValuePort`, `LockPort` and `StreamPort`; direct client access is only an
74
+ advanced escape hatch with `native_client: true`.
75
+
76
+ Streams use `XADD`, `XREADGROUP` and `XACK`. The configured consumer group is
77
+ created on first read and messages use the versioned
78
+ `muscles.data.message.v1` envelope. Retry and dead-letter decisions stay with
79
+ the application worker.
80
+
81
+ See `muscular-example/example_data_redis_1` for an executable example.
@@ -0,0 +1,6 @@
1
+ muscles_data_redis/__init__.py,sha256=X0sGySNXFGiluZxjTFgUa6VxZ_rgL-qjOhbXy53udCI,377
2
+ muscles_data_redis/adapter.py,sha256=5ipYbvtVyI2r6MTw1PVt-plw8sLIjod5hqM5AdEsLPU,15938
3
+ muscles_data_redis-0.1.0.dist-info/METADATA,sha256=kVJzGGp5Hdo-XlQmTZjf7qiEqrQIQYYEUrDBv3ssk6Q,2868
4
+ muscles_data_redis-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
5
+ muscles_data_redis-0.1.0.dist-info/top_level.txt,sha256=20VdbjApWnZ-1-OONmNVbkGayWYt42NvwQApv1gPCAg,19
6
+ muscles_data_redis-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (83.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ muscles_data_redis