shadowcache 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shadowcache/__init__.py +31 -0
- shadowcache/core.py +360 -0
- shadowcache/exceptions.py +17 -0
- shadowcache/logger.py +39 -0
- shadowcache/parser.py +86 -0
- shadowcache/py.typed +0 -0
- shadowcache-0.1.0.dist-info/METADATA +230 -0
- shadowcache-0.1.0.dist-info/RECORD +11 -0
- shadowcache-0.1.0.dist-info/WHEEL +5 -0
- shadowcache-0.1.0.dist-info/licenses/LICENSE +21 -0
- shadowcache-0.1.0.dist-info/top_level.txt +1 -0
shadowcache/__init__.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""ShadowCache -- Transparent Redis caching for raw SQL connections.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
import mysql.connector
|
|
5
|
+
from shadowcache import ShadowCache
|
|
6
|
+
|
|
7
|
+
conn = mysql.connector.connect(host="localhost", database="mydb")
|
|
8
|
+
cache = ShadowCache(conn)
|
|
9
|
+
|
|
10
|
+
# SELECTs are transparently cached
|
|
11
|
+
cursor, rows = cache.execute("SELECT * FROM users WHERE id = %s", (42,))
|
|
12
|
+
|
|
13
|
+
# INSERT/UPDATE/DELETE automatically evict related cache entries
|
|
14
|
+
cache.execute("UPDATE users SET name = %s WHERE id = %s", ("Alice", 42))
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from shadowcache.exceptions import ShadowCacheError
|
|
18
|
+
|
|
19
|
+
__all__ = ["ShadowCache", "ShadowCacheError"]
|
|
20
|
+
__version__ = "0.1.0"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def __getattr__(name):
|
|
24
|
+
"""Lazily import ShadowCache so the package is usable even when
|
|
25
|
+
only a subset of modules have been installed."""
|
|
26
|
+
if name == "ShadowCache":
|
|
27
|
+
from shadowcache.core import ShadowCache as _sc
|
|
28
|
+
|
|
29
|
+
return _sc
|
|
30
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
31
|
+
|
shadowcache/core.py
ADDED
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""Core ShadowCache class -- transparent Redis caching for raw SQL connections."""
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
from decimal import Decimal
|
|
7
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
8
|
+
|
|
9
|
+
import redis
|
|
10
|
+
|
|
11
|
+
from shadowcache.exceptions import CacheBackendError
|
|
12
|
+
from shadowcache.logger import get_logger
|
|
13
|
+
from shadowcache.parser import extract_tables, extract_write_type, is_select_query
|
|
14
|
+
|
|
15
|
+
_log = get_logger(__name__)
|
|
16
|
+
|
|
17
|
+
_DEFAULT_KEY_PREFIX = "shadowcache"
|
|
18
|
+
|
|
19
|
+
# Sentinel keys for type-tagged JSON serialization.
|
|
20
|
+
# Each non-native value is encoded as a 3-element list:
|
|
21
|
+
# ["__sc__", "<type_name>", "<payload>"]
|
|
22
|
+
# A list is used because it is far less likely to appear as a real column
|
|
23
|
+
# value than a dict with particular keys.
|
|
24
|
+
_SC_SENTINEL = "__sc__"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _param_repr(value: Any) -> str:
|
|
28
|
+
"""Produce a type-aware string representation of a parameter value.
|
|
29
|
+
|
|
30
|
+
Prefixes the value with its Python type name so that ``1`` (int) and
|
|
31
|
+
``"1"`` (str) produce different cache keys.
|
|
32
|
+
"""
|
|
33
|
+
return f"{type(value).__name__}:{value}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _build_cache_key(prefix: str, sql: str, params: Optional[tuple]) -> str:
|
|
37
|
+
"""Produce a deterministic cache key from a SQL string and its parameters."""
|
|
38
|
+
raw = sql
|
|
39
|
+
if params:
|
|
40
|
+
raw += "|" + "|".join(_param_repr(p) for p in params)
|
|
41
|
+
digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
42
|
+
return f"{prefix}:{digest}"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _serialize(rows: List[Dict[str, Any]]) -> str:
|
|
46
|
+
"""JSON-serialise rows, handling non-native types via sentinel lists.
|
|
47
|
+
|
|
48
|
+
Supported types: bytes, datetime, date, time, timedelta, Decimal.
|
|
49
|
+
Each is encoded as ``["__sc__", "<type>", "<payload>"]``.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def _default(obj: Any) -> Any:
|
|
53
|
+
if isinstance(obj, bytes):
|
|
54
|
+
return [_SC_SENTINEL, "bytes", obj.hex()]
|
|
55
|
+
if isinstance(obj, datetime.datetime):
|
|
56
|
+
return [_SC_SENTINEL, "datetime", obj.isoformat()]
|
|
57
|
+
if isinstance(obj, datetime.date):
|
|
58
|
+
return [_SC_SENTINEL, "date", obj.isoformat()]
|
|
59
|
+
if isinstance(obj, datetime.time):
|
|
60
|
+
return [_SC_SENTINEL, "time", obj.isoformat()]
|
|
61
|
+
if isinstance(obj, datetime.timedelta):
|
|
62
|
+
return [_SC_SENTINEL, "timedelta", obj.total_seconds()]
|
|
63
|
+
if isinstance(obj, Decimal):
|
|
64
|
+
return [_SC_SENTINEL, "Decimal", str(obj)]
|
|
65
|
+
raise TypeError(f"Unsupported type: {type(obj)}")
|
|
66
|
+
|
|
67
|
+
return json.dumps(rows, default=_default)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _deserialize(payload: str) -> List[Dict[str, Any]]:
|
|
71
|
+
"""Deserialise rows, restoring non-native types from sentinel lists.
|
|
72
|
+
|
|
73
|
+
Recursively walks the decoded JSON tree. Any three-element list whose
|
|
74
|
+
first element is ``_SC_SENTINEL`` is converted back to its native type;
|
|
75
|
+
all other values pass through unchanged.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def _walk(obj: Any) -> Any:
|
|
79
|
+
if isinstance(obj, list):
|
|
80
|
+
if len(obj) == 3 and obj and obj[0] == _SC_SENTINEL:
|
|
81
|
+
_, t, d = obj
|
|
82
|
+
if t == "bytes":
|
|
83
|
+
return bytes.fromhex(d)
|
|
84
|
+
if t == "datetime":
|
|
85
|
+
return datetime.datetime.fromisoformat(d)
|
|
86
|
+
if t == "date":
|
|
87
|
+
return datetime.date.fromisoformat(d)
|
|
88
|
+
if t == "time":
|
|
89
|
+
return datetime.time.fromisoformat(d)
|
|
90
|
+
if t == "timedelta":
|
|
91
|
+
return datetime.timedelta(seconds=d)
|
|
92
|
+
if t == "Decimal":
|
|
93
|
+
return Decimal(d)
|
|
94
|
+
return [_walk(item) for item in obj]
|
|
95
|
+
if isinstance(obj, dict):
|
|
96
|
+
return {k: _walk(v) for k, v in obj.items()}
|
|
97
|
+
return obj
|
|
98
|
+
|
|
99
|
+
return _walk(json.loads(payload))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class ShadowCache:
|
|
103
|
+
"""Transparent Redis cache layer for a DB-API2 MySQL connection.
|
|
104
|
+
|
|
105
|
+
Parameters
|
|
106
|
+
----------
|
|
107
|
+
db_connection:
|
|
108
|
+
An open DB-API2 connection (e.g. from ``mysql.connector.connect``).
|
|
109
|
+
redis_client:
|
|
110
|
+
An optional pre-configured ``redis.Redis`` instance. If omitted a
|
|
111
|
+
client bound to ``redis_host``/``redis_port`` is created.
|
|
112
|
+
redis_host:
|
|
113
|
+
Hostname for Redis (ignored when *redis_client* is supplied).
|
|
114
|
+
redis_port:
|
|
115
|
+
Port for Redis (ignored when *redis_client* is supplied).
|
|
116
|
+
ttl:
|
|
117
|
+
Time-to-live in seconds for cached SELECT results. Default 300.
|
|
118
|
+
auto_invalidate:
|
|
119
|
+
When ``True`` (the default), INSERT/UPDATE/DELETE statements
|
|
120
|
+
automatically evict cached SELECT results that reference the same
|
|
121
|
+
table.
|
|
122
|
+
key_prefix:
|
|
123
|
+
Namespace prefix for all Redis keys. Change this to share a Redis
|
|
124
|
+
instance across multiple applications.
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
def __init__(
|
|
128
|
+
self,
|
|
129
|
+
db_connection,
|
|
130
|
+
redis_client: Optional[redis.Redis] = None,
|
|
131
|
+
*,
|
|
132
|
+
redis_host: str = "localhost",
|
|
133
|
+
redis_port: int = 6379,
|
|
134
|
+
ttl: int = 300,
|
|
135
|
+
auto_invalidate: bool = True,
|
|
136
|
+
key_prefix: str = _DEFAULT_KEY_PREFIX,
|
|
137
|
+
):
|
|
138
|
+
self._db = db_connection
|
|
139
|
+
self._ttl = ttl
|
|
140
|
+
self._auto_invalidate = auto_invalidate
|
|
141
|
+
self._key_prefix = key_prefix
|
|
142
|
+
self._index_prefix = f"{key_prefix}:index"
|
|
143
|
+
|
|
144
|
+
self._hits = 0
|
|
145
|
+
self._misses = 0
|
|
146
|
+
|
|
147
|
+
if redis_client is not None:
|
|
148
|
+
self._redis = redis_client
|
|
149
|
+
else:
|
|
150
|
+
try:
|
|
151
|
+
self._redis = redis.Redis(
|
|
152
|
+
host=redis_host,
|
|
153
|
+
port=redis_port,
|
|
154
|
+
decode_responses=True,
|
|
155
|
+
)
|
|
156
|
+
self._redis.ping()
|
|
157
|
+
except Exception as exc:
|
|
158
|
+
raise CacheBackendError(
|
|
159
|
+
f"Could not connect to Redis at {redis_host}:{redis_port}"
|
|
160
|
+
) from exc
|
|
161
|
+
|
|
162
|
+
# ---------------------------------------------------------------- public API
|
|
163
|
+
|
|
164
|
+
def execute(
|
|
165
|
+
self,
|
|
166
|
+
sql: str,
|
|
167
|
+
params: Optional[tuple] = None,
|
|
168
|
+
) -> Tuple[Any, Optional[List[Dict[str, Any]]]]:
|
|
169
|
+
"""Execute *sql* with optional *params*, applying caching automatically.
|
|
170
|
+
|
|
171
|
+
Returns ``(cursor, rows)``.
|
|
172
|
+
|
|
173
|
+
*For SELECT statements* the cache is consulted first. On a hit
|
|
174
|
+
``cursor`` is ``None`` and *rows* contains the cached result. On a
|
|
175
|
+
miss the query runs against MySQL, the result is stored in Redis,
|
|
176
|
+
and both the live cursor and rows are returned.
|
|
177
|
+
|
|
178
|
+
*For INSERT/UPDATE/DELETE* the statement runs against MySQL and any
|
|
179
|
+
cached SELECT results that reference the affected tables are
|
|
180
|
+
evicted. ``rows`` is ``None``; the caller inspects ``cursor`` for
|
|
181
|
+
``.lastrowid`` or ``.rowcount``.
|
|
182
|
+
|
|
183
|
+
*For all other statements* (DDL, administrative commands) the
|
|
184
|
+
statement is forwarded to MySQL unchanged.
|
|
185
|
+
"""
|
|
186
|
+
if not sql or not sql.strip():
|
|
187
|
+
return None, None
|
|
188
|
+
|
|
189
|
+
sql = sql.strip()
|
|
190
|
+
|
|
191
|
+
if is_select_query(sql):
|
|
192
|
+
return self._handle_select(sql, params)
|
|
193
|
+
|
|
194
|
+
write_type = extract_write_type(sql)
|
|
195
|
+
if write_type is not None:
|
|
196
|
+
return self._handle_write(sql, params, write_type)
|
|
197
|
+
|
|
198
|
+
return self._handle_other(sql, params)
|
|
199
|
+
|
|
200
|
+
def invalidate_table(self, table_name: str) -> int:
|
|
201
|
+
"""Evict all cached entries associated with *table_name*.
|
|
202
|
+
|
|
203
|
+
Returns the number of keys removed.
|
|
204
|
+
"""
|
|
205
|
+
index_key = f"{self._index_prefix}:{table_name}"
|
|
206
|
+
try:
|
|
207
|
+
members = self._redis.smembers(index_key)
|
|
208
|
+
if not members:
|
|
209
|
+
return 0
|
|
210
|
+
keys = list(members)
|
|
211
|
+
keys.append(index_key)
|
|
212
|
+
return self._redis.delete(*keys)
|
|
213
|
+
except redis.RedisError as exc:
|
|
214
|
+
_log.warning("Failed to invalidate table %r: %s", table_name, exc)
|
|
215
|
+
return 0
|
|
216
|
+
|
|
217
|
+
def flush_cache(self) -> int:
|
|
218
|
+
"""Remove all ShadowCache keys from Redis.
|
|
219
|
+
|
|
220
|
+
Returns the number of keys removed.
|
|
221
|
+
"""
|
|
222
|
+
pattern = f"{self._key_prefix}:*"
|
|
223
|
+
try:
|
|
224
|
+
keys = list(self._redis.scan_iter(match=pattern, count=100))
|
|
225
|
+
if not keys:
|
|
226
|
+
return 0
|
|
227
|
+
return self._redis.delete(*keys)
|
|
228
|
+
except redis.RedisError as exc:
|
|
229
|
+
_log.warning("Failed to flush cache: %s", exc)
|
|
230
|
+
return 0
|
|
231
|
+
|
|
232
|
+
def close(self) -> None:
|
|
233
|
+
"""Close the wrapped database connection."""
|
|
234
|
+
try:
|
|
235
|
+
if hasattr(self._db, "close"):
|
|
236
|
+
self._db.close()
|
|
237
|
+
except Exception as exc:
|
|
238
|
+
_log.warning("Error closing database connection: %s", exc)
|
|
239
|
+
|
|
240
|
+
@property
|
|
241
|
+
def stats(self) -> Dict[str, Any]:
|
|
242
|
+
"""Return a snapshot of cache performance counters."""
|
|
243
|
+
total = self._hits + self._misses
|
|
244
|
+
return {
|
|
245
|
+
"hits": self._hits,
|
|
246
|
+
"misses": self._misses,
|
|
247
|
+
"total_requests": total,
|
|
248
|
+
"hit_ratio": self._hits / total if total else 0.0,
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
# --------------------------------------------------------------- internals
|
|
252
|
+
|
|
253
|
+
def _handle_select(self, sql: str, params):
|
|
254
|
+
cache_key = _build_cache_key(self._key_prefix, sql, params)
|
|
255
|
+
|
|
256
|
+
# Try Redis first.
|
|
257
|
+
try:
|
|
258
|
+
cached = self._redis.get(cache_key)
|
|
259
|
+
except Exception as exc:
|
|
260
|
+
_log.warning("Redis read error, falling through to MySQL: %s", exc)
|
|
261
|
+
cached = None
|
|
262
|
+
|
|
263
|
+
if cached is not None:
|
|
264
|
+
try:
|
|
265
|
+
rows = _deserialize(cached)
|
|
266
|
+
except Exception as exc:
|
|
267
|
+
_log.warning(
|
|
268
|
+
"Cache deserialisation failed for key %r, falling through to MySQL: %s",
|
|
269
|
+
cache_key,
|
|
270
|
+
exc,
|
|
271
|
+
)
|
|
272
|
+
else:
|
|
273
|
+
self._hits += 1
|
|
274
|
+
_log.info("Cache HIT for key %s", cache_key)
|
|
275
|
+
return None, rows
|
|
276
|
+
|
|
277
|
+
self._misses += 1
|
|
278
|
+
_log.info("Cache MISS for key %s -- fetching from MySQL", cache_key)
|
|
279
|
+
|
|
280
|
+
cursor, rows = self._handle_other(sql, params)
|
|
281
|
+
if rows is None:
|
|
282
|
+
return cursor, None
|
|
283
|
+
|
|
284
|
+
# Store in Redis and update per-table indexes.
|
|
285
|
+
tables = extract_tables(sql)
|
|
286
|
+
self._store_result(cache_key, rows, tables)
|
|
287
|
+
return cursor, rows
|
|
288
|
+
|
|
289
|
+
def _handle_write(self, sql: str, params, write_type: str):
|
|
290
|
+
cursor, _ = self._handle_other(sql, params)
|
|
291
|
+
|
|
292
|
+
if self._auto_invalidate:
|
|
293
|
+
tables = extract_tables(sql)
|
|
294
|
+
for table in tables:
|
|
295
|
+
self.invalidate_table(table)
|
|
296
|
+
_log.debug("%s executed, tables evicted: %s", write_type, tables)
|
|
297
|
+
|
|
298
|
+
return cursor, None
|
|
299
|
+
|
|
300
|
+
def _handle_other(self, sql: str, params):
|
|
301
|
+
"""Execute SQL directly against MySQL, returning (cursor, rows).
|
|
302
|
+
|
|
303
|
+
Rows are normalised to ``list[dict]`` regardless of whether the
|
|
304
|
+
underlying driver returns tuples or dicts. The conversion uses
|
|
305
|
+
``cursor.description`` (standard DB-API2) so the wrapper works
|
|
306
|
+
with ``mysql-connector-python``, ``pymysql``, and other PEP 249
|
|
307
|
+
drivers.
|
|
308
|
+
"""
|
|
309
|
+
try:
|
|
310
|
+
cursor = self._db.cursor()
|
|
311
|
+
if params:
|
|
312
|
+
cursor.execute(sql, params)
|
|
313
|
+
else:
|
|
314
|
+
cursor.execute(sql)
|
|
315
|
+
except Exception:
|
|
316
|
+
try:
|
|
317
|
+
cursor.close()
|
|
318
|
+
except Exception:
|
|
319
|
+
pass
|
|
320
|
+
raise
|
|
321
|
+
|
|
322
|
+
try:
|
|
323
|
+
raw_rows = cursor.fetchall()
|
|
324
|
+
except Exception:
|
|
325
|
+
raw_rows = None
|
|
326
|
+
|
|
327
|
+
if raw_rows and cursor.description:
|
|
328
|
+
first_row = raw_rows[0]
|
|
329
|
+
if isinstance(first_row, dict):
|
|
330
|
+
rows = raw_rows
|
|
331
|
+
else:
|
|
332
|
+
columns = [col[0] for col in cursor.description]
|
|
333
|
+
rows = [dict(zip(columns, row)) for row in raw_rows]
|
|
334
|
+
elif raw_rows is not None:
|
|
335
|
+
rows = [] # empty result set
|
|
336
|
+
else:
|
|
337
|
+
rows = None # fetchall failed on a non-result-set statement
|
|
338
|
+
|
|
339
|
+
return cursor, rows
|
|
340
|
+
|
|
341
|
+
def _store_result(self, cache_key: str, rows, tables: set):
|
|
342
|
+
try:
|
|
343
|
+
payload = _serialize(rows)
|
|
344
|
+
except Exception as exc:
|
|
345
|
+
_log.warning("Failed to serialise result for key %r: %s", cache_key, exc)
|
|
346
|
+
return
|
|
347
|
+
|
|
348
|
+
try:
|
|
349
|
+
self._redis.set(cache_key, payload, ex=self._ttl)
|
|
350
|
+
except redis.RedisError as exc:
|
|
351
|
+
_log.warning("Failed to store cache key %r: %s", cache_key, exc)
|
|
352
|
+
return
|
|
353
|
+
|
|
354
|
+
for table in tables:
|
|
355
|
+
index_key = f"{self._index_prefix}:{table}"
|
|
356
|
+
try:
|
|
357
|
+
self._redis.sadd(index_key, cache_key)
|
|
358
|
+
self._redis.expire(index_key, self._ttl)
|
|
359
|
+
except redis.RedisError as exc:
|
|
360
|
+
_log.debug("Failed to update index for table %r: %s", table, exc)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Custom exceptions for ShadowCache."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ShadowCacheError(Exception):
|
|
5
|
+
"""Base exception for all ShadowCache errors."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CacheParseError(ShadowCacheError):
|
|
9
|
+
"""Raised when a SQL statement cannot be parsed by sqlglot.
|
|
10
|
+
|
|
11
|
+
The original SQL is still executed against MySQL; only the caching
|
|
12
|
+
or invalidation step is skipped.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CacheBackendError(ShadowCacheError):
|
|
17
|
+
"""Raised when Redis or MySQL is unreachable."""
|
shadowcache/logger.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Logging setup for ShadowCache.
|
|
2
|
+
|
|
3
|
+
Provides a get_logger() factory so consumers can obtain a configured
|
|
4
|
+
logger without relying on a module-level singleton.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
import os
|
|
9
|
+
|
|
10
|
+
_log_format = logging.Formatter(
|
|
11
|
+
"%(asctime)s - %(levelname)s - %(name)s:%(lineno)d - %(message)s"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
_loggers: dict[str, logging.Logger] = {}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def get_logger(name: str = "ShadowCache") -> logging.Logger:
|
|
18
|
+
"""Return a configured logger for the given name.
|
|
19
|
+
|
|
20
|
+
Log level is read from the LOG_LEVEL environment variable and defaults
|
|
21
|
+
to INFO. A StreamHandler writing to stderr is attached on the first
|
|
22
|
+
call for each *name*.
|
|
23
|
+
"""
|
|
24
|
+
if name in _loggers:
|
|
25
|
+
return _loggers[name]
|
|
26
|
+
|
|
27
|
+
level_name = os.getenv("LOG_LEVEL", "INFO").upper()
|
|
28
|
+
level = getattr(logging, level_name, logging.INFO)
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger(name)
|
|
31
|
+
logger.setLevel(level)
|
|
32
|
+
|
|
33
|
+
if not logger.handlers:
|
|
34
|
+
console = logging.StreamHandler()
|
|
35
|
+
console.setFormatter(_log_format)
|
|
36
|
+
logger.addHandler(console)
|
|
37
|
+
|
|
38
|
+
_loggers[name] = logger
|
|
39
|
+
return logger
|
shadowcache/parser.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""SQL parsing utilities using sqlglot.
|
|
2
|
+
|
|
3
|
+
Extracts table names and statement type information from raw SQL
|
|
4
|
+
strings so the core module can decide whether to cache or invalidate.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from typing import Optional, Set
|
|
8
|
+
|
|
9
|
+
import sqlglot
|
|
10
|
+
from sqlglot import exp
|
|
11
|
+
|
|
12
|
+
from shadowcache.logger import get_logger
|
|
13
|
+
|
|
14
|
+
_log = get_logger(__name__)
|
|
15
|
+
|
|
16
|
+
_READ_TYPES = frozenset({
|
|
17
|
+
exp.Select,
|
|
18
|
+
exp.Describe,
|
|
19
|
+
})
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _get_root_expression(sql: str) -> Optional[exp.Expression]:
|
|
23
|
+
"""Parse *sql* and return the root AST node, or None on failure."""
|
|
24
|
+
try:
|
|
25
|
+
return sqlglot.parse_one(sql, error_level=sqlglot.ErrorLevel.IGNORE)
|
|
26
|
+
except Exception:
|
|
27
|
+
_log.debug("sqlglot could not parse: %s", sql[:120])
|
|
28
|
+
return None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def extract_tables(sql: str) -> Set[str]:
|
|
32
|
+
"""Return the set of table names referenced in *sql*.
|
|
33
|
+
|
|
34
|
+
Handles SELECT, INSERT, UPDATE, DELETE, TRUNCATE, JOINs,
|
|
35
|
+
sub-queries, and schema-qualified names like ``shop.users``.
|
|
36
|
+
|
|
37
|
+
Returns an empty set if the SQL cannot be parsed, so callers can
|
|
38
|
+
safely fall back to TTL-based expiry.
|
|
39
|
+
"""
|
|
40
|
+
root = _get_root_expression(sql)
|
|
41
|
+
if root is None:
|
|
42
|
+
return set()
|
|
43
|
+
|
|
44
|
+
tables: Set[str] = set()
|
|
45
|
+
for table_node in root.find_all(exp.Table):
|
|
46
|
+
name = table_node.name
|
|
47
|
+
if not name:
|
|
48
|
+
continue
|
|
49
|
+
if table_node.db:
|
|
50
|
+
tables.add(f"{table_node.db}.{name}")
|
|
51
|
+
else:
|
|
52
|
+
tables.add(name)
|
|
53
|
+
return tables
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def extract_write_type(sql: str) -> Optional[str]:
|
|
57
|
+
"""Determine the write category of *sql*.
|
|
58
|
+
|
|
59
|
+
Returns
|
|
60
|
+
-------
|
|
61
|
+
``"INSERT"``, ``"UPDATE"``, ``"DELETE"``, or ``None`` if the
|
|
62
|
+
statement does not modify data (SELECT, SHOW, DDL, etc.).
|
|
63
|
+
TRUNCATE is treated as a write (returns ``"DELETE"``).
|
|
64
|
+
"""
|
|
65
|
+
root = _get_root_expression(sql)
|
|
66
|
+
if root is None:
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
root_type = type(root)
|
|
70
|
+
|
|
71
|
+
if root_type is exp.Insert:
|
|
72
|
+
return "INSERT"
|
|
73
|
+
if root_type is exp.Update:
|
|
74
|
+
return "UPDATE"
|
|
75
|
+
if root_type in (exp.Delete, exp.TruncateTable):
|
|
76
|
+
return "DELETE"
|
|
77
|
+
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def is_select_query(sql: str) -> bool:
|
|
82
|
+
"""Return True if *sql* is a SELECT (or SELECT-like) statement."""
|
|
83
|
+
root = _get_root_expression(sql)
|
|
84
|
+
if root is None:
|
|
85
|
+
return False
|
|
86
|
+
return type(root) in _READ_TYPES
|
shadowcache/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: shadowcache
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Transparent Redis caching for raw SQL connections - no ORM required
|
|
5
|
+
Author: Pratham Bhosale
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/pratham2402/ShadowCache
|
|
8
|
+
Project-URL: Repository, https://github.com/pratham2402/ShadowCache
|
|
9
|
+
Keywords: redis,mysql,cache,sql,caching,database,db-api,transparent-cache,query-cache
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Database
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: redis>=4.0
|
|
24
|
+
Requires-Dist: mysql-connector-python>=8.0
|
|
25
|
+
Requires-Dist: sqlglot>=20.0
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
|
|
30
|
+
<p align="center">
|
|
31
|
+
<img src="https://img.shields.io/static/v1?label=%F0%9F%8C%9F&message=If%20Useful&style=flat&color=BC4E99">
|
|
32
|
+
<img src="https://badges.frapsoft.com/os/v1/open-source.svg?v=103">
|
|
33
|
+
<a href="https://github.com/pratham2402"><img src="https://img.shields.io/badge/View-My_Profile-green?logo=GitHub"></a>
|
|
34
|
+
<a href="https://github.com/pratham2402?tab=repositories"><img src="https://img.shields.io/badge/View-My_Repositories-blue?logo=GitHub"></a>
|
|
35
|
+
</p>
|
|
36
|
+
|
|
37
|
+
<p align="center">
|
|
38
|
+
<img src="https://img.shields.io/badge/python-3.8+-blue?logo=python">
|
|
39
|
+
<img src="https://img.shields.io/badge/license-MIT-green">
|
|
40
|
+
<img src="https://img.shields.io/badge/platform-mysql%20%7C%20redis-red">
|
|
41
|
+
</p>
|
|
42
|
+
|
|
43
|
+
# ShadowCache
|
|
44
|
+
|
|
45
|
+
<p align="center">
|
|
46
|
+
<img src="./README%20Banner%20Art.png" alt="ShadowCache Banner">
|
|
47
|
+
</p>
|
|
48
|
+
|
|
49
|
+
> **Write SQL. Get caching. Nothing else.**
|
|
50
|
+
|
|
51
|
+
ShadowCache wraps your MySQL connection and transparently caches SELECT results
|
|
52
|
+
in Redis. INSERT, UPDATE, or DELETE statements automatically evict affected cache
|
|
53
|
+
entries so your reads never serve stale data. No ORM. No boilerplate. No config.
|
|
54
|
+
|
|
55
|
+
<br>
|
|
56
|
+
|
|
57
|
+
<details open>
|
|
58
|
+
<summary><b>Table of Contents</b></summary>
|
|
59
|
+
|
|
60
|
+
- [The Problem](#the-problem)
|
|
61
|
+
- [Features](#features)
|
|
62
|
+
- [Installation](#installation)
|
|
63
|
+
- [Quick Start](#quick-start)
|
|
64
|
+
- [API Reference](#api-reference)
|
|
65
|
+
- [Configuration](#configuration)
|
|
66
|
+
- [Running Tests](#running-tests)
|
|
67
|
+
- [License](#license)
|
|
68
|
+
|
|
69
|
+
</details>
|
|
70
|
+
|
|
71
|
+
## The Problem
|
|
72
|
+
|
|
73
|
+
Every developer who writes raw SQL eventually writes this:
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
# 8 lines of boilerplate for every cached query
|
|
77
|
+
cache_key = f"user:{user_id}"
|
|
78
|
+
cached = redis.get(cache_key)
|
|
79
|
+
if cached:
|
|
80
|
+
return json.loads(cached)
|
|
81
|
+
|
|
82
|
+
cursor.execute("SELECT * FROM users WHERE id = %s", (user_id,))
|
|
83
|
+
row = cursor.fetchone()
|
|
84
|
+
redis.set(cache_key, json.dumps(row), ex=300)
|
|
85
|
+
return row
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
And on every INSERT, UPDATE, or DELETE you need to remember:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
cursor.execute("UPDATE users SET name = %s WHERE id = %s", (name, user_id))
|
|
92
|
+
redis.delete(f"user:{user_id}") # easy to forget, easy to get wrong
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
```diff
|
|
96
|
+
- Boilerplate for every query
|
|
97
|
+
- Manual invalidation you will forget
|
|
98
|
+
+ One line. Caching and invalidation are automatic.
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
**With ShadowCache:**
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
cursor, rows = cache.execute("SELECT * FROM users WHERE id = %s", (42,))
|
|
105
|
+
cursor, _ = cache.execute("UPDATE users SET name = %s WHERE id = %s", ("Alice", 42))
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Features
|
|
109
|
+
|
|
110
|
+
| | |
|
|
111
|
+
|---|---|
|
|
112
|
+
| **Zero-schema caching** | Works with any MySQL table, any query. No model definitions needed. |
|
|
113
|
+
| **Write-triggered eviction** | INSERT, UPDATE, and DELETE automatically evict cached SELECTs for the same table. |
|
|
114
|
+
| **TTL safety net** | Cached entries expire after a configurable time-to-live. Eventual consistency guaranteed. |
|
|
115
|
+
| **Graceful fallback** | If Redis is unreachable, queries still execute against MySQL. |
|
|
116
|
+
|
|
117
|
+
## Installation
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
pip install shadowcache
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
> Or install from source:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
git clone https://github.com/pratham2402/ShadowCache.git
|
|
127
|
+
cd ShadowCache
|
|
128
|
+
pip install -r requirements.txt
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
**Requires:** Python 3.8+, Redis, MySQL.
|
|
132
|
+
|
|
133
|
+
## Quick Start
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
import mysql.connector
|
|
137
|
+
from shadowcache import ShadowCache
|
|
138
|
+
|
|
139
|
+
conn = mysql.connector.connect(
|
|
140
|
+
host="localhost", database="my_app",
|
|
141
|
+
user="app_user", password="secret",
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
cache = ShadowCache(conn)
|
|
145
|
+
|
|
146
|
+
# Cold miss -- hits MySQL, stores in Redis
|
|
147
|
+
cursor, rows = cache.execute("SELECT * FROM users WHERE id = %s", (42,))
|
|
148
|
+
|
|
149
|
+
# Warm hit -- returns from Redis instantly
|
|
150
|
+
cursor, rows = cache.execute("SELECT * FROM users WHERE id = %s", (42,))
|
|
151
|
+
|
|
152
|
+
# Write evicts the cache
|
|
153
|
+
cache.execute("UPDATE users SET name = %s WHERE id = %s", ("Alice", 42))
|
|
154
|
+
|
|
155
|
+
# Cache was evicted -- fresh data from MySQL
|
|
156
|
+
cursor, rows = cache.execute("SELECT * FROM users WHERE id = %s", (42,))
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## API Reference
|
|
160
|
+
|
|
161
|
+
### `ShadowCache(db_connection, *, ...)`
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
ShadowCache(
|
|
165
|
+
db_connection,
|
|
166
|
+
*,
|
|
167
|
+
redis_client=None,
|
|
168
|
+
redis_host="localhost",
|
|
169
|
+
redis_port=6379,
|
|
170
|
+
ttl=300,
|
|
171
|
+
auto_invalidate=True,
|
|
172
|
+
key_prefix="shadowcache",
|
|
173
|
+
)
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
| Parameter | Default | Description |
|
|
177
|
+
|---|---|---|
|
|
178
|
+
| `db_connection` | *(required)* | An open DB-API2 MySQL connection |
|
|
179
|
+
| `redis_client` | `None` | Pre-configured `redis.Redis` instance; created automatically if omitted |
|
|
180
|
+
| `redis_host` | `"localhost"` | Redis hostname |
|
|
181
|
+
| `redis_port` | `6379` | Redis port |
|
|
182
|
+
| `ttl` | `300` | Cache TTL in seconds |
|
|
183
|
+
| `auto_invalidate` | `True` | Whether writes automatically evict related cache entries |
|
|
184
|
+
| `key_prefix` | `"shadowcache"` | Namespace prefix for all Redis keys |
|
|
185
|
+
|
|
186
|
+
### `ShadowCache.execute(sql, params=None)`
|
|
187
|
+
|
|
188
|
+
Returns `(cursor, rows)`.
|
|
189
|
+
|
|
190
|
+
| SQL | Behaviour |
|
|
191
|
+
|---|---|
|
|
192
|
+
| `SELECT` | Checks Redis first. Hit returns `(None, cached_rows)`. Miss executes on MySQL, caches, returns `(cursor, rows)`. |
|
|
193
|
+
| `INSERT` | Executes on MySQL. Returns `(cursor, None)`. See `cursor.lastrowid`. |
|
|
194
|
+
| `UPDATE` / `DELETE` | Executes on MySQL, evicts cache for affected tables. Returns `(cursor, None)`. See `cursor.rowcount`. |
|
|
195
|
+
| DDL / other | Executes on MySQL. No caching, no eviction. |
|
|
196
|
+
|
|
197
|
+
### Other Methods
|
|
198
|
+
|
|
199
|
+
| Method | Description |
|
|
200
|
+
|---|---|
|
|
201
|
+
| `invalidate_table(name)` | Evict all cached entries for a table. Returns count of keys removed. |
|
|
202
|
+
| `flush_cache()` | Remove all ShadowCache keys from Redis. Returns count of keys removed. |
|
|
203
|
+
| `stats` | Property. Returns a dict with keys `hits`, `misses`, `total_requests`, `hit_ratio`. |
|
|
204
|
+
| `close()` | Close the wrapped database connection. |
|
|
205
|
+
|
|
206
|
+
## Configuration
|
|
207
|
+
|
|
208
|
+
Copy `.env.example` to `.env` and set your credentials:
|
|
209
|
+
|
|
210
|
+
```
|
|
211
|
+
REDIS_HOST=localhost
|
|
212
|
+
REDIS_PORT=6379
|
|
213
|
+
MYSQL_HOST=localhost
|
|
214
|
+
MYSQL_PORT=3306
|
|
215
|
+
MYSQL_USER=your_db_user
|
|
216
|
+
MYSQL_PASSWORD=your_db_password
|
|
217
|
+
MYSQL_DATABASE=your_database
|
|
218
|
+
LOG_LEVEL=INFO
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
## Running Tests
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
# Unit tests -- no Redis or MySQL needed, all mocks
|
|
225
|
+
python -m pytest tests/ -v
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
## License
|
|
229
|
+
|
|
230
|
+
MIT
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
shadowcache/__init__.py,sha256=G1ed7W_7dmQmKFlR7YapxJqPV8aHPZAmoGHf_sECwPQ,963
|
|
2
|
+
shadowcache/core.py,sha256=6VPhu1TsyXVfjWFdXQqjer4VVJv8AsLDdzX2rOhDnK8,12588
|
|
3
|
+
shadowcache/exceptions.py,sha256=1C4cuypOUgIgzNcbKLwrllt6WER1MK7eKV-Eq-FkXZc,453
|
|
4
|
+
shadowcache/logger.py,sha256=pUcwJpXHZCnYn49GFyBKpFFfH6vY99GxAo0eI7xgl9M,1061
|
|
5
|
+
shadowcache/parser.py,sha256=Rgz5mu3KGb3IIq8Y-uNucdJ2V7GA5mjorVNz4REBQpw,2292
|
|
6
|
+
shadowcache/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
shadowcache-0.1.0.dist-info/licenses/LICENSE,sha256=BsPow-BLXSrbXdSzG4LW7LvsH2oV5lIKAV1iulFjJyI,1072
|
|
8
|
+
shadowcache-0.1.0.dist-info/METADATA,sha256=DfdAynQ8CON0_qLBylShJoHqOUJr7uoEQy8RDwIn5mU,6956
|
|
9
|
+
shadowcache-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
10
|
+
shadowcache-0.1.0.dist-info/top_level.txt,sha256=FQmAUbmMq91kPaIW3DzZ9JsARkPmC3l4oesbRSozZQY,12
|
|
11
|
+
shadowcache-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Pratham Bhosale
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
shadowcache
|