gpdatacached 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gpdatacached-0.1.0/.gitignore +197 -0
- gpdatacached-0.1.0/LICENSE +21 -0
- gpdatacached-0.1.0/PKG-INFO +171 -0
- gpdatacached-0.1.0/README.md +134 -0
- gpdatacached-0.1.0/README.zh-CN.md +134 -0
- gpdatacached-0.1.0/docs/en/api.md +698 -0
- gpdatacached-0.1.0/docs/en/extensibility.md +586 -0
- gpdatacached-0.1.0/docs/en/index.md +41 -0
- gpdatacached-0.1.0/docs/en/lifecycle.md +385 -0
- gpdatacached-0.1.0/docs/en/locking.md +198 -0
- gpdatacached-0.1.0/docs/en/overview.md +403 -0
- gpdatacached-0.1.0/docs/en/pandas-support.md +239 -0
- gpdatacached-0.1.0/docs/en/performance-guide.md +327 -0
- gpdatacached-0.1.0/docs/en/pydantic-support.md +236 -0
- gpdatacached-0.1.0/docs/zh/api.md +698 -0
- gpdatacached-0.1.0/docs/zh/extensibility.md +586 -0
- gpdatacached-0.1.0/docs/zh/index.md +41 -0
- gpdatacached-0.1.0/docs/zh/lifecycle.md +385 -0
- gpdatacached-0.1.0/docs/zh/locking.md +198 -0
- gpdatacached-0.1.0/docs/zh/overview.md +403 -0
- gpdatacached-0.1.0/docs/zh/pandas-support.md +239 -0
- gpdatacached-0.1.0/docs/zh/performance-guide.md +327 -0
- gpdatacached-0.1.0/docs/zh/pydantic-support.md +236 -0
- gpdatacached-0.1.0/pyproject.toml +80 -0
- gpdatacached-0.1.0/src/gpdatacached/__init__.py +79 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/__init__.py +3 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/bool_object.py +25 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/datetime_object.py +72 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/dict_object.py +195 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/float_object.py +26 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/int_object.py +26 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/list_object.py +232 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/none_object.py +25 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/registry.py +110 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/set_object.py +161 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/str_object.py +23 -0
- gpdatacached-0.1.0/src/gpdatacached/builtins/tuple_object.py +49 -0
- gpdatacached-0.1.0/src/gpdatacached/codec.py +47 -0
- gpdatacached-0.1.0/src/gpdatacached/config.py +101 -0
- gpdatacached-0.1.0/src/gpdatacached/container.py +205 -0
- gpdatacached-0.1.0/src/gpdatacached/domain.py +362 -0
- gpdatacached-0.1.0/src/gpdatacached/exceptions.py +112 -0
- gpdatacached-0.1.0/src/gpdatacached/extensions/__init__.py +0 -0
- gpdatacached-0.1.0/src/gpdatacached/extensions/pandas_support.py +1635 -0
- gpdatacached-0.1.0/src/gpdatacached/extensions/pydantic_support.py +169 -0
- gpdatacached-0.1.0/src/gpdatacached/gc.py +282 -0
- gpdatacached-0.1.0/src/gpdatacached/keys.py +150 -0
- gpdatacached-0.1.0/src/gpdatacached/lifecycle.py +171 -0
- gpdatacached-0.1.0/src/gpdatacached/locking.py +163 -0
- gpdatacached-0.1.0/src/gpdatacached/meta.py +126 -0
- gpdatacached-0.1.0/src/gpdatacached/namespace.py +602 -0
- gpdatacached-0.1.0/src/gpdatacached/object.py +318 -0
- gpdatacached-0.1.0/src/gpdatacached/refset.py +54 -0
- gpdatacached-0.1.0/src/gpdatacached/scalar.py +56 -0
- gpdatacached-0.1.0/src/gpdatacached/types.py +296 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# git
|
|
10
|
+
.git/
|
|
11
|
+
.worktrees/
|
|
12
|
+
|
|
13
|
+
# Distribution / packaging
|
|
14
|
+
.Python
|
|
15
|
+
build/
|
|
16
|
+
develop-eggs/
|
|
17
|
+
dist/
|
|
18
|
+
downloads/
|
|
19
|
+
eggs/
|
|
20
|
+
.eggs/
|
|
21
|
+
lib/
|
|
22
|
+
lib64/
|
|
23
|
+
parts/
|
|
24
|
+
sdist/
|
|
25
|
+
var/
|
|
26
|
+
wheels/
|
|
27
|
+
share/python-wheels/
|
|
28
|
+
*.egg-info/
|
|
29
|
+
.installed.cfg
|
|
30
|
+
*.egg
|
|
31
|
+
MANIFEST
|
|
32
|
+
|
|
33
|
+
# PyInstaller
|
|
34
|
+
*.manifest
|
|
35
|
+
*.spec
|
|
36
|
+
|
|
37
|
+
# Installer logs
|
|
38
|
+
pip-log.txt
|
|
39
|
+
pip-delete-this-directory.txt
|
|
40
|
+
|
|
41
|
+
# Unit test / coverage reports
|
|
42
|
+
htmlcov/
|
|
43
|
+
.tox/
|
|
44
|
+
.nox/
|
|
45
|
+
.coverage
|
|
46
|
+
.coverage.*
|
|
47
|
+
.cache
|
|
48
|
+
nosetests.xml
|
|
49
|
+
coverage.xml
|
|
50
|
+
*.cover
|
|
51
|
+
*.py.cover
|
|
52
|
+
.hypothesis/
|
|
53
|
+
.pytest_cache/
|
|
54
|
+
cover/
|
|
55
|
+
|
|
56
|
+
# Translations
|
|
57
|
+
*.mo
|
|
58
|
+
*.pot
|
|
59
|
+
|
|
60
|
+
# Django stuff:
|
|
61
|
+
*.log
|
|
62
|
+
local_settings.py
|
|
63
|
+
db.sqlite3
|
|
64
|
+
db.sqlite3-journal
|
|
65
|
+
|
|
66
|
+
# Flask stuff:
|
|
67
|
+
instance/
|
|
68
|
+
.webassets-cache
|
|
69
|
+
|
|
70
|
+
# Scrapy stuff:
|
|
71
|
+
.scrapy
|
|
72
|
+
|
|
73
|
+
# Sphinx documentation
|
|
74
|
+
docs/_build/
|
|
75
|
+
|
|
76
|
+
# PyBuilder
|
|
77
|
+
.pybuilder/
|
|
78
|
+
target/
|
|
79
|
+
|
|
80
|
+
# Jupyter Notebook
|
|
81
|
+
.ipynb_checkpoints
|
|
82
|
+
|
|
83
|
+
# IPython
|
|
84
|
+
profile_default/
|
|
85
|
+
ipython_config.py
|
|
86
|
+
|
|
87
|
+
# pyenv
|
|
88
|
+
.python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
Pipfile.lock
|
|
92
|
+
|
|
93
|
+
# UV
|
|
94
|
+
uv.lock
|
|
95
|
+
|
|
96
|
+
# poetry
|
|
97
|
+
poetry.lock
|
|
98
|
+
poetry.toml
|
|
99
|
+
|
|
100
|
+
# pdm
|
|
101
|
+
pdm.lock
|
|
102
|
+
pdm.toml
|
|
103
|
+
.pdm-python
|
|
104
|
+
.pdm-build/
|
|
105
|
+
|
|
106
|
+
# pixi
|
|
107
|
+
.pixi
|
|
108
|
+
|
|
109
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
110
|
+
__pypackages__/
|
|
111
|
+
|
|
112
|
+
# Celery stuff
|
|
113
|
+
celerybeat-schedule
|
|
114
|
+
celerybeat.pid
|
|
115
|
+
|
|
116
|
+
# SageMath parsed files
|
|
117
|
+
*.sage.py
|
|
118
|
+
|
|
119
|
+
# Environments
|
|
120
|
+
.env
|
|
121
|
+
.envrc
|
|
122
|
+
.venv
|
|
123
|
+
env/
|
|
124
|
+
venv/
|
|
125
|
+
ENV/
|
|
126
|
+
env.bak/
|
|
127
|
+
venv.bak/
|
|
128
|
+
|
|
129
|
+
# Spyder project settings
|
|
130
|
+
.spyderproject
|
|
131
|
+
.spyproject
|
|
132
|
+
|
|
133
|
+
# Rope project settings
|
|
134
|
+
.ropeproject
|
|
135
|
+
|
|
136
|
+
# mkdocs documentation
|
|
137
|
+
/site
|
|
138
|
+
|
|
139
|
+
# mypy
|
|
140
|
+
.mypy_cache/
|
|
141
|
+
.dmypy.json
|
|
142
|
+
dmypy.json
|
|
143
|
+
|
|
144
|
+
# Pyre type checker
|
|
145
|
+
.pyre/
|
|
146
|
+
|
|
147
|
+
# pytype static type analyzer
|
|
148
|
+
.pytype/
|
|
149
|
+
|
|
150
|
+
# Cython debug symbols
|
|
151
|
+
cython_debug/
|
|
152
|
+
|
|
153
|
+
# PyCharm
|
|
154
|
+
.idea/
|
|
155
|
+
|
|
156
|
+
# Abstra
|
|
157
|
+
.abstra/
|
|
158
|
+
|
|
159
|
+
# Visual Studio Code
|
|
160
|
+
.vscode/
|
|
161
|
+
|
|
162
|
+
# Ruff stuff:
|
|
163
|
+
.ruff_cache/
|
|
164
|
+
|
|
165
|
+
# PyPI configuration file
|
|
166
|
+
.pypirc
|
|
167
|
+
|
|
168
|
+
# Cursor
|
|
169
|
+
.cursorignore
|
|
170
|
+
.cursorindexingignore
|
|
171
|
+
|
|
172
|
+
# Calude Code
|
|
173
|
+
CLAUDE.md
|
|
174
|
+
.claude/
|
|
175
|
+
|
|
176
|
+
# OpenCode
|
|
177
|
+
AGENTS.md
|
|
178
|
+
.opencode/
|
|
179
|
+
|
|
180
|
+
# Github Copilot
|
|
181
|
+
.github/copilot-instructions.md
|
|
182
|
+
|
|
183
|
+
# Marimo
|
|
184
|
+
marimo/_static/
|
|
185
|
+
marimo/_lsp/
|
|
186
|
+
__marimo__/
|
|
187
|
+
|
|
188
|
+
# User drafts
|
|
189
|
+
draft/
|
|
190
|
+
dev_docs/
|
|
191
|
+
|
|
192
|
+
# user progress
|
|
193
|
+
PROGRESS.md
|
|
194
|
+
progress.md
|
|
195
|
+
|
|
196
|
+
# user ignore
|
|
197
|
+
tests/mocks/log/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 LinnetCodes
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gpdatacached
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Redis-based cross-process data sharing and caching library
|
|
5
|
+
Project-URL: Homepage, https://github.com/linnetcodes/gpdatacached
|
|
6
|
+
Project-URL: Documentation, https://linnetcodes.github.io/gpdatacached
|
|
7
|
+
Project-URL: Repository, https://github.com/linnetcodes/gpdatacached
|
|
8
|
+
Project-URL: Issues, https://github.com/linnetcodes/gpdatacached/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/LinnetCodes/gpdatacached/releases
|
|
10
|
+
Author-email: Linnet Codes <linnet.codes@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: cache,cross-process,data-sharing,redis
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: gpconfig>=0.3.0
|
|
24
|
+
Requires-Dist: pydantic>=2.0
|
|
25
|
+
Requires-Dist: redis>=5.0
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
28
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
29
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
30
|
+
Provides-Extra: docs
|
|
31
|
+
Requires-Dist: mkdocs-material>=9.7; extra == 'docs'
|
|
32
|
+
Requires-Dist: mkdocs-static-i18n>=1.3; extra == 'docs'
|
|
33
|
+
Requires-Dist: mkdocs>=1.6; extra == 'docs'
|
|
34
|
+
Provides-Extra: pandas
|
|
35
|
+
Requires-Dist: pandas>=2.0; extra == 'pandas'
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# gpdatacached
|
|
39
|
+
|
|
40
|
+
> *General-Purpose Data Cached*
|
|
41
|
+
|
|
42
|
+
[](LICENSE)
|
|
43
|
+
[](https://www.python.org/)
|
|
44
|
+
[](https://pypi.org/project/gpdatacached/)
|
|
45
|
+
|
|
46
|
+
A Redis-based **cross-process data sharing and caching** library for Python. Any process that can reach the same Redis instance can share live, mutable, typed data structures — scalars, lists, dicts, sets — with reference semantics, lifecycle control, and optional distributed locking.
|
|
47
|
+
|
|
48
|
+
> Read this in another language: [简体中文](./README.zh-CN.md)
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## Why
|
|
53
|
+
|
|
54
|
+
Sharing mutable state across processes is painful. `multiprocessing.Manager` is single-process-bottlenecked; raw Redis forces you to (de)serialize everything by hand and gives you no type safety, no reference tracking, no TTL cascades. `gpdatacached` gives you a dict-like API on top of Redis with:
|
|
55
|
+
|
|
56
|
+
- **Typed objects** — store `int`, `str`, `datetime`, `list`, `dict`, `set`, … and get the same type back.
|
|
57
|
+
- **Live containers** — `ns["tags"].append("x")` mutates Redis directly; no read-modify-write races on the whole structure.
|
|
58
|
+
- **Reference sharing** — bind the same container under multiple names inside a namespace; the data is stored once.
|
|
59
|
+
- **Lifecycle control** — per-object `permanent` / `ttl` / `sliding-window` cache modes, with cascade propagation to owned subtrees.
|
|
60
|
+
- **Background GC** — reclaims expired and orphaned anonymous objects.
|
|
61
|
+
- **Opt-in distributed locking** — serialize writes across processes with `obj.lock()`.
|
|
62
|
+
- **Optional extensions** — cache [pydantic](https://docs.pydantic.dev/) `BaseModel` records and [pandas](https://pandas.pydata.org/) `Series` / `DataFrame` (see [Extensions](#extensions)).
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## Install
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install gpdatacached
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Optional extras:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pip install "gpdatacached[pandas]" # pandas DataFrame / Series support
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Requirements: **Python ≥ 3.10**, **Redis ≥ 5.0**.
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Quick Start
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
from gpdatacached import GPDCDomain, GPDCDomainConfig
|
|
86
|
+
|
|
87
|
+
config = GPDCDomainConfig(
|
|
88
|
+
redis_host="127.0.0.1",
|
|
89
|
+
redis_port=6379,
|
|
90
|
+
redis_password="secret",
|
|
91
|
+
)
|
|
92
|
+
domain = GPDCDomain("mydomain", config)
|
|
93
|
+
ns = domain.ensure_namespace("analytics")
|
|
94
|
+
|
|
95
|
+
# Scalars read back as native Python values.
|
|
96
|
+
ns["counter"] = 42
|
|
97
|
+
assert ns["counter"] == 42
|
|
98
|
+
|
|
99
|
+
# Containers are live Redis-backed objects.
|
|
100
|
+
ns["tags"] = ["a", "b", "c"]
|
|
101
|
+
ns["tags"].append("d")
|
|
102
|
+
ns["tags"].pop(0)
|
|
103
|
+
assert ns["tags"].value == ["b", "c", "d"]
|
|
104
|
+
|
|
105
|
+
# Lifecycle control.
|
|
106
|
+
ns["temp"] = "x"
|
|
107
|
+
ns.get_object("temp").expire_in(60) # requires cache_mode="ttl"
|
|
108
|
+
|
|
109
|
+
domain.close()
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## Core Concepts
|
|
115
|
+
|
|
116
|
+
```
|
|
117
|
+
GPDCDomain ── GPDCNamespace ── GPDCDataObject (scalar | container)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
| Concept | Role |
|
|
121
|
+
|---|---|
|
|
122
|
+
| **Domain** | Configuration root + Redis connection pool + lock/GC defaults. |
|
|
123
|
+
| **Namespace** | Logical isolation space; behaves like a `MutableMapping`. Cross-namespace references are unsupported by design. |
|
|
124
|
+
| **Data Object** | A named, typed value. Scalars return native Python values on read; containers return live objects. |
|
|
125
|
+
|
|
126
|
+
**Cache modes:** `permanent` (default, never expires), `ttl` (fixed expiry), `sliding` (refreshed on every mutation).
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## Documentation
|
|
131
|
+
|
|
132
|
+
📖 **[Full documentation index](./docs/index.md)** — the entry point to all guides and references below.
|
|
133
|
+
|
|
134
|
+
- **[Overview](./docs/overview.md)** — design purpose, philosophy, and end-to-end architecture (start here to understand the library as a whole).
|
|
135
|
+
- **[API Reference](./docs/api.md)** — complete public API reference.
|
|
136
|
+
- **[Lifecycle Management](./docs/lifecycle.md)** — how object creation, expiry, refresh, and deletion work, including anonymous and reference objects (read this to set correct expectations).
|
|
137
|
+
- **[Extensibility](./docs/extensibility.md)** — the type system explained, with worked examples for adding your own scalar and container types (codecs, object classes, registration).
|
|
138
|
+
- **[Performance Guide](./docs/performance-guide.md)** — usage patterns that degrade performance (sliding TTL overuse, deep nesting, unnecessary GC/locking) and their recommended alternatives. Read before putting GPDC on a hot path.
|
|
139
|
+
- **[Multi-Process Locking](./docs/locking.md)** — when to lock, performance impact, recommended usage, deadlock avoidance, and GC interaction.
|
|
140
|
+
|
|
141
|
+
## Extensions
|
|
142
|
+
|
|
143
|
+
Optional built-in extensions under `gpdatacached.extensions`:
|
|
144
|
+
|
|
145
|
+
- **[Pydantic Support](./docs/pydantic-support.md)** — cache pydantic `BaseModel` records. Two storage modes: scalar (atomic JSON) for scalar-only models, container (Redis hash) for models needing field-level access or nested `list`/`dict`/`set` fields. Pydantic is a core dependency; no extra install needed.
|
|
146
|
+
- **[Pandas Support](./docs/pandas-support.md)** — cache pandas `Series` / `DataFrame` as live objects. Online selective accessors (`.iloc`, `.loc`, `df[col]`) for one-shot subset reads; `.value` for full restore; `extend` for appends. Requires `pip install "gpdatacached[pandas]"`.
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## Running Tests
|
|
151
|
+
|
|
152
|
+
The test suite expects a **local Redis instance** reachable at `127.0.0.1:6379` with password `123456`:
|
|
153
|
+
|
|
154
|
+
- **Linux** — run Redis directly on the host.
|
|
155
|
+
- **Windows** — run Redis inside WSL; it will be reachable at `localhost` / `127.0.0.1` from the Windows side.
|
|
156
|
+
|
|
157
|
+
> ⚠️ Some tests **flush the Redis database** (`FLUSHDB`). Always use a dedicated Redis instance for testing — never point the tests at a production database.
|
|
158
|
+
|
|
159
|
+
Run the suite with:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
pytest
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
You can edit the Redis settings in `tests/conftest.py` to point at a different instance, but this is not recommended — running a dedicated throwaway test Redis is usually cheaper and safer.
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
|
|
171
|
+
[MIT](./LICENSE) © Linnet Codes
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# gpdatacached
|
|
2
|
+
|
|
3
|
+
> *General-Purpose Data Cached*
|
|
4
|
+
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](https://www.python.org/)
|
|
7
|
+
[](https://pypi.org/project/gpdatacached/)
|
|
8
|
+
|
|
9
|
+
A Redis-based **cross-process data sharing and caching** library for Python. Any process that can reach the same Redis instance can share live, mutable, typed data structures — scalars, lists, dicts, sets — with reference semantics, lifecycle control, and optional distributed locking.
|
|
10
|
+
|
|
11
|
+
> Read this in another language: [简体中文](./README.zh-CN.md)
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## Why
|
|
16
|
+
|
|
17
|
+
Sharing mutable state across processes is painful. `multiprocessing.Manager` is single-process-bottlenecked; raw Redis forces you to (de)serialize everything by hand and gives you no type safety, no reference tracking, no TTL cascades. `gpdatacached` gives you a dict-like API on top of Redis with:
|
|
18
|
+
|
|
19
|
+
- **Typed objects** — store `int`, `str`, `datetime`, `list`, `dict`, `set`, … and get the same type back.
|
|
20
|
+
- **Live containers** — `ns["tags"].append("x")` mutates Redis directly; no read-modify-write races on the whole structure.
|
|
21
|
+
- **Reference sharing** — bind the same container under multiple names inside a namespace; the data is stored once.
|
|
22
|
+
- **Lifecycle control** — per-object `permanent` / `ttl` / `sliding-window` cache modes, with cascade propagation to owned subtrees.
|
|
23
|
+
- **Background GC** — reclaims expired and orphaned anonymous objects.
|
|
24
|
+
- **Opt-in distributed locking** — serialize writes across processes with `obj.lock()`.
|
|
25
|
+
- **Optional extensions** — cache [pydantic](https://docs.pydantic.dev/) `BaseModel` records and [pandas](https://pandas.pydata.org/) `Series` / `DataFrame` (see [Extensions](#extensions)).
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install gpdatacached
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Optional extras:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install "gpdatacached[pandas]" # pandas DataFrame / Series support
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Requirements: **Python ≥ 3.10**, **Redis ≥ 5.0**.
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## Quick Start
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from gpdatacached import GPDCDomain, GPDCDomainConfig
|
|
49
|
+
|
|
50
|
+
config = GPDCDomainConfig(
|
|
51
|
+
redis_host="127.0.0.1",
|
|
52
|
+
redis_port=6379,
|
|
53
|
+
redis_password="secret",
|
|
54
|
+
)
|
|
55
|
+
domain = GPDCDomain("mydomain", config)
|
|
56
|
+
ns = domain.ensure_namespace("analytics")
|
|
57
|
+
|
|
58
|
+
# Scalars read back as native Python values.
|
|
59
|
+
ns["counter"] = 42
|
|
60
|
+
assert ns["counter"] == 42
|
|
61
|
+
|
|
62
|
+
# Containers are live Redis-backed objects.
|
|
63
|
+
ns["tags"] = ["a", "b", "c"]
|
|
64
|
+
ns["tags"].append("d")
|
|
65
|
+
ns["tags"].pop(0)
|
|
66
|
+
assert ns["tags"].value == ["b", "c", "d"]
|
|
67
|
+
|
|
68
|
+
# Lifecycle control.
|
|
69
|
+
ns["temp"] = "x"
|
|
70
|
+
ns.get_object("temp").expire_in(60) # requires cache_mode="ttl"
|
|
71
|
+
|
|
72
|
+
domain.close()
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## Core Concepts
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
GPDCDomain ── GPDCNamespace ── GPDCDataObject (scalar | container)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
| Concept | Role |
|
|
84
|
+
|---|---|
|
|
85
|
+
| **Domain** | Configuration root + Redis connection pool + lock/GC defaults. |
|
|
86
|
+
| **Namespace** | Logical isolation space; behaves like a `MutableMapping`. Cross-namespace references are unsupported by design. |
|
|
87
|
+
| **Data Object** | A named, typed value. Scalars return native Python values on read; containers return live objects. |
|
|
88
|
+
|
|
89
|
+
**Cache modes:** `permanent` (default, never expires), `ttl` (fixed expiry), `sliding` (refreshed on every mutation).
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Documentation
|
|
94
|
+
|
|
95
|
+
📖 **[Full documentation index](./docs/index.md)** — the entry point to all guides and references below.
|
|
96
|
+
|
|
97
|
+
- **[Overview](./docs/overview.md)** — design purpose, philosophy, and end-to-end architecture (start here to understand the library as a whole).
|
|
98
|
+
- **[API Reference](./docs/api.md)** — complete public API reference.
|
|
99
|
+
- **[Lifecycle Management](./docs/lifecycle.md)** — how object creation, expiry, refresh, and deletion work, including anonymous and reference objects (read this to set correct expectations).
|
|
100
|
+
- **[Extensibility](./docs/extensibility.md)** — the type system explained, with worked examples for adding your own scalar and container types (codecs, object classes, registration).
|
|
101
|
+
- **[Performance Guide](./docs/performance-guide.md)** — usage patterns that degrade performance (sliding TTL overuse, deep nesting, unnecessary GC/locking) and their recommended alternatives. Read before putting GPDC on a hot path.
|
|
102
|
+
- **[Multi-Process Locking](./docs/locking.md)** — when to lock, performance impact, recommended usage, deadlock avoidance, and GC interaction.
|
|
103
|
+
|
|
104
|
+
## Extensions
|
|
105
|
+
|
|
106
|
+
Optional built-in extensions under `gpdatacached.extensions`:
|
|
107
|
+
|
|
108
|
+
- **[Pydantic Support](./docs/pydantic-support.md)** — cache pydantic `BaseModel` records. Two storage modes: scalar (atomic JSON) for scalar-only models, container (Redis hash) for models needing field-level access or nested `list`/`dict`/`set` fields. Pydantic is a core dependency; no extra install needed.
|
|
109
|
+
- **[Pandas Support](./docs/pandas-support.md)** — cache pandas `Series` / `DataFrame` as live objects. Online selective accessors (`.iloc`, `.loc`, `df[col]`) for one-shot subset reads; `.value` for full restore; `extend` for appends. Requires `pip install "gpdatacached[pandas]"`.
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## Running Tests
|
|
114
|
+
|
|
115
|
+
The test suite expects a **local Redis instance** reachable at `127.0.0.1:6379` with password `123456`:
|
|
116
|
+
|
|
117
|
+
- **Linux** — run Redis directly on the host.
|
|
118
|
+
- **Windows** — run Redis inside WSL; it will be reachable at `localhost` / `127.0.0.1` from the Windows side.
|
|
119
|
+
|
|
120
|
+
> ⚠️ Some tests **flush the Redis database** (`FLUSHDB`). Always use a dedicated Redis instance for testing — never point the tests at a production database.
|
|
121
|
+
|
|
122
|
+
Run the suite with:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pytest
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
You can edit the Redis settings in `tests/conftest.py` to point at a different instance, but this is not recommended — running a dedicated throwaway test Redis is usually cheaper and safer.
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## License
|
|
133
|
+
|
|
134
|
+
[MIT](./LICENSE) © Linnet Codes
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# gpdatacached
|
|
2
|
+
|
|
3
|
+
> *General-Purpose Data Cached*(通用数据缓存)
|
|
4
|
+
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](https://www.python.org/)
|
|
7
|
+
[](https://pypi.org/project/gpdatacached/)
|
|
8
|
+
|
|
9
|
+
一个基于 Redis 的**跨进程数据共享与缓存** Python 库。任何能够访问同一 Redis 实例的进程,都可以共享活跃的、可变的、带类型的数据结构 —— 标量、list、dict、set —— 并具备引用语义、生命周期控制以及可选的分布式锁。
|
|
10
|
+
|
|
11
|
+
> 其他语言:[English](./README.md)
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## 为什么需要它
|
|
16
|
+
|
|
17
|
+
跨进程共享可变状态一直很麻烦:`multiprocessing.Manager` 会形成单进程瓶颈;直接使用 Redis 又需要你手动序列化所有数据,且没有类型安全、没有引用追踪、没有 TTL 级联。`gpdatacached` 在 Redis 之上提供了一套类 dict 的 API,特性包括:
|
|
18
|
+
|
|
19
|
+
- **带类型的对象** —— 存储 `int`、`str`、`datetime`、`list`、`dict`、`set` …,读取时得到相同类型。
|
|
20
|
+
- **活跃容器** —— `ns["tags"].append("x")` 直接修改 Redis,避免对整个结构的“读-改-写”竞态。
|
|
21
|
+
- **引用共享** —— 在同一 namespace 内,同一个容器可绑定到多个名称;数据仅存储一份。
|
|
22
|
+
- **生命周期控制** —— 每个对象可设置 `permanent` / `ttl` / `sliding`(滑动窗口)缓存模式,并将策略级联传播到所拥有的子树。
|
|
23
|
+
- **后台 GC** —— 自动回收过期或成为孤儿的匿名对象。
|
|
24
|
+
- **可选的分布式锁** —— 通过 `obj.lock()` 在多进程间串行化写入。
|
|
25
|
+
- **可选扩展** —— 可缓存 [pydantic](https://docs.pydantic.dev/) `BaseModel` 记录与 [pandas](https://pandas.pydata.org/) `Series` / `DataFrame`(见[扩展](#扩展))。
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## 安装
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install gpdatacached
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
可选扩展:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install "gpdatacached[pandas]" # 支持 pandas DataFrame / Series
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
环境要求:**Python ≥ 3.10**,**Redis ≥ 5.0**。
|
|
42
|
+
|
|
43
|
+
---
|
|
44
|
+
|
|
45
|
+
## 快速开始
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from gpdatacached import GPDCDomain, GPDCDomainConfig
|
|
49
|
+
|
|
50
|
+
config = GPDCDomainConfig(
|
|
51
|
+
redis_host="127.0.0.1",
|
|
52
|
+
redis_port=6379,
|
|
53
|
+
redis_password="secret",
|
|
54
|
+
)
|
|
55
|
+
domain = GPDCDomain("mydomain", config)
|
|
56
|
+
ns = domain.ensure_namespace("analytics")
|
|
57
|
+
|
|
58
|
+
# 标量读取出来是原生 Python 值。
|
|
59
|
+
ns["counter"] = 42
|
|
60
|
+
assert ns["counter"] == 42
|
|
61
|
+
|
|
62
|
+
# 容器是活跃的、由 Redis 支撑的对象。
|
|
63
|
+
ns["tags"] = ["a", "b", "c"]
|
|
64
|
+
ns["tags"].append("d")
|
|
65
|
+
ns["tags"].pop(0)
|
|
66
|
+
assert ns["tags"].value == ["b", "c", "d"]
|
|
67
|
+
|
|
68
|
+
# 生命周期控制。
|
|
69
|
+
ns["temp"] = "x"
|
|
70
|
+
ns.get_object("temp").expire_in(60) # 要求 cache_mode="ttl"
|
|
71
|
+
|
|
72
|
+
domain.close()
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## 核心概念
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
GPDCDomain ── GPDCNamespace ── GPDCDataObject (标量 | 容器)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
| 概念 | 角色 |
|
|
84
|
+
|---|---|
|
|
85
|
+
| **Domain(域)** | 配置根 + Redis 连接池 + 锁/GC 默认值。 |
|
|
86
|
+
| **Namespace(命名空间)** | 逻辑隔离空间,行为类似 `MutableMapping`。**不支持**跨 namespace 的引用。 |
|
|
87
|
+
| **Data Object(数据对象)** | 一个具名、带类型的值。标量读取返回原生 Python 值;容器返回活跃对象。 |
|
|
88
|
+
|
|
89
|
+
**缓存模式:** `permanent`(默认,永不过期)、`ttl`(固定过期)、`sliding`(每次变更都刷新)。
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## 文档
|
|
94
|
+
|
|
95
|
+
📖 **[完整文档索引](./docs/zh-CN/index.md)** —— 以下所有指南与参考的入口。
|
|
96
|
+
|
|
97
|
+
- **[总体概览](./docs/zh-CN/overview.md)** —— 设计目的、设计理念与端到端整体架构(从这里开始,从整体上理解本库)。
|
|
98
|
+
- **[API 参考](./docs/zh-CN/api.md)** —— 完整的公共 API 参考。
|
|
99
|
+
- **[生命周期管理](./docs/zh-CN/lifecycle.md)** —— 对象的创建、过期、刷新与删除如何运作,含匿名对象与引用对象(阅读本文以建立合理预期)。
|
|
100
|
+
- **[可扩展性](./docs/zh-CN/extensibility.md)** —— 类型系统详解,含添加自定义标量与容器类型的完整示例(codec、对象类、注册)。
|
|
101
|
+
- **[性能指南](./docs/zh-CN/performance-guide.md)** —— 会拖累性能的用法(滥用 sliding TTL、过深嵌套、不必要地启用 GC/加锁)及其推荐替代。把 GPDC 放上热路径前必读。
|
|
102
|
+
- **[多进程加锁](./docs/zh-CN/locking.md)** —— 何时需要加锁、性能影响、推荐用法、避免死锁的顺序以及与 GC 的交互。
|
|
103
|
+
|
|
104
|
+
## 扩展
|
|
105
|
+
|
|
106
|
+
`gpdatacached.extensions` 下的可选内建扩展:
|
|
107
|
+
|
|
108
|
+
- **[Pydantic 支持](./docs/zh-CN/pydantic-support.md)** —— 缓存 pydantic `BaseModel` 记录。两种存储模式:标量模式(原子 JSON,适用于纯标量模型)和容器模式(Redis hash,适用于需要字段级访问或嵌套 `list`/`dict`/`set` 字段的模型)。pydantic 是核心依赖,无需额外安装。
|
|
109
|
+
- **[Pandas 支持](./docs/zh-CN/pandas-support.md)** —— 将 pandas `Series` / `DataFrame` 缓存为活跃对象。在线选择性访问器(`.iloc`、`.loc`、`df[col]`)用于一次性子集读取;`.value` 用于完整还原;`extend` 用于追加。需要 `pip install "gpdatacached[pandas]"`。
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## 运行测试
|
|
114
|
+
|
|
115
|
+
测试套件需要一个**本地 Redis 实例**,可通过 `127.0.0.1:6379` 访问,密码为 `123456`:
|
|
116
|
+
|
|
117
|
+
- **Linux** —— 直接在主机上运行 Redis。
|
|
118
|
+
- **Windows** —— 在 WSL 中运行 Redis;从 Windows 侧可通过 `localhost` / `127.0.0.1` 访问。
|
|
119
|
+
|
|
120
|
+
> ⚠️ 某些测试会**清空 Redis 数据库**(`FLUSHDB`)。请务必使用专供测试用的 Redis 实例 —— 切勿将测试指向生产数据库。
|
|
121
|
+
|
|
122
|
+
运行测试:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
pytest
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
你也可以修改 `tests/conftest.py` 中的 Redis 配置来适配自己的实例,但不推荐这样做 —— 运行一个专用的、一次性的测试 Redis 通常成本更低也更安全。
|
|
129
|
+
|
|
130
|
+
---
|
|
131
|
+
|
|
132
|
+
## 许可证
|
|
133
|
+
|
|
134
|
+
[MIT](./LICENSE) © Linnet Codes
|