kvgit 0.2.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kvgit-0.2.2 → kvgit-0.3.0}/PKG-INFO +38 -4
- {kvgit-0.2.2 → kvgit-0.3.0}/README.md +25 -3
- kvgit-0.3.0/kvgit/codecs/__init__.py +107 -0
- kvgit-0.3.0/kvgit/codecs/_hash.py +29 -0
- kvgit-0.3.0/kvgit/codecs/base.py +79 -0
- kvgit-0.3.0/kvgit/codecs/numpy.py +237 -0
- kvgit-0.3.0/kvgit/codecs/pandas.py +28 -0
- kvgit-0.3.0/kvgit/codecs/pickler.py +91 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/staged.py +153 -15
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/store.py +24 -2
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/versioned/base.py +26 -2
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/versioned/gp.py +13 -1
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/versioned/keyset.py +32 -6
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/versioned/kv.py +118 -37
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/versioned/protocol.py +2 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit.egg-info/PKG-INFO +38 -4
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit.egg-info/SOURCES.txt +6 -0
- kvgit-0.3.0/kvgit.egg-info/requires.txt +31 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/pyproject.toml +20 -3
- kvgit-0.2.2/kvgit.egg-info/requires.txt +0 -16
- {kvgit-0.2.2 → kvgit-0.3.0}/LICENSE +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/__init__.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/content_types.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/encoding.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/errors.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/hamt.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/kv/__init__.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/kv/base.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/kv/composite.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/kv/disk.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/kv/indexeddb.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/kv/memory.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/namespaced.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/py.typed +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/versioned/__init__.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/versioned/helpers.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit/versioned/merge.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit.egg-info/dependency_links.txt +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/kvgit.egg-info/top_level.txt +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/setup.cfg +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/tests/test_content_types.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/tests/test_hamt.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/tests/test_namespaced.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/tests/test_staged.py +0 -0
- {kvgit-0.2.2 → kvgit-0.3.0}/tests/test_store_factory.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kvgit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Versioned key-value store with git-like commit, branch, and merge semantics.
|
|
5
5
|
Author: ashenfad
|
|
6
6
|
License: MIT
|
|
@@ -29,14 +29,26 @@ Provides-Extra: disk
|
|
|
29
29
|
Requires-Dist: diskcache; extra == "disk"
|
|
30
30
|
Provides-Extra: git
|
|
31
31
|
Requires-Dist: gitpython>=3.1; extra == "git"
|
|
32
|
+
Provides-Extra: numpy
|
|
33
|
+
Requires-Dist: numpy>=1.24; extra == "numpy"
|
|
34
|
+
Provides-Extra: pandas
|
|
35
|
+
Requires-Dist: numpy>=1.24; extra == "pandas"
|
|
36
|
+
Requires-Dist: pandas>=2.0; extra == "pandas"
|
|
37
|
+
Provides-Extra: scientific
|
|
38
|
+
Requires-Dist: numpy>=1.24; extra == "scientific"
|
|
39
|
+
Requires-Dist: pandas>=2.0; extra == "scientific"
|
|
32
40
|
Provides-Extra: all
|
|
33
41
|
Requires-Dist: diskcache; extra == "all"
|
|
34
42
|
Requires-Dist: gitpython>=3.1; extra == "all"
|
|
43
|
+
Requires-Dist: numpy>=1.24; extra == "all"
|
|
44
|
+
Requires-Dist: pandas>=2.0; extra == "all"
|
|
35
45
|
Provides-Extra: dev
|
|
36
46
|
Requires-Dist: pytest; extra == "dev"
|
|
37
47
|
Requires-Dist: diskcache; extra == "dev"
|
|
38
48
|
Requires-Dist: gitpython>=3.1; extra == "dev"
|
|
39
49
|
Requires-Dist: ruff; extra == "dev"
|
|
50
|
+
Requires-Dist: numpy>=1.24; extra == "dev"
|
|
51
|
+
Requires-Dist: pandas>=2.0; extra == "dev"
|
|
40
52
|
Dynamic: license-file
|
|
41
53
|
|
|
42
54
|
# kvgit 🔀
|
|
@@ -50,13 +62,15 @@ Git-style versioning for your data. Commits, branches, and merges -- backed by a
|
|
|
50
62
|
| **Branches** | Cheap forks with CAS-based optimistic concurrency |
|
|
51
63
|
| **Three-way merge** | Auto-merges non-overlapping changes; pluggable merge fns for conflicts |
|
|
52
64
|
| **Pluggable backends** | In-memory, disk (diskcache), git (GitPython), IndexedDB (Pyodide/browser), or bring your own `KVStore` |
|
|
65
|
+
| **Chunked codecs** | Optional content-addressed dedup for large numpy arrays and pandas DataFrames -- equal buffers stored once across keys, commits, and branches |
|
|
53
66
|
|
|
54
67
|
## Install
|
|
55
68
|
|
|
56
69
|
```bash
|
|
57
|
-
pip install kvgit
|
|
58
|
-
pip install kvgit[disk]
|
|
59
|
-
pip install kvgit[git]
|
|
70
|
+
pip install kvgit # in-memory only
|
|
71
|
+
pip install kvgit[disk] # adds disk backend via diskcache
|
|
72
|
+
pip install kvgit[git] # adds git backend via GitPython (requires git on PATH)
|
|
73
|
+
pip install kvgit[scientific] # adds chunked codecs for numpy / pandas
|
|
60
74
|
# IndexedDB backend is available automatically in Pyodide (browser) environments
|
|
61
75
|
```
|
|
62
76
|
|
|
@@ -80,6 +94,26 @@ print(main["score"]) # 0 (main unchanged)
|
|
|
80
94
|
print(dev["score"]) # 999 (dev branch)
|
|
81
95
|
```
|
|
82
96
|
|
|
97
|
+
## Chunked codecs (numpy / pandas)
|
|
98
|
+
|
|
99
|
+
Large numpy arrays and pandas DataFrames -- and any sliced views of them -- can be stored once and shared across keys, commits, and branches:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
import kvgit
|
|
103
|
+
import numpy as np
|
|
104
|
+
|
|
105
|
+
s = kvgit.store(codecs="scientific")
|
|
106
|
+
|
|
107
|
+
big = np.arange(1_000_000, dtype="float64") # ~8 MB
|
|
108
|
+
s["full"] = big
|
|
109
|
+
s["head"] = big[:100_000]
|
|
110
|
+
s["tail"] = big[-100_000:]
|
|
111
|
+
s.commit()
|
|
112
|
+
# All three keys reference the same chunk on disk -- ~8 MB total, not ~24 MB.
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Pandas DataFrames piggyback on the numpy codec via their underlying block ndarrays. See [`docs/quick-start.md`](docs/quick-start.md#storing-scientific-data-efficiently-chunked-codecs) and the [API reference](docs/api.md#chunked-codecs).
|
|
116
|
+
|
|
83
117
|
## Part of the agex stack
|
|
84
118
|
|
|
85
119
|
kvgit provides versioned agent memory in [agex](https://github.com/ashenfad/agex) with branching and rollback. It also works as a versioned backing store for [monkeyfs](https://github.com/ashenfad/monkeyfs) virtual filesystems -- pass a `Staged` instance anywhere a dict is expected.
|
|
@@ -9,13 +9,15 @@ Git-style versioning for your data. Commits, branches, and merges -- backed by a
|
|
|
9
9
|
| **Branches** | Cheap forks with CAS-based optimistic concurrency |
|
|
10
10
|
| **Three-way merge** | Auto-merges non-overlapping changes; pluggable merge fns for conflicts |
|
|
11
11
|
| **Pluggable backends** | In-memory, disk (diskcache), git (GitPython), IndexedDB (Pyodide/browser), or bring your own `KVStore` |
|
|
12
|
+
| **Chunked codecs** | Optional content-addressed dedup for large numpy arrays and pandas DataFrames -- equal buffers stored once across keys, commits, and branches |
|
|
12
13
|
|
|
13
14
|
## Install
|
|
14
15
|
|
|
15
16
|
```bash
|
|
16
|
-
pip install kvgit
|
|
17
|
-
pip install kvgit[disk]
|
|
18
|
-
pip install kvgit[git]
|
|
17
|
+
pip install kvgit # in-memory only
|
|
18
|
+
pip install kvgit[disk] # adds disk backend via diskcache
|
|
19
|
+
pip install kvgit[git] # adds git backend via GitPython (requires git on PATH)
|
|
20
|
+
pip install kvgit[scientific] # adds chunked codecs for numpy / pandas
|
|
19
21
|
# IndexedDB backend is available automatically in Pyodide (browser) environments
|
|
20
22
|
```
|
|
21
23
|
|
|
@@ -39,6 +41,26 @@ print(main["score"]) # 0 (main unchanged)
|
|
|
39
41
|
print(dev["score"]) # 999 (dev branch)
|
|
40
42
|
```
|
|
41
43
|
|
|
44
|
+
## Chunked codecs (numpy / pandas)
|
|
45
|
+
|
|
46
|
+
Large numpy arrays and pandas DataFrames -- and any sliced views of them -- can be stored once and shared across keys, commits, and branches:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
import kvgit
|
|
50
|
+
import numpy as np
|
|
51
|
+
|
|
52
|
+
s = kvgit.store(codecs="scientific")
|
|
53
|
+
|
|
54
|
+
big = np.arange(1_000_000, dtype="float64") # ~8 MB
|
|
55
|
+
s["full"] = big
|
|
56
|
+
s["head"] = big[:100_000]
|
|
57
|
+
s["tail"] = big[-100_000:]
|
|
58
|
+
s.commit()
|
|
59
|
+
# All three keys reference the same chunk on disk -- ~8 MB total, not ~24 MB.
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Pandas DataFrames piggyback on the numpy codec via their underlying block ndarrays. See [`docs/quick-start.md`](docs/quick-start.md#storing-scientific-data-efficiently-chunked-codecs) and the [API reference](docs/api.md#chunked-codecs).
|
|
63
|
+
|
|
42
64
|
## Part of the agex stack
|
|
43
65
|
|
|
44
66
|
kvgit provides versioned agent memory in [agex](https://github.com/ashenfad/agex) with branching and rollback. It also works as a versioned backing store for [monkeyfs](https://github.com/ashenfad/monkeyfs) virtual filesystems -- pass a `Staged` instance anywhere a dict is expected.
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Chunked codecs for kvgit.
|
|
2
|
+
|
|
3
|
+
A chunked codec externalizes large sub-values (numpy buffers, Arrow
|
|
4
|
+
tables, ...) as content-addressed chunks stored in a separate
|
|
5
|
+
namespace, so equal data is stored once across keys, commits, and
|
|
6
|
+
branches.
|
|
7
|
+
|
|
8
|
+
Quick start::
|
|
9
|
+
|
|
10
|
+
from kvgit import store
|
|
11
|
+
from kvgit.codecs import compose
|
|
12
|
+
from kvgit.codecs.numpy import NumpyCodec
|
|
13
|
+
|
|
14
|
+
encoder, decoder = compose(NumpyCodec())
|
|
15
|
+
s = store(encoder=encoder, decoder=decoder)
|
|
16
|
+
|
|
17
|
+
The encoder/decoder pair is plug-compatible with ``Staged``'s
|
|
18
|
+
existing slots: it accepts a ``ChunkSink`` / ``ChunkReader`` second
|
|
19
|
+
argument, and ``Staged`` detects the extended arity automatically.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from .base import ChunkReader, ChunkSink, Codec
|
|
25
|
+
from .pickler import (
|
|
26
|
+
ChunkingPickler,
|
|
27
|
+
ChunkingUnpickler,
|
|
28
|
+
decode as _decode,
|
|
29
|
+
encode as _encode,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def compose(*codecs: Codec):
|
|
34
|
+
"""Build an ``(encoder, decoder)`` pair from a list of codecs.
|
|
35
|
+
|
|
36
|
+
The codecs are tried in order during encoding; the first to
|
|
37
|
+
return a non-None token from ``try_externalize`` wins. Plain
|
|
38
|
+
pickling handles anything no codec claims — there is no need to
|
|
39
|
+
register a "pickle codec" explicitly.
|
|
40
|
+
|
|
41
|
+
Order matters when codecs claim overlapping types. Put the more
|
|
42
|
+
specific codec first.
|
|
43
|
+
"""
|
|
44
|
+
codec_list = list(codecs)
|
|
45
|
+
|
|
46
|
+
def encoder(value, sink):
|
|
47
|
+
return _encode(value, sink, codec_list)
|
|
48
|
+
|
|
49
|
+
def decoder(blob, reader):
|
|
50
|
+
return _decode(blob, reader, codec_list)
|
|
51
|
+
|
|
52
|
+
return encoder, decoder
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def scientific():
|
|
56
|
+
"""Return an ``(encoder, decoder)`` pair using scientific codecs.
|
|
57
|
+
|
|
58
|
+
Currently composes the numpy codec, which transparently handles
|
|
59
|
+
pandas ``DataFrame`` / ``Series`` block buffers via pandas' pickle
|
|
60
|
+
path. Future scientific codecs (Arrow, etc.) will slot in here
|
|
61
|
+
when both the codec and its dependency are available.
|
|
62
|
+
|
|
63
|
+
Raises:
|
|
64
|
+
ImportError: if numpy is not importable in this environment.
|
|
65
|
+
Install with ``pip install kvgit[scientific]``.
|
|
66
|
+
"""
|
|
67
|
+
try:
|
|
68
|
+
from .numpy import NumpyCodec
|
|
69
|
+
except ImportError as e:
|
|
70
|
+
raise ImportError(
|
|
71
|
+
"kvgit.codecs.scientific() requires numpy. "
|
|
72
|
+
"Install with `pip install kvgit[scientific]`."
|
|
73
|
+
) from e
|
|
74
|
+
return compose(NumpyCodec())
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
# Registry of named codec presets used by ``kvgit.store(codecs=...)``.
|
|
78
|
+
# Keep this sparse: each preset is a deliberate, well-documented bundle.
|
|
79
|
+
_NAMED_PRESETS = {
|
|
80
|
+
"scientific": scientific,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _resolve_named(name: str):
|
|
85
|
+
"""Resolve a named codec preset to an ``(encoder, decoder)`` pair.
|
|
86
|
+
|
|
87
|
+
Internal helper used by ``kvgit.store(codecs=...)``. Raises
|
|
88
|
+
``ValueError`` for unknown names with a list of valid options.
|
|
89
|
+
"""
|
|
90
|
+
factory = _NAMED_PRESETS.get(name)
|
|
91
|
+
if factory is None:
|
|
92
|
+
raise ValueError(
|
|
93
|
+
f"unknown codec preset {name!r}. "
|
|
94
|
+
f"Available presets: {sorted(_NAMED_PRESETS)}"
|
|
95
|
+
)
|
|
96
|
+
return factory()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
__all__ = [
|
|
100
|
+
"ChunkReader",
|
|
101
|
+
"ChunkSink",
|
|
102
|
+
"ChunkingPickler",
|
|
103
|
+
"ChunkingUnpickler",
|
|
104
|
+
"Codec",
|
|
105
|
+
"compose",
|
|
106
|
+
"scientific",
|
|
107
|
+
]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Canonical buffer hashing for chunk references.
|
|
2
|
+
|
|
3
|
+
Centralizes the hash function so all codecs and chunk-sink
|
|
4
|
+
implementations agree on how a chunk is identified. The hash must
|
|
5
|
+
be stable across processes, Python versions, and platforms — only
|
|
6
|
+
the raw bytes are hashed; never object identity or implementation
|
|
7
|
+
metadata.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
|
|
14
|
+
# 40 hex chars (160 bits) — same length as kvgit's commit hashes,
|
|
15
|
+
# enough collision resistance for billions of chunks.
|
|
16
|
+
HASH_LEN = 40
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def hash_bytes(data: bytes | memoryview) -> str:
|
|
20
|
+
"""Hash arbitrary bytes-like data to a hex digest."""
|
|
21
|
+
h = hashlib.sha256()
|
|
22
|
+
if isinstance(data, memoryview):
|
|
23
|
+
# Cast to byte-level memoryview for a stable read regardless of
|
|
24
|
+
# original element format. ``hashlib.update`` accepts memoryview
|
|
25
|
+
# without copying.
|
|
26
|
+
h.update(data.cast("B"))
|
|
27
|
+
else:
|
|
28
|
+
h.update(data)
|
|
29
|
+
return h.hexdigest()[:HASH_LEN]
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Codec protocols for chunked encoding.
|
|
2
|
+
|
|
3
|
+
A *codec* recognizes a specific value type during encoding, externalizes
|
|
4
|
+
its bulk data as content-addressed chunks, and emits a small
|
|
5
|
+
picklable token in place of the value. On decode it reads the token,
|
|
6
|
+
fetches the chunks it references, and rebuilds the value.
|
|
7
|
+
|
|
8
|
+
Codecs are composed via :func:`kvgit.codecs.compose`, which wraps them
|
|
9
|
+
in a ``ChunkingPickler`` / ``ChunkingUnpickler`` pair. Pickle handles
|
|
10
|
+
container traversal and primitive types; the codec only sees leaf
|
|
11
|
+
objects it cares about.
|
|
12
|
+
|
|
13
|
+
The kvgit core sees codecs as opaque ``(encoder, decoder)`` callables
|
|
14
|
+
on ``Staged``. Users opt in by passing them at construction.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from typing import Any, Protocol, runtime_checkable
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@runtime_checkable
|
|
23
|
+
class ChunkSink(Protocol):
|
|
24
|
+
"""Encoder-side: emit a chunk, get back a content-addressed reference.
|
|
25
|
+
|
|
26
|
+
Implementations hash the data once and cache the hash. Re-emitting
|
|
27
|
+
the same bytes within an encode session must short-circuit to the
|
|
28
|
+
same reference without rehashing — this matters for codecs that
|
|
29
|
+
can cheaply produce the same bytes from the same Python object
|
|
30
|
+
(e.g., the same ndarray reaching ``put`` twice via different
|
|
31
|
+
container paths).
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def put(self, data: bytes | memoryview) -> str:
|
|
35
|
+
"""Register a chunk and return its content-addressed reference."""
|
|
36
|
+
...
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@runtime_checkable
|
|
40
|
+
class ChunkReader(Protocol):
|
|
41
|
+
"""Decoder-side: fetch chunks by reference."""
|
|
42
|
+
|
|
43
|
+
def get(self, ref: str) -> bytes:
|
|
44
|
+
"""Fetch a single chunk. Raises ``KeyError`` if missing."""
|
|
45
|
+
...
|
|
46
|
+
|
|
47
|
+
def get_many(self, refs: list[str]) -> dict[str, bytes]:
|
|
48
|
+
"""Bulk-fetch. Returns only refs that exist; missing refs absent."""
|
|
49
|
+
...
|
|
50
|
+
|
|
51
|
+
def prefetch(self, refs: list[str]) -> None:
|
|
52
|
+
"""Hint that these refs will be needed soon. May be a no-op."""
|
|
53
|
+
...
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@runtime_checkable
|
|
57
|
+
class Codec(Protocol):
|
|
58
|
+
"""A type-specific chunking codec.
|
|
59
|
+
|
|
60
|
+
Attributes:
|
|
61
|
+
name: Short tag stored in tokens (e.g. ``"np"``, ``"pa"``).
|
|
62
|
+
Must be unique within a composed codec set.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
name: str
|
|
66
|
+
|
|
67
|
+
def try_externalize(self, obj: Any, sink: ChunkSink) -> Any | None:
|
|
68
|
+
"""Recognize ``obj`` and emit chunks; return a picklable token.
|
|
69
|
+
|
|
70
|
+
Returns ``None`` to pass — the next codec in the chain (or
|
|
71
|
+
plain pickling) will handle the object. The returned token
|
|
72
|
+
becomes the second element of pickle's ``persistent_id``
|
|
73
|
+
(the first element is :attr:`name`).
|
|
74
|
+
"""
|
|
75
|
+
...
|
|
76
|
+
|
|
77
|
+
def materialize(self, token: Any, reader: ChunkReader) -> Any:
|
|
78
|
+
"""Reconstruct the value from a token by fetching its chunks."""
|
|
79
|
+
...
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""NumPy ndarray codec.
|
|
2
|
+
|
|
3
|
+
Externalizes the underlying buffer of an ndarray as a content-addressed
|
|
4
|
+
chunk; the token records the dtype, shape, layout order, and (for
|
|
5
|
+
views) the slice geometry needed to reconstruct the array against the
|
|
6
|
+
chunked root buffer.
|
|
7
|
+
|
|
8
|
+
Dedup story:
|
|
9
|
+
|
|
10
|
+
* Two arrays with identical bytes hash to the same chunk reference.
|
|
11
|
+
* A view (``arr.base is not None``) hashes its **root** buffer, not
|
|
12
|
+
the slice. Multiple slices of the same parent therefore share one
|
|
13
|
+
chunk on disk.
|
|
14
|
+
* Both C-contiguous and F-contiguous roots are stored verbatim
|
|
15
|
+
(zero-copy via ``memoryview``) and the layout order is recorded
|
|
16
|
+
so the strides recovered for any view remain valid against the
|
|
17
|
+
exact bytes we wrote.
|
|
18
|
+
* Roots with non-canonical layouts (neither C- nor F-contig — rare,
|
|
19
|
+
usually only created via ``as_strided`` abuse) are not safe to
|
|
20
|
+
reuse as a shared buffer; the codec falls back to a C-contiguous
|
|
21
|
+
copy of the input array, treating it as its own root. Dedup
|
|
22
|
+
against the original parent is lost in that case.
|
|
23
|
+
* Object-dtype arrays hold Python pointers, not data — they fall
|
|
24
|
+
through to plain pickling (the elements may themselves be
|
|
25
|
+
externalized by other codecs).
|
|
26
|
+
|
|
27
|
+
Materialized arrays are independent, writable copies — same mutation
|
|
28
|
+
semantics as plain ``pickle.loads``. The dedup happens at the storage
|
|
29
|
+
layer; reads always allocate a fresh array. (Reads pay one memcpy per
|
|
30
|
+
key, equivalent to pickle's allocate-and-copy. The savings come from
|
|
31
|
+
storing each unique buffer once instead of N times.)
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
from typing import Any, TYPE_CHECKING
|
|
37
|
+
|
|
38
|
+
if TYPE_CHECKING:
|
|
39
|
+
import numpy as np
|
|
40
|
+
|
|
41
|
+
from .base import ChunkReader, ChunkSink
|
|
42
|
+
|
|
43
|
+
# Below this size, the chunk indirection costs more than the savings.
|
|
44
|
+
# Tunable via constructor; this default is conservative for IndexedDB
|
|
45
|
+
# where each entry has hundreds of bytes of fixed overhead.
|
|
46
|
+
DEFAULT_MIN_BYTES = 1024
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _dtype_to_descr(dt) -> Any:
|
|
50
|
+
"""Serialize a dtype to a JSON-safe descriptor.
|
|
51
|
+
|
|
52
|
+
``dtype.str`` would lose metadata: a datetime64[ns] collapses to
|
|
53
|
+
``'<M8'`` (no unit), a structured dtype collapses to ``'|V20'``
|
|
54
|
+
(no fields). ``np.lib.format.dtype_to_descr`` is the ``.npy``
|
|
55
|
+
format's serializer and preserves both — round-trips through
|
|
56
|
+
``descr_to_dtype`` for every dtype numpy can ``.npy``-save.
|
|
57
|
+
"""
|
|
58
|
+
from numpy.lib import format as npf
|
|
59
|
+
|
|
60
|
+
return npf.dtype_to_descr(dt)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _descr_to_dtype(descr):
|
|
64
|
+
from numpy.lib import format as npf
|
|
65
|
+
|
|
66
|
+
return npf.descr_to_dtype(descr)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _root_and_offset(arr: "np.ndarray") -> tuple["np.ndarray", int]:
|
|
70
|
+
"""Walk ``.base`` to the root buffer; return (root, byte_offset).
|
|
71
|
+
|
|
72
|
+
Returns ``(arr, 0)`` if the .base chain leads somewhere we can't
|
|
73
|
+
safely interpret as a parent buffer (foreign owner, mismatched
|
|
74
|
+
address arithmetic).
|
|
75
|
+
"""
|
|
76
|
+
import numpy as np
|
|
77
|
+
|
|
78
|
+
root = arr
|
|
79
|
+
while isinstance(root.base, np.ndarray):
|
|
80
|
+
root = root.base
|
|
81
|
+
|
|
82
|
+
# If the root is identical to the input, no view computation needed.
|
|
83
|
+
if root is arr:
|
|
84
|
+
return arr, 0
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
arr_addr = arr.__array_interface__["data"][0]
|
|
88
|
+
root_addr = root.__array_interface__["data"][0]
|
|
89
|
+
except (KeyError, AttributeError, TypeError):
|
|
90
|
+
return arr, 0
|
|
91
|
+
|
|
92
|
+
offset = arr_addr - root_addr
|
|
93
|
+
if offset < 0 or offset + arr.nbytes > root.nbytes:
|
|
94
|
+
# The view's memory extends outside the root we walked to —
|
|
95
|
+
# likely a strided view with negative strides or some unusual
|
|
96
|
+
# layout. Treat the array as its own root for safety.
|
|
97
|
+
return arr, 0
|
|
98
|
+
return root, offset
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class NumpyCodec:
|
|
102
|
+
"""Codec for :class:`numpy.ndarray` instances."""
|
|
103
|
+
|
|
104
|
+
name = "np"
|
|
105
|
+
|
|
106
|
+
def __init__(self, min_bytes: int = DEFAULT_MIN_BYTES) -> None:
|
|
107
|
+
self.min_bytes = min_bytes
|
|
108
|
+
|
|
109
|
+
def try_externalize(self, obj: Any, sink: ChunkSink) -> Any | None:
|
|
110
|
+
try:
|
|
111
|
+
import numpy as np
|
|
112
|
+
except ImportError:
|
|
113
|
+
return None
|
|
114
|
+
if not isinstance(obj, np.ndarray):
|
|
115
|
+
return None
|
|
116
|
+
# Object/struct-with-object dtypes contain Python pointers,
|
|
117
|
+
# not data — let pickle handle element-wise so other codecs
|
|
118
|
+
# can fire on the elements.
|
|
119
|
+
if obj.dtype.hasobject:
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
is_view = isinstance(obj.base, np.ndarray)
|
|
123
|
+
# Small standalone arrays: chunk overhead exceeds savings.
|
|
124
|
+
if not is_view and obj.nbytes < self.min_bytes:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
root, byte_offset = _root_and_offset(obj)
|
|
128
|
+
|
|
129
|
+
# Pick a layout we can faithfully round-trip. C-contig and
|
|
130
|
+
# F-contig roots are both contiguous blocks of bytes — we
|
|
131
|
+
# store them verbatim and tag the order so materialize can
|
|
132
|
+
# reshape correctly. A non-contiguous root would force us to
|
|
133
|
+
# canonicalize the bytes (via ``tobytes()``) but the recorded
|
|
134
|
+
# strides would no longer describe that byte layout, leading
|
|
135
|
+
# to data corruption on reconstruction. Fall back to a
|
|
136
|
+
# C-contig copy of ``obj`` instead, treating it as its own
|
|
137
|
+
# root (loses dedup against the original parent for this
|
|
138
|
+
# branch, but keeps correctness).
|
|
139
|
+
if root.flags["C_CONTIGUOUS"]:
|
|
140
|
+
# ``.view(uint8)`` first: not all dtypes are buffer-protocol
|
|
141
|
+
# compatible (datetime64, timedelta64, structured void
|
|
142
|
+
# types all reject ``memoryview(arr)``), but every numpy
|
|
143
|
+
# array can be reinterpreted as raw bytes via a uint8 view,
|
|
144
|
+
# which then *is* buffer-compatible. The resulting view
|
|
145
|
+
# shares memory with ``root``, so this is zero-copy.
|
|
146
|
+
ref = sink.put(memoryview(root.view(np.uint8)))
|
|
147
|
+
order = "C"
|
|
148
|
+
elif root.flags["F_CONTIGUOUS"]:
|
|
149
|
+
# ``memoryview.cast`` only accepts C-contig sources, so
|
|
150
|
+
# collapse to a 1-D view in memory order. ``ravel(order='K')``
|
|
151
|
+
# is zero-copy when strides are positive (always true for
|
|
152
|
+
# F-contig arrays produced via numpy's normal APIs); the
|
|
153
|
+
# uint8 view is then buffer-compatible regardless of dtype.
|
|
154
|
+
ref = sink.put(memoryview(np.ravel(root, order="K").view(np.uint8)))
|
|
155
|
+
order = "F"
|
|
156
|
+
else:
|
|
157
|
+
canonical = np.ascontiguousarray(obj)
|
|
158
|
+
ref = sink.put(memoryview(canonical.view(np.uint8)))
|
|
159
|
+
return {
|
|
160
|
+
"ref": ref,
|
|
161
|
+
"root_shape": list(canonical.shape),
|
|
162
|
+
"root_dtype": _dtype_to_descr(canonical.dtype),
|
|
163
|
+
"shape": list(obj.shape),
|
|
164
|
+
"dtype": _dtype_to_descr(obj.dtype),
|
|
165
|
+
"strides": list(canonical.strides),
|
|
166
|
+
"offset": 0,
|
|
167
|
+
"order": "C",
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return {
|
|
171
|
+
"ref": ref,
|
|
172
|
+
"root_shape": list(root.shape),
|
|
173
|
+
"root_dtype": _dtype_to_descr(root.dtype),
|
|
174
|
+
"shape": list(obj.shape),
|
|
175
|
+
"dtype": _dtype_to_descr(obj.dtype),
|
|
176
|
+
"strides": list(obj.strides),
|
|
177
|
+
"offset": int(byte_offset),
|
|
178
|
+
"order": order,
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
def materialize(self, token: Any, reader: ChunkReader) -> Any:
|
|
182
|
+
import numpy as np
|
|
183
|
+
|
|
184
|
+
raw = reader.get(token["ref"])
|
|
185
|
+
root_dtype = _descr_to_dtype(token["root_dtype"])
|
|
186
|
+
# ``order`` defaults to 'C' for tokens written before the field
|
|
187
|
+
# existed (those tokens are only correct for C-contig roots,
|
|
188
|
+
# which is the only path the older code took without bugs).
|
|
189
|
+
order = token.get("order", "C")
|
|
190
|
+
|
|
191
|
+
offset = token["offset"]
|
|
192
|
+
out_dtype = _descr_to_dtype(token["dtype"])
|
|
193
|
+
out_shape = tuple(token["shape"])
|
|
194
|
+
out_strides = tuple(token["strides"])
|
|
195
|
+
root_shape = tuple(token["root_shape"])
|
|
196
|
+
|
|
197
|
+
# Build a read-only view of the requested data first (zero-copy
|
|
198
|
+
# against the chunk bytes), then ``np.array(view)`` allocates a
|
|
199
|
+
# fresh, writable, independent copy. This matches plain
|
|
200
|
+
# ``pickle.loads`` semantics: the array the caller gets back is
|
|
201
|
+
# safe to mutate and won't surprise neighbouring keys that share
|
|
202
|
+
# the same chunk on the underlying store. The dedup story still
|
|
203
|
+
# holds — chunks are stored once on disk, decoded into private
|
|
204
|
+
# copies on demand.
|
|
205
|
+
|
|
206
|
+
# Fast path: the array IS the root (identical shape/dtype, no
|
|
207
|
+
# offset). Reshape the bytes in the recorded memory order.
|
|
208
|
+
if offset == 0 and out_shape == root_shape and out_dtype == root_dtype:
|
|
209
|
+
view = np.frombuffer(raw, dtype=root_dtype).reshape(root_shape, order=order)
|
|
210
|
+
# ``order='K'`` is np.array's default but we spell it out
|
|
211
|
+
# so an F-contig source comes back as F-contig.
|
|
212
|
+
return np.array(view, order="K")
|
|
213
|
+
|
|
214
|
+
# General path: reconstruct the view via stride tricks against
|
|
215
|
+
# the raw bytes. Critically, we go through a 1-D uint8 view
|
|
216
|
+
# of ``raw`` rather than a multi-dim view of the reshaped
|
|
217
|
+
# root. ``ndarray.view(uint8)`` on a multi-dim array preserves
|
|
218
|
+
# the leading dimensions, so ``[offset:]`` would silently
|
|
219
|
+
# slice the wrong axis when the root has rank > 1 — producing
|
|
220
|
+
# data corruption for any non-trivial offset (e.g., a
|
|
221
|
+
# row-slice tail of a multi-column DataFrame block).
|
|
222
|
+
flat_bytes = np.frombuffer(raw, dtype=np.uint8)
|
|
223
|
+
if offset:
|
|
224
|
+
flat_bytes = flat_bytes[offset:]
|
|
225
|
+
typed = flat_bytes.view(out_dtype)
|
|
226
|
+
view = np.lib.stride_tricks.as_strided(
|
|
227
|
+
typed,
|
|
228
|
+
shape=out_shape,
|
|
229
|
+
strides=out_strides,
|
|
230
|
+
writeable=False,
|
|
231
|
+
)
|
|
232
|
+
# ``np.array`` honours the view's strides while allocating
|
|
233
|
+
# contiguous, writable storage — exactly the shape/contents
|
|
234
|
+
# the caller would get from a fresh pickle.loads. ``order='K'``
|
|
235
|
+
# picks F-contig vs C-contig based on the input view's stride
|
|
236
|
+
# pattern, so an F-strided slice round-trips as F-contig.
|
|
237
|
+
return np.array(view, order="K")
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Pandas codec — currently a thin alias for the numpy codec.
|
|
2
|
+
|
|
3
|
+
When pandas pickles a ``DataFrame`` or ``Series``, its
|
|
4
|
+
``BlockManager.__reduce__`` exposes the per-block ndarrays as Python
|
|
5
|
+
objects. Pickle visits those ndarrays before reducing them, which is
|
|
6
|
+
exactly when our ``persistent_id`` hook fires — the numpy codec then
|
|
7
|
+
catches the block buffers and chunks them.
|
|
8
|
+
|
|
9
|
+
Net effect: ``compose(NumpyCodec(), ...)`` already deduplicates the
|
|
10
|
+
underlying buffers of DataFrames and Series with no extra code. We
|
|
11
|
+
re-export under a pandas-flavored name for clarity in user code.
|
|
12
|
+
|
|
13
|
+
Caveats (no-op cases that fall back to opaque pickle, no chunking):
|
|
14
|
+
|
|
15
|
+
* Extension dtypes whose pickle path doesn't expose an ndarray:
|
|
16
|
+
``ArrowDtype``, ``CategoricalDtype.codes`` is ndarray-backed
|
|
17
|
+
(so it chunks), but ``categories`` may not always; ``MaskedArray``
|
|
18
|
+
surfaces ``.data`` and ``.mask`` ndarrays cleanly.
|
|
19
|
+
* DataFrames with non-default attrs / index types may grow the
|
|
20
|
+
pickle size slightly because the non-block parts still pickle in
|
|
21
|
+
full. The block buffers — usually 99% of the bytes — still chunk.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from .numpy import NumpyCodec as PandasCodec
|
|
27
|
+
|
|
28
|
+
__all__ = ["PandasCodec"]
|