weftcodec 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- weftcodec/__init__.py +62 -0
- weftcodec/__main__.py +6 -0
- weftcodec/ai_modes.py +3360 -0
- weftcodec/cli.py +252 -0
- weftcodec/crc32c.py +33 -0
- weftcodec/dcp.py +2108 -0
- weftcodec/dependency.py +224 -0
- weftcodec/errors.py +119 -0
- weftcodec/frame.py +455 -0
- weftcodec/hashing.py +56 -0
- weftcodec/huffman.py +212 -0
- weftcodec/identity.py +141 -0
- weftcodec/limits.py +58 -0
- weftcodec/modes.py +718 -0
- weftcodec/orbax.py +1734 -0
- weftcodec/service.py +959 -0
- weftcodec/torch.py +94 -0
- weftcodec/varint.py +88 -0
- weftcodec/wcb.py +310 -0
- weftcodec-1.2.0.dist-info/METADATA +257 -0
- weftcodec-1.2.0.dist-info/RECORD +24 -0
- weftcodec-1.2.0.dist-info/WHEEL +5 -0
- weftcodec-1.2.0.dist-info/entry_points.txt +2 -0
- weftcodec-1.2.0.dist-info/top_level.txt +1 -0
weftcodec/__init__.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Independent Python implementation of the WeftCodec v1 reader.
|
|
2
|
+
|
|
3
|
+
This package reads WFT1 frames and canonical WCB bytes, verifies content roots
|
|
4
|
+
with domain-separated BLAKE3, and exposes a PyTorch-facing dataset over a
|
|
5
|
+
mounted directory. It contains no Rust or Go extension modules.
|
|
6
|
+
|
|
7
|
+
Implemented payload modes: RAW, ZSTD, REFERENCE, DELTA, DICTIONARY,
|
|
8
|
+
STRUCTURED, TENSOR, TOKEN_SEQUENCE, QUANTIZED_TENSOR, FLOAT_TENSOR,
|
|
9
|
+
TENSOR_SHARDED, WEIGHT_DELTA, KV_CACHE, ACTIVATION_STREAM, SEMANTIC_DELTA,
|
|
10
|
+
TENSOR_TILED, and SPARSE_DELTA (every allocated mode 0-16). Reserved and
|
|
11
|
+
experimental mode bytes raise :class:`UnsupportedMode` in
|
|
12
|
+
:mod:`weftcodec.frame`.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from .errors import (
|
|
18
|
+
Corrupt,
|
|
19
|
+
DependencyCycle,
|
|
20
|
+
DependencyDenied,
|
|
21
|
+
DependencyMismatch,
|
|
22
|
+
DependencyMissing,
|
|
23
|
+
LimitExceeded,
|
|
24
|
+
Malformed,
|
|
25
|
+
NonCanonical,
|
|
26
|
+
UnsupportedFeature,
|
|
27
|
+
UnsupportedMode,
|
|
28
|
+
UnsupportedVersion,
|
|
29
|
+
WeftError,
|
|
30
|
+
)
|
|
31
|
+
from .frame import Frame, FrameHeader, IMPLEMENTED_MODES, MODE_NAMES, parse
|
|
32
|
+
from .hashing import content_root, content_root_hex, hash_domain
|
|
33
|
+
from .identity import VerifyResult, verify_bytes, verify_frame
|
|
34
|
+
|
|
35
|
+
__version__ = "1.1.0"
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
"Corrupt",
|
|
39
|
+
"DependencyCycle",
|
|
40
|
+
"DependencyDenied",
|
|
41
|
+
"DependencyMismatch",
|
|
42
|
+
"DependencyMissing",
|
|
43
|
+
"Frame",
|
|
44
|
+
"FrameHeader",
|
|
45
|
+
"IMPLEMENTED_MODES",
|
|
46
|
+
"LimitExceeded",
|
|
47
|
+
"MODE_NAMES",
|
|
48
|
+
"Malformed",
|
|
49
|
+
"NonCanonical",
|
|
50
|
+
"UnsupportedFeature",
|
|
51
|
+
"UnsupportedMode",
|
|
52
|
+
"UnsupportedVersion",
|
|
53
|
+
"VerifyResult",
|
|
54
|
+
"WeftError",
|
|
55
|
+
"__version__",
|
|
56
|
+
"content_root",
|
|
57
|
+
"content_root_hex",
|
|
58
|
+
"hash_domain",
|
|
59
|
+
"parse",
|
|
60
|
+
"verify_bytes",
|
|
61
|
+
"verify_frame",
|
|
62
|
+
]
|