zarr-template 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from .impl import (
|
|
2
|
+
TemplateChunkKeyEncodingConfiguration,
|
|
3
|
+
TemplateChunkKeyEncodingMetadata,
|
|
4
|
+
)
|
|
5
|
+
from .zarr_python import TemplateChunkKeyEncoding
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"TemplateChunkKeyEncoding",
|
|
9
|
+
"TemplateChunkKeyEncodingConfiguration",
|
|
10
|
+
"TemplateChunkKeyEncodingMetadata",
|
|
11
|
+
]
|
zarr_template/impl.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import Iterable, Sequence
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Literal, NotRequired, Self, TypedDict
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _pad(val: int, pad: int):
|
|
12
|
+
return f"{val:0{pad}}"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class CatchAll:
|
|
17
|
+
pad: int = 0
|
|
18
|
+
|
|
19
|
+
def encode(self, coord: Sequence[int], exclude: set[int], separator: str) -> str:
|
|
20
|
+
return separator.join(
|
|
21
|
+
_pad(c, self.pad) for idx, c in enumerate(coord) if idx not in exclude
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Index:
|
|
27
|
+
index: int
|
|
28
|
+
pad: int = 0
|
|
29
|
+
|
|
30
|
+
def pos_idx(self, length: int) -> int:
|
|
31
|
+
if self.index >= 0:
|
|
32
|
+
if self.index >= length:
|
|
33
|
+
raise ValueError("Index longer than coordinate")
|
|
34
|
+
return self.index
|
|
35
|
+
return self.index % length
|
|
36
|
+
|
|
37
|
+
def encode(self, coord: Sequence[int]) -> str:
|
|
38
|
+
return _pad(coord[self.index], self.pad)
|
|
39
|
+
|
|
40
|
+
def is_neg(self) -> bool:
|
|
41
|
+
return self.index < 0
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
type FormatPart = str | CatchAll | Index
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class TemplateChunkKeyEncodingConfiguration(TypedDict):
|
|
48
|
+
"""TypedDict representing the configuration object within the Zarr v3 metadata representing the template chunk key encoding.
|
|
49
|
+
|
|
50
|
+
i.e.
|
|
51
|
+
|
|
52
|
+
{
|
|
53
|
+
"format": "{*}",
|
|
54
|
+
"separator": "/"
|
|
55
|
+
}
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
format: str
|
|
59
|
+
separator: NotRequired[str | None]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class TemplateChunkKeyEncodingMetadata(TypedDict):
|
|
63
|
+
"""TypedDict representing the Zarr v3 metadata representing the template chunk key encoding.
|
|
64
|
+
|
|
65
|
+
i.e.
|
|
66
|
+
|
|
67
|
+
{
|
|
68
|
+
"name": "template",
|
|
69
|
+
"configuration": {
|
|
70
|
+
... // see TemplateChunkKeyEncodingConfiguration type
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
name: Literal["template"]
|
|
76
|
+
configuration: TemplateChunkKeyEncodingConfiguration
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class TemplateChunkKeyEncoder:
|
|
80
|
+
"""Generic chunk key encoder for use with any zarr library."""
|
|
81
|
+
|
|
82
|
+
def __init__(self, parts: list[FormatPart], separator: str | None = None) -> None:
|
|
83
|
+
self.parts = parts
|
|
84
|
+
self.separator = separator
|
|
85
|
+
self.has_catchall = False
|
|
86
|
+
self.idxs: set[int] = set()
|
|
87
|
+
self.largest_idx = -1
|
|
88
|
+
for p in parts:
|
|
89
|
+
if isinstance(p, CatchAll):
|
|
90
|
+
if self.has_catchall:
|
|
91
|
+
raise ValueError("More than one catchall found")
|
|
92
|
+
else:
|
|
93
|
+
self.has_catchall = True
|
|
94
|
+
elif isinstance(p, Index):
|
|
95
|
+
if p.index in self.idxs:
|
|
96
|
+
raise ValueError(f"Index referred to more than once: {self.idxs}")
|
|
97
|
+
self.idxs.add(p.index)
|
|
98
|
+
self.largest_idx = max(self.largest_idx, p.index)
|
|
99
|
+
|
|
100
|
+
@classmethod
|
|
101
|
+
def from_format(cls, format: str, separator: str | None = None) -> Self:
|
|
102
|
+
parts = list(parse_format(format))
|
|
103
|
+
return cls(parts, separator)
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def from_config(cls, config: TemplateChunkKeyEncodingConfiguration) -> Self:
|
|
107
|
+
return cls.from_format(**config)
|
|
108
|
+
|
|
109
|
+
@classmethod
|
|
110
|
+
def from_metadata(cls, metadata: TemplateChunkKeyEncodingMetadata) -> Self:
|
|
111
|
+
name = metadata.get("name")
|
|
112
|
+
if name != "template":
|
|
113
|
+
raise ValueError(f"Expected name 'template', got '{name!r}'")
|
|
114
|
+
return cls.from_config(metadata.get("configuration"))
|
|
115
|
+
|
|
116
|
+
def encode(self, coord: Sequence[int]) -> str:
|
|
117
|
+
if self.largest_idx > len(coord):
|
|
118
|
+
raise ValueError(
|
|
119
|
+
f"Coordinate is too short; expected at least {self.largest_idx} elements"
|
|
120
|
+
)
|
|
121
|
+
out_parts: list[str] = []
|
|
122
|
+
catchall: tuple[int, CatchAll] | None = None
|
|
123
|
+
used_idxs = set()
|
|
124
|
+
for part in self.parts:
|
|
125
|
+
if isinstance(part, str):
|
|
126
|
+
out_parts.append(part)
|
|
127
|
+
elif isinstance(part, CatchAll):
|
|
128
|
+
if catchall is not None:
|
|
129
|
+
raise ValueError("More than one catchall found")
|
|
130
|
+
catchall = (len(out_parts), part)
|
|
131
|
+
out_parts.append("") # to be filled later
|
|
132
|
+
elif isinstance(part, Index):
|
|
133
|
+
pos = part.pos_idx(len(coord))
|
|
134
|
+
if pos in used_idxs:
|
|
135
|
+
raise ValueError(f"Index {pos} used more than once")
|
|
136
|
+
used_idxs.add(pos)
|
|
137
|
+
out_parts.append(part.encode(coord))
|
|
138
|
+
else:
|
|
139
|
+
raise TypeError(f"Unknown template encoder part type: {type(part)}")
|
|
140
|
+
|
|
141
|
+
if catchall is not None:
|
|
142
|
+
if self.separator is None:
|
|
143
|
+
raise ValueError("Format had catchall but no separator given")
|
|
144
|
+
out_parts[catchall[0]] = catchall[1].encode(
|
|
145
|
+
coord, used_idxs, self.separator
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
return "".join(out_parts)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _parse_braced(s: str) -> CatchAll | Index:
|
|
152
|
+
"""Return index (or none for catchall) and padding (or none)"""
|
|
153
|
+
logger.debug("parsing braced expression %s", s)
|
|
154
|
+
before, *after = s.split(":")
|
|
155
|
+
if len(after) > 1:
|
|
156
|
+
raise ValueError(f"Only expected one colon in '{s}'")
|
|
157
|
+
if not after:
|
|
158
|
+
padding = 0
|
|
159
|
+
else:
|
|
160
|
+
post = after.pop()
|
|
161
|
+
if not post.startswith("0"):
|
|
162
|
+
raise ValueError(f"Unknown padding format '{post}'")
|
|
163
|
+
padding = int(post[1:])
|
|
164
|
+
|
|
165
|
+
if before == "*":
|
|
166
|
+
return CatchAll(padding)
|
|
167
|
+
return Index(int(before), padding)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _strget(s: str, idx: int) -> str | None:
|
|
171
|
+
if idx >= len(s) or idx < 0:
|
|
172
|
+
return None
|
|
173
|
+
return s[idx]
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def parse_format(fmt: str) -> Iterable[FormatPart]:
|
|
177
|
+
idx = 0
|
|
178
|
+
buf: list[str] = []
|
|
179
|
+
in_brace = False
|
|
180
|
+
has_catchall = False
|
|
181
|
+
|
|
182
|
+
while idx < len(fmt):
|
|
183
|
+
c = fmt[idx]
|
|
184
|
+
idx += 1
|
|
185
|
+
if in_brace and c == "}":
|
|
186
|
+
in_brace = False
|
|
187
|
+
braced = _parse_braced("".join(buf))
|
|
188
|
+
if isinstance(braced, CatchAll):
|
|
189
|
+
if has_catchall:
|
|
190
|
+
raise ValueError("More than one catchall variant found")
|
|
191
|
+
else:
|
|
192
|
+
has_catchall = True
|
|
193
|
+
logger.debug("emitting %s", braced)
|
|
194
|
+
yield braced
|
|
195
|
+
buf.clear()
|
|
196
|
+
elif not in_brace and c == "{":
|
|
197
|
+
if (c2 := _strget(fmt, idx)) and c2 == "{":
|
|
198
|
+
buf.append("{")
|
|
199
|
+
idx += 1
|
|
200
|
+
continue
|
|
201
|
+
if buf:
|
|
202
|
+
s = "".join(buf)
|
|
203
|
+
logger.debug("emitting '%s'", s)
|
|
204
|
+
yield s
|
|
205
|
+
buf.clear()
|
|
206
|
+
in_brace = True
|
|
207
|
+
continue
|
|
208
|
+
elif not in_brace and c == "}":
|
|
209
|
+
if (c2 := _strget(fmt, idx)) and c2 == "}":
|
|
210
|
+
buf.append("}")
|
|
211
|
+
idx += 1
|
|
212
|
+
continue
|
|
213
|
+
raise ValueError("Unmatched closing brace")
|
|
214
|
+
else:
|
|
215
|
+
buf.append(c)
|
|
216
|
+
|
|
217
|
+
if in_brace:
|
|
218
|
+
raise ValueError("Unmatched opening brace")
|
|
219
|
+
if buf:
|
|
220
|
+
s = "".join(buf)
|
|
221
|
+
logger.debug("emitting '%s'", s)
|
|
222
|
+
yield s
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import ClassVar, Literal, cast
|
|
3
|
+
|
|
4
|
+
from zarr.core.chunk_key_encodings import ChunkKeyEncoding
|
|
5
|
+
from zarr.core.common import JSON
|
|
6
|
+
from zarr.registry import register_chunk_key_encoding
|
|
7
|
+
|
|
8
|
+
from .impl import (
|
|
9
|
+
TemplateChunkKeyEncoder,
|
|
10
|
+
TemplateChunkKeyEncodingConfiguration,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class TemplateChunkKeyEncoding(ChunkKeyEncoding):
|
|
16
|
+
"""Template chunk key encoding for use with zarr-python."""
|
|
17
|
+
|
|
18
|
+
format: str
|
|
19
|
+
separator: str | None = None
|
|
20
|
+
encoder: TemplateChunkKeyEncoder = field(init=False)
|
|
21
|
+
name: ClassVar[Literal["template"]] = "template"
|
|
22
|
+
|
|
23
|
+
def __post_init__(self):
|
|
24
|
+
encoder = TemplateChunkKeyEncoder.from_format(self.format, self.separator)
|
|
25
|
+
object.__setattr__(self, "encoder", encoder)
|
|
26
|
+
|
|
27
|
+
def encode_chunk_key(self, chunk_coords: tuple[int, ...]) -> str:
|
|
28
|
+
return self.encoder.encode(chunk_coords)
|
|
29
|
+
|
|
30
|
+
def to_dict(self) -> dict[str, JSON]:
|
|
31
|
+
config: TemplateChunkKeyEncodingConfiguration = {"format": self.format}
|
|
32
|
+
if self.separator:
|
|
33
|
+
config["separator"] = self.separator
|
|
34
|
+
|
|
35
|
+
return {"name": self.name, "configuration": cast(JSON, config)}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
register_chunk_key_encoding("template", TemplateChunkKeyEncoding)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: zarr-template
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Implementation of the template extension for chunk key encoding in zarr v3
|
|
5
|
+
Requires-Dist: zarr>=3.4.0
|
|
6
|
+
Requires-Python: >=3.12
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
|
|
9
|
+
# template-cke
|
|
10
|
+
|
|
11
|
+
Implementation of the [template extension](https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-key-encodings/template) for [chunk key encoding](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/core/index.html#array-metadata-chunk-key-encoding) in [Zarr v3](https://zarr.dev/).
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
zarr_template/__init__.py,sha256=EAEYYenKGmP4xworRala9o7A3fQ3AagoYSr8Tw8pfbE,285
|
|
2
|
+
zarr_template/impl.py,sha256=9WzLc4J1SxN6WHh3iWNr8vwWJtZmPqEN51ameoBvGNk,6838
|
|
3
|
+
zarr_template/zarr_python.py,sha256=jLFz4Exh7sBDjxTNWKIwye74Yjg0kvx9OIdNFyx_-0g,1253
|
|
4
|
+
zarr_template-0.1.0.dist-info/WHEEL,sha256=si2p8vW03RnhyBnhWl4tj-ObJa78SpE-c9HsEtblGR0,80
|
|
5
|
+
zarr_template-0.1.0.dist-info/entry_points.txt,sha256=YZD37c-yCBaI2aZ1ThTKdxbTbqhRgxfIrCYX2yof-nQ,77
|
|
6
|
+
zarr_template-0.1.0.dist-info/METADATA,sha256=J9WYcxHmEisXQOXM4LRIb14R_e9PEB_dje5xeJjGS_M,539
|
|
7
|
+
zarr_template-0.1.0.dist-info/RECORD,,
|