atdata 0.2.0a1__py3-none-any.whl → 0.2.3b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atdata/__init__.py +43 -10
- atdata/_cid.py +144 -0
- atdata/_helpers.py +7 -5
- atdata/_hf_api.py +690 -0
- atdata/_protocols.py +504 -0
- atdata/_schema_codec.py +438 -0
- atdata/_sources.py +508 -0
- atdata/_stub_manager.py +534 -0
- atdata/_type_utils.py +104 -0
- atdata/atmosphere/__init__.py +269 -1
- atdata/atmosphere/_types.py +4 -2
- atdata/atmosphere/client.py +146 -3
- atdata/atmosphere/lens.py +4 -3
- atdata/atmosphere/records.py +168 -7
- atdata/atmosphere/schema.py +29 -82
- atdata/atmosphere/store.py +204 -0
- atdata/cli/__init__.py +222 -0
- atdata/cli/diagnose.py +169 -0
- atdata/cli/local.py +283 -0
- atdata/dataset.py +615 -257
- atdata/lens.py +53 -54
- atdata/local.py +1456 -228
- atdata/promote.py +195 -0
- {atdata-0.2.0a1.dist-info → atdata-0.2.3b1.dist-info}/METADATA +106 -14
- atdata-0.2.3b1.dist-info/RECORD +28 -0
- atdata-0.2.0a1.dist-info/RECORD +0 -16
- {atdata-0.2.0a1.dist-info → atdata-0.2.3b1.dist-info}/WHEEL +0 -0
- {atdata-0.2.0a1.dist-info → atdata-0.2.3b1.dist-info}/entry_points.txt +0 -0
- {atdata-0.2.0a1.dist-info → atdata-0.2.3b1.dist-info}/licenses/LICENSE +0 -0
atdata/lens.py
CHANGED
|
@@ -14,7 +14,7 @@ Key components:
|
|
|
14
14
|
Lenses support the functional programming concept of composable, well-behaved
|
|
15
15
|
transformations that satisfy lens laws (GetPut and PutGet).
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
Examples:
|
|
18
18
|
>>> @packable
|
|
19
19
|
... class FullData:
|
|
20
20
|
... name: str
|
|
@@ -54,21 +54,23 @@ from typing import (
|
|
|
54
54
|
Optional,
|
|
55
55
|
Generic,
|
|
56
56
|
#
|
|
57
|
-
TYPE_CHECKING
|
|
57
|
+
TYPE_CHECKING,
|
|
58
58
|
)
|
|
59
59
|
|
|
60
60
|
if TYPE_CHECKING:
|
|
61
61
|
from .dataset import PackableSample
|
|
62
62
|
|
|
63
|
+
from ._protocols import Packable
|
|
64
|
+
|
|
63
65
|
|
|
64
66
|
##
|
|
65
67
|
# Typing helpers
|
|
66
68
|
|
|
67
|
-
DatasetType: TypeAlias = Type[
|
|
69
|
+
DatasetType: TypeAlias = Type["PackableSample"]
|
|
68
70
|
LensSignature: TypeAlias = Tuple[DatasetType, DatasetType]
|
|
69
71
|
|
|
70
|
-
S = TypeVar(
|
|
71
|
-
V = TypeVar(
|
|
72
|
+
S = TypeVar("S", bound=Packable)
|
|
73
|
+
V = TypeVar("V", bound=Packable)
|
|
72
74
|
type LensGetter[S, V] = Callable[[S], V]
|
|
73
75
|
type LensPutter[S, V] = Callable[[V, S], S]
|
|
74
76
|
|
|
@@ -76,7 +78,8 @@ type LensPutter[S, V] = Callable[[V, S], S]
|
|
|
76
78
|
##
|
|
77
79
|
# Shortcut decorators
|
|
78
80
|
|
|
79
|
-
|
|
81
|
+
|
|
82
|
+
class Lens(Generic[S, V]):
|
|
80
83
|
"""A bidirectional transformation between two sample types.
|
|
81
84
|
|
|
82
85
|
A lens provides a way to view and update data of type ``S`` (source) as if
|
|
@@ -84,11 +87,11 @@ class Lens( Generic[S, V] ):
|
|
|
84
87
|
and an optional putter that transforms ``(V, S) -> S``, enabling updates to
|
|
85
88
|
the view to be reflected back in the source.
|
|
86
89
|
|
|
87
|
-
|
|
90
|
+
Parameters:
|
|
88
91
|
S: The source type, must derive from ``PackableSample``.
|
|
89
92
|
V: The view type, must derive from ``PackableSample``.
|
|
90
93
|
|
|
91
|
-
|
|
94
|
+
Examples:
|
|
92
95
|
>>> @lens
|
|
93
96
|
... def name_lens(full: FullData) -> NameOnly:
|
|
94
97
|
... return NameOnly(name=full.name)
|
|
@@ -98,9 +101,11 @@ class Lens( Generic[S, V] ):
|
|
|
98
101
|
... return FullData(name=view.name, age=source.age)
|
|
99
102
|
"""
|
|
100
103
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
+
# TODO The above has a line for "Parameters:" that should be "Type Parameters:"; this is a temporary fix for `quartodoc` auto-generation bugs.
|
|
105
|
+
|
|
106
|
+
def __init__(
|
|
107
|
+
self, get: LensGetter[S, V], put: Optional[LensPutter[S, V]] = None
|
|
108
|
+
) -> None:
|
|
104
109
|
"""Initialize a lens with a getter and optional putter function.
|
|
105
110
|
|
|
106
111
|
Args:
|
|
@@ -113,23 +118,25 @@ class Lens( Generic[S, V] ):
|
|
|
113
118
|
trivial putter is used that ignores updates to the view.
|
|
114
119
|
|
|
115
120
|
Raises:
|
|
116
|
-
|
|
117
|
-
parameter.
|
|
121
|
+
ValueError: If the getter function doesn't have exactly one parameter.
|
|
118
122
|
"""
|
|
119
123
|
##
|
|
120
124
|
|
|
121
125
|
# Check argument validity
|
|
122
126
|
|
|
123
|
-
sig = inspect.signature(
|
|
124
|
-
input_types = list(
|
|
125
|
-
|
|
126
|
-
|
|
127
|
+
sig = inspect.signature(get)
|
|
128
|
+
input_types = list(sig.parameters.values())
|
|
129
|
+
if len(input_types) != 1:
|
|
130
|
+
raise ValueError(
|
|
131
|
+
f"Lens getter must have exactly one parameter, got {len(input_types)}: "
|
|
132
|
+
f"{[p.name for p in input_types]}"
|
|
133
|
+
)
|
|
127
134
|
|
|
128
135
|
# Update function details for this object as returned by annotation
|
|
129
|
-
functools.update_wrapper(
|
|
136
|
+
functools.update_wrapper(self, get)
|
|
130
137
|
|
|
131
|
-
self.source_type: Type[
|
|
132
|
-
self.view_type: Type[
|
|
138
|
+
self.source_type: Type[Packable] = input_types[0].annotation
|
|
139
|
+
self.view_type: Type[Packable] = sig.return_annotation
|
|
133
140
|
|
|
134
141
|
# Store the getter
|
|
135
142
|
self._getter = get
|
|
@@ -137,14 +144,15 @@ class Lens( Generic[S, V] ):
|
|
|
137
144
|
# Determine and store the putter
|
|
138
145
|
if put is None:
|
|
139
146
|
# Trivial putter does not update the source
|
|
140
|
-
def _trivial_put(
|
|
147
|
+
def _trivial_put(v: V, s: S) -> S:
|
|
141
148
|
return s
|
|
149
|
+
|
|
142
150
|
put = _trivial_put
|
|
143
151
|
self._putter = put
|
|
144
|
-
|
|
152
|
+
|
|
145
153
|
#
|
|
146
154
|
|
|
147
|
-
def putter(
|
|
155
|
+
def putter(self, put: LensPutter[S, V]) -> LensPutter[S, V]:
|
|
148
156
|
"""Decorator to register a putter function for this lens.
|
|
149
157
|
|
|
150
158
|
Args:
|
|
@@ -154,18 +162,18 @@ class Lens( Generic[S, V] ):
|
|
|
154
162
|
Returns:
|
|
155
163
|
The putter function, allowing this to be used as a decorator.
|
|
156
164
|
|
|
157
|
-
|
|
165
|
+
Examples:
|
|
158
166
|
>>> @my_lens.putter
|
|
159
167
|
... def my_lens_put(view: ViewType, source: SourceType) -> SourceType:
|
|
160
|
-
... return SourceType(
|
|
168
|
+
... return SourceType(field=view.field, other=source.other)
|
|
161
169
|
"""
|
|
162
170
|
##
|
|
163
171
|
self._putter = put
|
|
164
172
|
return put
|
|
165
|
-
|
|
173
|
+
|
|
166
174
|
# Methods to actually execute transformations
|
|
167
175
|
|
|
168
|
-
def put(
|
|
176
|
+
def put(self, v: V, s: S) -> S:
|
|
169
177
|
"""Update the source based on a modified view.
|
|
170
178
|
|
|
171
179
|
Args:
|
|
@@ -175,9 +183,9 @@ class Lens( Generic[S, V] ):
|
|
|
175
183
|
Returns:
|
|
176
184
|
An updated source of type ``S`` that reflects changes from the view.
|
|
177
185
|
"""
|
|
178
|
-
return self._putter(
|
|
186
|
+
return self._putter(v, s)
|
|
179
187
|
|
|
180
|
-
def get(
|
|
188
|
+
def get(self, s: S) -> V:
|
|
181
189
|
"""Transform the source into the view type.
|
|
182
190
|
|
|
183
191
|
Args:
|
|
@@ -186,23 +194,14 @@ class Lens( Generic[S, V] ):
|
|
|
186
194
|
Returns:
|
|
187
195
|
A view of the source as type ``V``.
|
|
188
196
|
"""
|
|
189
|
-
return self(
|
|
190
|
-
|
|
191
|
-
# Convenience to enable calling the lens as its getter
|
|
192
|
-
|
|
193
|
-
def __call__( self, s: S ) -> V:
|
|
194
|
-
"""Apply the lens transformation (same as ``get()``).
|
|
197
|
+
return self(s)
|
|
195
198
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
Returns:
|
|
200
|
-
A view of the source as type ``V``.
|
|
201
|
-
"""
|
|
202
|
-
return self._getter( s )
|
|
199
|
+
def __call__(self, s: S) -> V:
|
|
200
|
+
"""Apply the lens transformation (same as ``get()``)."""
|
|
201
|
+
return self._getter(s)
|
|
203
202
|
|
|
204
203
|
|
|
205
|
-
def lens(
|
|
204
|
+
def lens(f: LensGetter[S, V]) -> Lens[S, V]:
|
|
206
205
|
"""Decorator to create and register a lens transformation.
|
|
207
206
|
|
|
208
207
|
This decorator converts a getter function into a ``Lens`` object and
|
|
@@ -216,7 +215,7 @@ def lens( f: LensGetter[S, V] ) -> Lens[S, V]:
|
|
|
216
215
|
A ``Lens[S, V]`` object that can be called to apply the transformation
|
|
217
216
|
or decorated with ``@lens_name.putter`` to add a putter function.
|
|
218
217
|
|
|
219
|
-
|
|
218
|
+
Examples:
|
|
220
219
|
>>> @lens
|
|
221
220
|
... def extract_name(full: FullData) -> NameOnly:
|
|
222
221
|
... return NameOnly(name=full.name)
|
|
@@ -225,8 +224,8 @@ def lens( f: LensGetter[S, V] ) -> Lens[S, V]:
|
|
|
225
224
|
... def extract_name_put(view: NameOnly, source: FullData) -> FullData:
|
|
226
225
|
... return FullData(name=view.name, age=source.age)
|
|
227
226
|
"""
|
|
228
|
-
ret = Lens[S, V](
|
|
229
|
-
_network.register(
|
|
227
|
+
ret = Lens[S, V](f)
|
|
228
|
+
_network.register(ret)
|
|
230
229
|
return ret
|
|
231
230
|
|
|
232
231
|
|
|
@@ -255,11 +254,11 @@ class LensNetwork:
|
|
|
255
254
|
|
|
256
255
|
def __init__(self):
|
|
257
256
|
"""Initialize the lens registry (only on first instantiation)."""
|
|
258
|
-
if not hasattr(self,
|
|
257
|
+
if not hasattr(self, "_initialized"): # Check if already initialized
|
|
259
258
|
self._registry: Dict[LensSignature, Lens] = dict()
|
|
260
259
|
self._initialized = True
|
|
261
|
-
|
|
262
|
-
def register(
|
|
260
|
+
|
|
261
|
+
def register(self, _lens: Lens):
|
|
263
262
|
"""Register a lens as the canonical transformation between two types.
|
|
264
263
|
|
|
265
264
|
Args:
|
|
@@ -271,8 +270,8 @@ class LensNetwork:
|
|
|
271
270
|
overwritten.
|
|
272
271
|
"""
|
|
273
272
|
self._registry[_lens.source_type, _lens.view_type] = _lens
|
|
274
|
-
|
|
275
|
-
def transform(
|
|
273
|
+
|
|
274
|
+
def transform(self, source: DatasetType, view: DatasetType) -> Lens:
|
|
276
275
|
"""Look up the lens transformation between two sample types.
|
|
277
276
|
|
|
278
277
|
Args:
|
|
@@ -289,12 +288,12 @@ class LensNetwork:
|
|
|
289
288
|
Currently only supports direct transformations. Compositional
|
|
290
289
|
transformations (chaining multiple lenses) are not yet implemented.
|
|
291
290
|
"""
|
|
292
|
-
ret = self._registry.get(
|
|
291
|
+
ret = self._registry.get((source, view), None)
|
|
293
292
|
if ret is None:
|
|
294
|
-
raise ValueError(
|
|
293
|
+
raise ValueError(f"No registered lens from source {source} to view {view}")
|
|
295
294
|
|
|
296
295
|
return ret
|
|
297
296
|
|
|
298
297
|
|
|
299
298
|
# Global singleton registry instance
|
|
300
|
-
_network = LensNetwork()
|
|
299
|
+
_network = LensNetwork()
|