atdata 0.3.0b1__py3-none-any.whl → 0.3.2b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atdata/__init__.py +11 -0
- atdata/_cid.py +0 -21
- atdata/_helpers.py +12 -0
- atdata/_hf_api.py +46 -1
- atdata/_logging.py +43 -0
- atdata/_protocols.py +81 -182
- atdata/_schema_codec.py +2 -2
- atdata/_sources.py +24 -4
- atdata/_stub_manager.py +5 -25
- atdata/atmosphere/__init__.py +60 -21
- atdata/atmosphere/_lexicon_types.py +595 -0
- atdata/atmosphere/_types.py +73 -245
- atdata/atmosphere/client.py +64 -12
- atdata/atmosphere/lens.py +60 -53
- atdata/atmosphere/records.py +291 -100
- atdata/atmosphere/schema.py +91 -65
- atdata/atmosphere/store.py +68 -66
- atdata/cli/__init__.py +16 -16
- atdata/cli/diagnose.py +2 -2
- atdata/cli/{local.py → infra.py} +10 -10
- atdata/dataset.py +266 -47
- atdata/index/__init__.py +54 -0
- atdata/{local → index}/_entry.py +6 -2
- atdata/{local → index}/_index.py +617 -72
- atdata/{local → index}/_schema.py +5 -5
- atdata/lexicons/__init__.py +127 -0
- atdata/lexicons/ac.foundation.dataset.arrayFormat.json +16 -0
- atdata/lexicons/ac.foundation.dataset.getLatestSchema.json +78 -0
- atdata/lexicons/ac.foundation.dataset.lens.json +101 -0
- atdata/lexicons/ac.foundation.dataset.record.json +117 -0
- atdata/lexicons/ac.foundation.dataset.schema.json +107 -0
- atdata/lexicons/ac.foundation.dataset.schemaType.json +16 -0
- atdata/lexicons/ac.foundation.dataset.storageBlobs.json +46 -0
- atdata/lexicons/ac.foundation.dataset.storageExternal.json +25 -0
- atdata/lexicons/ac.foundation.dataset.storageHttp.json +45 -0
- atdata/lexicons/ac.foundation.dataset.storageS3.json +61 -0
- atdata/lexicons/ndarray_shim.json +16 -0
- atdata/local/__init__.py +12 -13
- atdata/local/_repo_legacy.py +3 -3
- atdata/manifest/__init__.py +4 -0
- atdata/manifest/_proxy.py +321 -0
- atdata/promote.py +14 -10
- atdata/repository.py +66 -16
- atdata/stores/__init__.py +23 -0
- atdata/stores/_disk.py +131 -0
- atdata/{local → stores}/_s3.py +134 -112
- atdata/testing.py +12 -8
- {atdata-0.3.0b1.dist-info → atdata-0.3.2b1.dist-info}/METADATA +2 -2
- atdata-0.3.2b1.dist-info/RECORD +71 -0
- atdata-0.3.0b1.dist-info/RECORD +0 -54
- {atdata-0.3.0b1.dist-info → atdata-0.3.2b1.dist-info}/WHEEL +0 -0
- {atdata-0.3.0b1.dist-info → atdata-0.3.2b1.dist-info}/entry_points.txt +0 -0
- {atdata-0.3.0b1.dist-info → atdata-0.3.2b1.dist-info}/licenses/LICENSE +0 -0
atdata/atmosphere/lens.py
CHANGED
|
@@ -9,18 +9,11 @@ Note:
|
|
|
9
9
|
implementations.
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
|
-
from typing import Optional
|
|
12
|
+
from typing import Optional, TYPE_CHECKING
|
|
13
13
|
|
|
14
|
-
from .client import
|
|
15
|
-
from ._types import
|
|
16
|
-
|
|
17
|
-
LensRecord,
|
|
18
|
-
CodeReference,
|
|
19
|
-
LEXICON_NAMESPACE,
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
# Import for type checking only
|
|
23
|
-
from typing import TYPE_CHECKING
|
|
14
|
+
from .client import Atmosphere
|
|
15
|
+
from ._types import AtUri, LEXICON_NAMESPACE
|
|
16
|
+
from ._lexicon_types import LexLensRecord, LexCodeReference
|
|
24
17
|
|
|
25
18
|
if TYPE_CHECKING:
|
|
26
19
|
from ..lens import Lens
|
|
@@ -37,14 +30,13 @@ class LensPublisher:
|
|
|
37
30
|
... def my_lens(source: SourceType) -> TargetType:
|
|
38
31
|
... return TargetType(field=source.other_field)
|
|
39
32
|
>>>
|
|
40
|
-
>>>
|
|
41
|
-
>>> client.login("handle", "password")
|
|
33
|
+
>>> atmo = Atmosphere.login("handle", "password")
|
|
42
34
|
>>>
|
|
43
|
-
>>> publisher = LensPublisher(
|
|
35
|
+
>>> publisher = LensPublisher(atmo)
|
|
44
36
|
>>> uri = publisher.publish(
|
|
45
37
|
... name="my_lens",
|
|
46
|
-
... source_schema_uri="at://did:plc:abc/ac.foundation.dataset.
|
|
47
|
-
... target_schema_uri="at://did:plc:abc/ac.foundation.dataset.
|
|
38
|
+
... source_schema_uri="at://did:plc:abc/ac.foundation.dataset.schema/source",
|
|
39
|
+
... target_schema_uri="at://did:plc:abc/ac.foundation.dataset.schema/target",
|
|
48
40
|
... code_repository="https://github.com/user/repo",
|
|
49
41
|
... code_commit="abc123def456",
|
|
50
42
|
... getter_path="mymodule.lenses:my_lens",
|
|
@@ -57,11 +49,11 @@ class LensPublisher:
|
|
|
57
49
|
records. Users must manually install and trust lens implementations.
|
|
58
50
|
"""
|
|
59
51
|
|
|
60
|
-
def __init__(self, client:
|
|
52
|
+
def __init__(self, client: Atmosphere):
|
|
61
53
|
"""Initialize the lens publisher.
|
|
62
54
|
|
|
63
55
|
Args:
|
|
64
|
-
client: Authenticated
|
|
56
|
+
client: Authenticated Atmosphere instance.
|
|
65
57
|
"""
|
|
66
58
|
self.client = client
|
|
67
59
|
|
|
@@ -71,59 +63,58 @@ class LensPublisher:
|
|
|
71
63
|
name: str,
|
|
72
64
|
source_schema_uri: str,
|
|
73
65
|
target_schema_uri: str,
|
|
66
|
+
code_repository: str,
|
|
67
|
+
code_commit: str,
|
|
68
|
+
getter_path: str,
|
|
69
|
+
putter_path: str,
|
|
74
70
|
description: Optional[str] = None,
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
getter_path: Optional[str] = None,
|
|
78
|
-
putter_path: Optional[str] = None,
|
|
71
|
+
language: Optional[str] = None,
|
|
72
|
+
metadata: Optional[dict] = None,
|
|
79
73
|
rkey: Optional[str] = None,
|
|
80
74
|
) -> AtUri:
|
|
81
75
|
"""Publish a lens transformation record to ATProto.
|
|
82
76
|
|
|
77
|
+
Code references are required by the ATProto lexicon. Each lens must
|
|
78
|
+
point to a getter and putter implementation in a git repository.
|
|
79
|
+
|
|
83
80
|
Args:
|
|
84
81
|
name: Human-readable lens name.
|
|
85
82
|
source_schema_uri: AT URI of the source schema.
|
|
86
83
|
target_schema_uri: AT URI of the target schema.
|
|
87
|
-
description: What this transformation does.
|
|
88
84
|
code_repository: Git repository URL containing the lens code.
|
|
89
85
|
code_commit: Git commit hash for reproducibility.
|
|
90
86
|
getter_path: Module path to the getter function
|
|
91
87
|
(e.g., 'mymodule.lenses:my_getter').
|
|
92
88
|
putter_path: Module path to the putter function
|
|
93
89
|
(e.g., 'mymodule.lenses:my_putter').
|
|
90
|
+
description: What this transformation does.
|
|
91
|
+
language: Programming language (e.g., 'python').
|
|
92
|
+
metadata: Arbitrary metadata dictionary.
|
|
94
93
|
rkey: Optional explicit record key.
|
|
95
94
|
|
|
96
95
|
Returns:
|
|
97
96
|
The AT URI of the created lens record.
|
|
98
|
-
|
|
99
|
-
Raises:
|
|
100
|
-
ValueError: If code references are incomplete.
|
|
101
97
|
"""
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
putter_code = CodeReference(
|
|
115
|
-
repository=code_repository,
|
|
116
|
-
commit=code_commit,
|
|
117
|
-
path=putter_path,
|
|
118
|
-
)
|
|
119
|
-
|
|
120
|
-
lens_record = LensRecord(
|
|
98
|
+
getter_code = LexCodeReference(
|
|
99
|
+
repository=code_repository,
|
|
100
|
+
commit=code_commit,
|
|
101
|
+
path=getter_path,
|
|
102
|
+
)
|
|
103
|
+
putter_code = LexCodeReference(
|
|
104
|
+
repository=code_repository,
|
|
105
|
+
commit=code_commit,
|
|
106
|
+
path=putter_path,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
lens_record = LexLensRecord(
|
|
121
110
|
name=name,
|
|
122
111
|
source_schema=source_schema_uri,
|
|
123
112
|
target_schema=target_schema_uri,
|
|
124
|
-
description=description,
|
|
125
113
|
getter_code=getter_code,
|
|
126
114
|
putter_code=putter_code,
|
|
115
|
+
description=description,
|
|
116
|
+
language=language,
|
|
117
|
+
metadata=metadata,
|
|
127
118
|
)
|
|
128
119
|
|
|
129
120
|
return self.client.create_record(
|
|
@@ -143,6 +134,8 @@ class LensPublisher:
|
|
|
143
134
|
code_repository: str,
|
|
144
135
|
code_commit: str,
|
|
145
136
|
description: Optional[str] = None,
|
|
137
|
+
language: Optional[str] = None,
|
|
138
|
+
metadata: Optional[dict] = None,
|
|
146
139
|
rkey: Optional[str] = None,
|
|
147
140
|
) -> AtUri:
|
|
148
141
|
"""Publish a lens record from an existing Lens object.
|
|
@@ -158,16 +151,16 @@ class LensPublisher:
|
|
|
158
151
|
code_repository: Git repository URL.
|
|
159
152
|
code_commit: Git commit hash.
|
|
160
153
|
description: What this transformation does.
|
|
154
|
+
language: Programming language (e.g., 'python').
|
|
155
|
+
metadata: Arbitrary metadata dictionary.
|
|
161
156
|
rkey: Optional explicit record key.
|
|
162
157
|
|
|
163
158
|
Returns:
|
|
164
159
|
The AT URI of the created lens record.
|
|
165
160
|
"""
|
|
166
|
-
# Extract function names from the lens
|
|
167
161
|
getter_name = lens_obj._getter.__name__
|
|
168
162
|
putter_name = lens_obj._putter.__name__
|
|
169
163
|
|
|
170
|
-
# Get module info if available
|
|
171
164
|
getter_module = getattr(lens_obj._getter, "__module__", "")
|
|
172
165
|
putter_module = getattr(lens_obj._putter, "__module__", "")
|
|
173
166
|
|
|
@@ -178,11 +171,13 @@ class LensPublisher:
|
|
|
178
171
|
name=name,
|
|
179
172
|
source_schema_uri=source_schema_uri,
|
|
180
173
|
target_schema_uri=target_schema_uri,
|
|
181
|
-
description=description,
|
|
182
174
|
code_repository=code_repository,
|
|
183
175
|
code_commit=code_commit,
|
|
184
176
|
getter_path=getter_path,
|
|
185
177
|
putter_path=putter_path,
|
|
178
|
+
description=description,
|
|
179
|
+
language=language,
|
|
180
|
+
metadata=metadata,
|
|
186
181
|
rkey=rkey,
|
|
187
182
|
)
|
|
188
183
|
|
|
@@ -195,8 +190,8 @@ class LensLoader:
|
|
|
195
190
|
it manually.
|
|
196
191
|
|
|
197
192
|
Examples:
|
|
198
|
-
>>>
|
|
199
|
-
>>> loader = LensLoader(
|
|
193
|
+
>>> atmo = Atmosphere.login("handle", "password")
|
|
194
|
+
>>> loader = LensLoader(atmo)
|
|
200
195
|
>>>
|
|
201
196
|
>>> record = loader.get("at://did:plc:abc/ac.foundation.dataset.lens/xyz")
|
|
202
197
|
>>> print(record["name"])
|
|
@@ -204,11 +199,11 @@ class LensLoader:
|
|
|
204
199
|
>>> print(record.get("getterCode", {}).get("repository"))
|
|
205
200
|
"""
|
|
206
201
|
|
|
207
|
-
def __init__(self, client:
|
|
202
|
+
def __init__(self, client: Atmosphere):
|
|
208
203
|
"""Initialize the lens loader.
|
|
209
204
|
|
|
210
205
|
Args:
|
|
211
|
-
client:
|
|
206
|
+
client: Atmosphere instance.
|
|
212
207
|
"""
|
|
213
208
|
self.client = client
|
|
214
209
|
|
|
@@ -235,6 +230,18 @@ class LensLoader:
|
|
|
235
230
|
|
|
236
231
|
return record
|
|
237
232
|
|
|
233
|
+
def get_typed(self, uri: str | AtUri) -> LexLensRecord:
|
|
234
|
+
"""Fetch a lens record and return as a typed object.
|
|
235
|
+
|
|
236
|
+
Args:
|
|
237
|
+
uri: The AT URI of the lens record.
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
LexLensRecord instance.
|
|
241
|
+
"""
|
|
242
|
+
record = self.get(uri)
|
|
243
|
+
return LexLensRecord.from_record(record)
|
|
244
|
+
|
|
238
245
|
def list_all(
|
|
239
246
|
self,
|
|
240
247
|
repo: Optional[str] = None,
|