pystac-ext-storage 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
File without changes
|
|
@@ -0,0 +1,753 @@
|
|
|
1
|
+
"""Implements the Storage Extension.
|
|
2
|
+
|
|
3
|
+
https://github.com/stac-extensions/storage
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
import warnings
|
|
10
|
+
from typing import (
|
|
11
|
+
Any,
|
|
12
|
+
Generic,
|
|
13
|
+
Literal,
|
|
14
|
+
TypeVar,
|
|
15
|
+
cast,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
import pystac
|
|
19
|
+
from pystac.errors import RequiredPropertyMissing
|
|
20
|
+
from pystac.extensions.base import (
|
|
21
|
+
ExtensionManagementMixin,
|
|
22
|
+
PropertiesExtension,
|
|
23
|
+
SummariesExtension,
|
|
24
|
+
)
|
|
25
|
+
from pystac.extensions.hooks import ExtensionHooks
|
|
26
|
+
from pystac.serialization.identify import STACJSONDescription, STACVersionID
|
|
27
|
+
from pystac.utils import StringEnum, get_required, map_opt
|
|
28
|
+
|
|
29
|
+
#: Generalized version of :class:`~pystac.Catalog`, :class:`~pystac.Collection`,
|
|
30
|
+
#: :class:`~pystac.Item`, :class:`~pystac.Asset`, :class:`~pystac.Link`,
|
|
31
|
+
#: or :class:`~pystac.ItemAssetDefinition`
|
|
32
|
+
T = TypeVar(
|
|
33
|
+
"T",
|
|
34
|
+
pystac.Catalog,
|
|
35
|
+
pystac.Collection,
|
|
36
|
+
pystac.Item,
|
|
37
|
+
pystac.Asset,
|
|
38
|
+
pystac.Link,
|
|
39
|
+
pystac.ItemAssetDefinition,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
SCHEMA_URI_PATTERN: str = (
|
|
43
|
+
"https://stac-extensions.github.io/storage/v{version}/schema.json"
|
|
44
|
+
)
|
|
45
|
+
DEFAULT_VERSION: str = "2.0.0"
|
|
46
|
+
SUPPORTED_VERSIONS: list[str] = ["2.0.0", "1.0.0"]
|
|
47
|
+
|
|
48
|
+
PREFIX: str = "storage:"
|
|
49
|
+
|
|
50
|
+
# Field names
|
|
51
|
+
REFS_PROP: str = PREFIX + "refs"
|
|
52
|
+
SCHEMES_PROP: str = PREFIX + "schemes"
|
|
53
|
+
|
|
54
|
+
# Storage scheme object names
|
|
55
|
+
TYPE_PROP: str = "type"
|
|
56
|
+
PLATFORM_PROP: str = "platform"
|
|
57
|
+
REGION_PROP: str = "region"
|
|
58
|
+
REQUESTER_PAYS_PROP: str = "requester_pays"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class StorageSchemeType(StringEnum):
|
|
62
|
+
AWS_S3 = "aws-s3"
|
|
63
|
+
CUSTOM_S3 = "custom-s3"
|
|
64
|
+
AZURE = "ms-azure"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class StorageScheme:
|
|
68
|
+
"""
|
|
69
|
+
Helper class for storage scheme objects.
|
|
70
|
+
|
|
71
|
+
Can set well-defined properties, or if needed,
|
|
72
|
+
any arbitrary property.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
_known_fields = {"type", "platform", "region", "requester_pays"}
|
|
76
|
+
_properties: dict[str, Any]
|
|
77
|
+
|
|
78
|
+
def __init__(self, properties: dict[str, Any]):
|
|
79
|
+
super().__setattr__("_properties", properties)
|
|
80
|
+
|
|
81
|
+
def __setattr__(self, name: str, value: Any) -> None:
|
|
82
|
+
if hasattr(type(self), name):
|
|
83
|
+
object.__setattr__(self, name, value)
|
|
84
|
+
return
|
|
85
|
+
|
|
86
|
+
if name in self._known_fields:
|
|
87
|
+
prop = getattr(type(self), name)
|
|
88
|
+
prop.fset(self, value)
|
|
89
|
+
return
|
|
90
|
+
|
|
91
|
+
props = object.__getattribute__(self, "_properties")
|
|
92
|
+
props[name] = value
|
|
93
|
+
|
|
94
|
+
def __getattr__(self, name: str) -> Any:
|
|
95
|
+
props = object.__getattribute__(self, "_properties")
|
|
96
|
+
|
|
97
|
+
if name in props:
|
|
98
|
+
return props[name]
|
|
99
|
+
|
|
100
|
+
raise AttributeError(name)
|
|
101
|
+
|
|
102
|
+
def __eq__(self, other: object) -> bool:
|
|
103
|
+
if not isinstance(other, StorageScheme):
|
|
104
|
+
return NotImplemented
|
|
105
|
+
return bool(self._properties == other._properties)
|
|
106
|
+
|
|
107
|
+
def __repr__(self) -> str:
|
|
108
|
+
return f"<StorageScheme platform={self.platform}>"
|
|
109
|
+
|
|
110
|
+
def apply(
|
|
111
|
+
self,
|
|
112
|
+
type: str,
|
|
113
|
+
platform: str,
|
|
114
|
+
region: str | None = None,
|
|
115
|
+
requester_pays: bool | None = None,
|
|
116
|
+
**kwargs: Any,
|
|
117
|
+
) -> None:
|
|
118
|
+
self.type = type
|
|
119
|
+
self.platform = platform
|
|
120
|
+
self.region = region
|
|
121
|
+
self.requester_pays = requester_pays
|
|
122
|
+
self._properties.update(kwargs)
|
|
123
|
+
|
|
124
|
+
@classmethod
|
|
125
|
+
def create(
|
|
126
|
+
cls,
|
|
127
|
+
type: str,
|
|
128
|
+
platform: str,
|
|
129
|
+
region: str | None = None,
|
|
130
|
+
requester_pays: bool | None = None,
|
|
131
|
+
**kwargs: Any,
|
|
132
|
+
) -> StorageScheme:
|
|
133
|
+
"""Set the properties for a new StorageScheme object.
|
|
134
|
+
|
|
135
|
+
Additional properties can be set through kwargs to fulfill
|
|
136
|
+
any additional variables in a templated uri.
|
|
137
|
+
|
|
138
|
+
Args:
|
|
139
|
+
type (str): Type identifier for the platform.
|
|
140
|
+
platform (str): The cloud provider where data is stored as URI or URI
|
|
141
|
+
template to the API.
|
|
142
|
+
region (str | None): The region where the data is stored.
|
|
143
|
+
Defaults to None.
|
|
144
|
+
requester_pays (bool | None): requester pays or data manager/cloud
|
|
145
|
+
provider pays. Defaults to None.
|
|
146
|
+
kwargs (dict[str | Any]): Additional properties to set on scheme
|
|
147
|
+
|
|
148
|
+
Returns:
|
|
149
|
+
StorageScheme: storage scheme
|
|
150
|
+
"""
|
|
151
|
+
c = cls({})
|
|
152
|
+
c.apply(
|
|
153
|
+
type=type,
|
|
154
|
+
platform=platform,
|
|
155
|
+
region=region,
|
|
156
|
+
requester_pays=requester_pays,
|
|
157
|
+
**kwargs,
|
|
158
|
+
)
|
|
159
|
+
return c
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def type(self) -> str:
|
|
163
|
+
"""
|
|
164
|
+
Get or set the required type property
|
|
165
|
+
"""
|
|
166
|
+
return cast(
|
|
167
|
+
str,
|
|
168
|
+
get_required(
|
|
169
|
+
self._properties.get(TYPE_PROP),
|
|
170
|
+
self,
|
|
171
|
+
TYPE_PROP,
|
|
172
|
+
),
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
@type.setter
|
|
176
|
+
def type(self, v: str) -> None:
|
|
177
|
+
self._properties[TYPE_PROP] = v
|
|
178
|
+
|
|
179
|
+
@property
|
|
180
|
+
def platform(self) -> str:
|
|
181
|
+
"""
|
|
182
|
+
Get or set the required platform property
|
|
183
|
+
"""
|
|
184
|
+
return cast(
|
|
185
|
+
str,
|
|
186
|
+
get_required(
|
|
187
|
+
self._properties.get(PLATFORM_PROP),
|
|
188
|
+
self,
|
|
189
|
+
PLATFORM_PROP,
|
|
190
|
+
),
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
@platform.setter
|
|
194
|
+
def platform(self, v: str) -> None:
|
|
195
|
+
self._properties[PLATFORM_PROP] = v
|
|
196
|
+
|
|
197
|
+
@property
|
|
198
|
+
def region(self) -> str | None:
|
|
199
|
+
"""
|
|
200
|
+
Get or set the optional region property
|
|
201
|
+
"""
|
|
202
|
+
return self._properties.get(REGION_PROP)
|
|
203
|
+
|
|
204
|
+
@region.setter
|
|
205
|
+
def region(self, v: str | None) -> None:
|
|
206
|
+
if v is not None:
|
|
207
|
+
self._properties[REGION_PROP] = v
|
|
208
|
+
else:
|
|
209
|
+
self._properties.pop(REGION_PROP, None)
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def requester_pays(self) -> bool | None:
|
|
213
|
+
"""
|
|
214
|
+
Get or set the optional requester_pays property
|
|
215
|
+
"""
|
|
216
|
+
return self._properties.get(REQUESTER_PAYS_PROP)
|
|
217
|
+
|
|
218
|
+
@requester_pays.setter
|
|
219
|
+
def requester_pays(self, v: bool | None) -> None:
|
|
220
|
+
if v is not None:
|
|
221
|
+
self._properties[REQUESTER_PAYS_PROP] = v
|
|
222
|
+
else:
|
|
223
|
+
self._properties.pop(REQUESTER_PAYS_PROP, None)
|
|
224
|
+
|
|
225
|
+
def to_dict(self) -> dict[str, Any]:
|
|
226
|
+
"""
|
|
227
|
+
Returns the dictionary encoding of this object
|
|
228
|
+
|
|
229
|
+
Returns:
|
|
230
|
+
dict[str, Any]: The dictionary encoding of this object
|
|
231
|
+
"""
|
|
232
|
+
return self._properties
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class StorageExtension(
|
|
236
|
+
Generic[T],
|
|
237
|
+
PropertiesExtension,
|
|
238
|
+
ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog],
|
|
239
|
+
):
|
|
240
|
+
"""An class that can be used to extend the properties of an
|
|
241
|
+
:class:`~pystac.Catalog`, :class:`~pystac.Collection`, :class:`~pystac.Item`,
|
|
242
|
+
:class:`~pystac.Asset`, :class:`~pystac.Link`, or
|
|
243
|
+
:class:`~pystac.ItemAssetDefinition` with properties from the
|
|
244
|
+
:stac-ext:`Storage Extension <storage>`.
|
|
245
|
+
This class is generic over the type of STAC Object to be extended (e.g.
|
|
246
|
+
:class:`~pystac.Item`, :class:`~pystac.Collection`).
|
|
247
|
+
To create a concrete instance of :class:`StorageExtension`, use the
|
|
248
|
+
:meth:`StorageExtension.ext` method. For example:
|
|
249
|
+
|
|
250
|
+
.. code-block:: python
|
|
251
|
+
|
|
252
|
+
>>> item: pystac.Item = ...
|
|
253
|
+
>>> storage_ext = StorageExtension.ext(item)
|
|
254
|
+
"""
|
|
255
|
+
|
|
256
|
+
name: Literal["storage"] = "storage"
|
|
257
|
+
|
|
258
|
+
@classmethod
|
|
259
|
+
def get_schema_uri(cls) -> str:
|
|
260
|
+
return SCHEMA_URI_PATTERN.format(version=DEFAULT_VERSION)
|
|
261
|
+
|
|
262
|
+
# For type checking purposes only, these methods are overridden in mixins
|
|
263
|
+
def apply(
|
|
264
|
+
self,
|
|
265
|
+
*,
|
|
266
|
+
schemes: dict[str, StorageScheme] | None = None,
|
|
267
|
+
refs: list[str] | None = None,
|
|
268
|
+
) -> None:
|
|
269
|
+
raise NotImplementedError()
|
|
270
|
+
|
|
271
|
+
@property
|
|
272
|
+
def schemes(self) -> dict[str, StorageScheme]:
|
|
273
|
+
raise NotImplementedError()
|
|
274
|
+
|
|
275
|
+
@schemes.setter
|
|
276
|
+
def schemes(self, v: dict[str, StorageScheme]) -> None:
|
|
277
|
+
raise NotImplementedError()
|
|
278
|
+
|
|
279
|
+
def add_scheme(self, key: str, scheme: StorageScheme) -> None:
|
|
280
|
+
raise NotImplementedError()
|
|
281
|
+
|
|
282
|
+
@property
|
|
283
|
+
def refs(self) -> list[str]:
|
|
284
|
+
raise NotImplementedError()
|
|
285
|
+
|
|
286
|
+
@refs.setter
|
|
287
|
+
def refs(self, v: list[str]) -> None:
|
|
288
|
+
raise NotImplementedError()
|
|
289
|
+
|
|
290
|
+
def add_ref(self, ref: str) -> None:
|
|
291
|
+
raise NotImplementedError()
|
|
292
|
+
|
|
293
|
+
@classmethod
|
|
294
|
+
def ext(cls, obj: T, add_if_missing: bool = False) -> StorageExtension[T]:
|
|
295
|
+
"""Extends the given STAC Object with properties from the :stac-ext:`Storage
|
|
296
|
+
Extension <storage>`.
|
|
297
|
+
|
|
298
|
+
This extension can be applied to instances of :class:`~pystac.Catalog`,
|
|
299
|
+
:class:`~pystac.Collection`, :class:`~pystac.Item`, :class:`~pystac.Asset`,
|
|
300
|
+
:class:`~pystac.Link`, or :class:`~pystac.ItemAssetDefinition`.
|
|
301
|
+
|
|
302
|
+
Raises:
|
|
303
|
+
pystac.ExtensionTypeError : If an invalid object type is passed.
|
|
304
|
+
"""
|
|
305
|
+
if isinstance(obj, pystac.Item):
|
|
306
|
+
cls.ensure_has_extension(obj, add_if_missing)
|
|
307
|
+
return cast(StorageExtension[T], ItemStorageExtension(obj))
|
|
308
|
+
|
|
309
|
+
elif isinstance(obj, pystac.Collection):
|
|
310
|
+
cls.ensure_has_extension(obj, add_if_missing)
|
|
311
|
+
return cast(StorageExtension[T], CollectionStorageExtension(obj))
|
|
312
|
+
|
|
313
|
+
elif isinstance(obj, pystac.Catalog):
|
|
314
|
+
cls.ensure_has_extension(obj, add_if_missing)
|
|
315
|
+
return cast(StorageExtension[T], CatalogStorageExtension(obj))
|
|
316
|
+
|
|
317
|
+
elif isinstance(obj, pystac.Asset):
|
|
318
|
+
cls.ensure_owner_has_extension(obj, add_if_missing)
|
|
319
|
+
return cast(StorageExtension[T], AssetStorageExtension(obj))
|
|
320
|
+
|
|
321
|
+
elif isinstance(obj, pystac.Link):
|
|
322
|
+
cls.ensure_owner_has_extension(obj, add_if_missing)
|
|
323
|
+
return cast(StorageExtension[T], LinkStorageExtension(obj))
|
|
324
|
+
|
|
325
|
+
elif isinstance(obj, pystac.ItemAssetDefinition):
|
|
326
|
+
cls.ensure_owner_has_extension(obj, add_if_missing)
|
|
327
|
+
return cast(StorageExtension[T], ItemAssetsStorageExtension(obj))
|
|
328
|
+
|
|
329
|
+
else:
|
|
330
|
+
raise pystac.ExtensionTypeError(cls._ext_error_message(obj))
|
|
331
|
+
|
|
332
|
+
@classmethod
|
|
333
|
+
def summaries(
|
|
334
|
+
cls, obj: pystac.Collection, add_if_missing: bool = False
|
|
335
|
+
) -> SummariesStorageExtension:
|
|
336
|
+
"""Returns the extended summaries object for the given collection."""
|
|
337
|
+
cls.ensure_has_extension(obj, add_if_missing)
|
|
338
|
+
return SummariesStorageExtension(obj)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class _SchemesMixin:
|
|
342
|
+
"""Mixin for objects that support Storage Schemes (Items, Collections, Catalogs)."""
|
|
343
|
+
|
|
344
|
+
properties: dict[str, Any]
|
|
345
|
+
_set_property: Any
|
|
346
|
+
|
|
347
|
+
def apply(
|
|
348
|
+
self,
|
|
349
|
+
*,
|
|
350
|
+
schemes: dict[str, StorageScheme] | None = None,
|
|
351
|
+
refs: list[str] | None = None,
|
|
352
|
+
) -> None:
|
|
353
|
+
if refs is not None:
|
|
354
|
+
raise ValueError("'refs' cannot be applied with this STAC object type.")
|
|
355
|
+
if schemes is None:
|
|
356
|
+
raise RequiredPropertyMissing(
|
|
357
|
+
self,
|
|
358
|
+
SCHEMES_PROP,
|
|
359
|
+
"'schemes' property is required for this object type.",
|
|
360
|
+
)
|
|
361
|
+
self.schemes = schemes
|
|
362
|
+
|
|
363
|
+
@property
|
|
364
|
+
def schemes(self) -> dict[str, StorageScheme]:
|
|
365
|
+
schemes_dict: dict[str, Any] = get_required(
|
|
366
|
+
self.properties.get(SCHEMES_PROP), self, SCHEMES_PROP
|
|
367
|
+
)
|
|
368
|
+
return {k: StorageScheme(v) for k, v in schemes_dict.items()}
|
|
369
|
+
|
|
370
|
+
@schemes.setter
|
|
371
|
+
def schemes(self, v: dict[str, StorageScheme]) -> None:
|
|
372
|
+
v_trans = {k: c.to_dict() for k, c in v.items()}
|
|
373
|
+
self._set_property(SCHEMES_PROP, v_trans)
|
|
374
|
+
|
|
375
|
+
def add_scheme(self, key: str, scheme: StorageScheme) -> None:
|
|
376
|
+
current = self.properties.get(SCHEMES_PROP, {})
|
|
377
|
+
current[key] = scheme.to_dict()
|
|
378
|
+
self._set_property(SCHEMES_PROP, current)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
class _RefsMixin:
|
|
382
|
+
"""Mixin for objects that support Storage Refs (Assets, Links)."""
|
|
383
|
+
|
|
384
|
+
properties: dict[str, Any]
|
|
385
|
+
_set_property: Any
|
|
386
|
+
|
|
387
|
+
def apply(
|
|
388
|
+
self,
|
|
389
|
+
*,
|
|
390
|
+
schemes: dict[str, StorageScheme] | None = None,
|
|
391
|
+
refs: list[str] | None = None,
|
|
392
|
+
) -> None:
|
|
393
|
+
if schemes is not None:
|
|
394
|
+
raise ValueError("'schemes' cannot be applied with this STAC object type.")
|
|
395
|
+
if refs is None:
|
|
396
|
+
raise RequiredPropertyMissing(
|
|
397
|
+
self, REFS_PROP, "'refs' property is required for this object type."
|
|
398
|
+
)
|
|
399
|
+
self.refs = refs
|
|
400
|
+
|
|
401
|
+
@property
|
|
402
|
+
def refs(self) -> list[str]:
|
|
403
|
+
return get_required(self.properties.get(REFS_PROP), self, REFS_PROP)
|
|
404
|
+
|
|
405
|
+
@refs.setter
|
|
406
|
+
def refs(self, v: list[str]) -> None:
|
|
407
|
+
self._set_property(REFS_PROP, v)
|
|
408
|
+
|
|
409
|
+
def add_ref(self, ref: str) -> None:
|
|
410
|
+
try:
|
|
411
|
+
current = self.refs
|
|
412
|
+
if ref not in current:
|
|
413
|
+
current.append(ref)
|
|
414
|
+
self.refs = current
|
|
415
|
+
except RequiredPropertyMissing:
|
|
416
|
+
self.refs = [ref]
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
class ItemStorageExtension(_SchemesMixin, StorageExtension[pystac.Item]):
|
|
420
|
+
def __init__(self, item: pystac.Item):
|
|
421
|
+
self.item = item
|
|
422
|
+
self.properties = item.properties
|
|
423
|
+
|
|
424
|
+
def __repr__(self) -> str:
|
|
425
|
+
return f"<ItemStorageExtension Item id={self.item.id}>"
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
class CatalogStorageExtension(_SchemesMixin, StorageExtension[pystac.Catalog]):
|
|
429
|
+
"""A concrete implementation of :class:`StorageExtension` on an
|
|
430
|
+
:class:`~pystac.Catalog` that extends the properties of the Catalog to include
|
|
431
|
+
properties defined in the :stac-ext:`Storage Extension <storage>`.
|
|
432
|
+
|
|
433
|
+
This class should generally not be instantiated directly. Instead, call
|
|
434
|
+
:meth:`StorageExtension.ext` on an :class:`~pystac.Catalog` to extend it.
|
|
435
|
+
"""
|
|
436
|
+
|
|
437
|
+
catalog: pystac.Catalog
|
|
438
|
+
"""The :class:`~pystac.Catalog` being extended."""
|
|
439
|
+
|
|
440
|
+
properties: dict[str, Any]
|
|
441
|
+
"""The :class:`~pystac.Catalog` properties, including extension properties."""
|
|
442
|
+
|
|
443
|
+
def __init__(self, catalog: pystac.Catalog):
|
|
444
|
+
self.catalog = catalog
|
|
445
|
+
self.properties = catalog.extra_fields
|
|
446
|
+
|
|
447
|
+
def __repr__(self) -> str:
|
|
448
|
+
return f"<CatalogStorageExtension Catalog id={self.catalog.id}>"
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
class CollectionStorageExtension(_SchemesMixin, StorageExtension[pystac.Collection]):
|
|
452
|
+
"""A concrete implementation of :class:`StorageExtension` on an
|
|
453
|
+
:class:`~pystac.Collection` that extends the properties of the Collection to include
|
|
454
|
+
properties defined in the :stac-ext:`Storage Extension <storage>`.
|
|
455
|
+
|
|
456
|
+
This class should generally not be instantiated directly. Instead, call
|
|
457
|
+
:meth:`StorageExtension.ext` on an :class:`~pystac.Collection` to extend it.
|
|
458
|
+
"""
|
|
459
|
+
|
|
460
|
+
collection: pystac.Collection
|
|
461
|
+
"""The :class:`~pystac.Collection` being extended."""
|
|
462
|
+
|
|
463
|
+
properties: dict[str, Any]
|
|
464
|
+
"""The :class:`~pystac.Collection` properties, including extension properties."""
|
|
465
|
+
|
|
466
|
+
def __init__(self, collection: pystac.Collection):
|
|
467
|
+
self.collection = collection
|
|
468
|
+
self.properties = collection.extra_fields
|
|
469
|
+
|
|
470
|
+
def __repr__(self) -> str:
|
|
471
|
+
return f"<CollectionStorageExtension Collection id={self.collection.id}>"
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
class AssetStorageExtension(_RefsMixin, StorageExtension[pystac.Asset]):
|
|
475
|
+
"""A concrete implementation of :class:`StorageExtension` on an
|
|
476
|
+
:class:`~pystac.Asset` that extends the properties of the Asset to include
|
|
477
|
+
properties defined in the :stac-ext:`Storage Extension <storage>`.
|
|
478
|
+
|
|
479
|
+
This class should generally not be instantiated directly. Instead, call
|
|
480
|
+
:meth:`StorageExtension.ext` on an :class:`~pystac.Asset` to extend it.
|
|
481
|
+
"""
|
|
482
|
+
|
|
483
|
+
asset: pystac.Asset
|
|
484
|
+
"""The :class:`~pystac.Asset` being extended."""
|
|
485
|
+
|
|
486
|
+
properties: dict[str, Any]
|
|
487
|
+
"""The :class:`~pystac.Asset` properties, including extension properties."""
|
|
488
|
+
|
|
489
|
+
def __init__(self, asset: pystac.Asset):
|
|
490
|
+
self.asset = asset
|
|
491
|
+
self.properties = asset.extra_fields
|
|
492
|
+
|
|
493
|
+
def __repr__(self) -> str:
|
|
494
|
+
return f"<AssetStorageExtension Asset href={self.asset.href}>"
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
class LinkStorageExtension(_RefsMixin, StorageExtension[pystac.Link]):
|
|
498
|
+
"""A concrete implementation of :class:`StorageExtension` on an
|
|
499
|
+
:class:`~pystac.Link` that extends the properties of the Link to include
|
|
500
|
+
properties defined in the :stac-ext:`Storage Extension <storage>`.
|
|
501
|
+
|
|
502
|
+
This class should generally not be instantiated directly. Instead, call
|
|
503
|
+
:meth:`StorageExtension.ext` on an :class:`~pystac.Link` to extend it.
|
|
504
|
+
"""
|
|
505
|
+
|
|
506
|
+
link: pystac.Link
|
|
507
|
+
"""The :class:`~pystac.Link` being extended."""
|
|
508
|
+
|
|
509
|
+
properties: dict[str, Any]
|
|
510
|
+
"""The :class:`~pystac.Link` properties, including extension properties."""
|
|
511
|
+
|
|
512
|
+
def __init__(self, link: pystac.Link):
|
|
513
|
+
self.link = link
|
|
514
|
+
self.properties = link.extra_fields
|
|
515
|
+
|
|
516
|
+
def __repr__(self) -> str:
|
|
517
|
+
return f"<LinkStorageExtension Link href={self.link.href}>"
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
class ItemAssetsStorageExtension(
|
|
521
|
+
_RefsMixin, StorageExtension[pystac.ItemAssetDefinition]
|
|
522
|
+
):
|
|
523
|
+
"""A concrete implementation of :class:`StorageExtension` on an
|
|
524
|
+
:class:`~pystac.ItemAssetDefinition` that extends the properties of the
|
|
525
|
+
ItemAssetDefinition to include properties defined in the
|
|
526
|
+
:stac-ext:`Storage Extension <storage>`.
|
|
527
|
+
|
|
528
|
+
This class should generally not be instantiated directly. Instead, call
|
|
529
|
+
:meth:`StorageExtension.ext` on an :class:`~pystac.ItemAssetDefinition`
|
|
530
|
+
to extend it.
|
|
531
|
+
"""
|
|
532
|
+
|
|
533
|
+
item_asset: pystac.ItemAssetDefinition
|
|
534
|
+
"""The :class:`~pystac.ItemAssetDefinition` being extended."""
|
|
535
|
+
|
|
536
|
+
properties: dict[str, Any]
|
|
537
|
+
"""The :class:`~pystac.ItemAssetDefinition` properties,
|
|
538
|
+
including extension properties."""
|
|
539
|
+
|
|
540
|
+
def __init__(self, item_asset: pystac.ItemAssetDefinition):
|
|
541
|
+
self.item_asset = item_asset
|
|
542
|
+
self.properties = item_asset.properties
|
|
543
|
+
|
|
544
|
+
def __repr__(self) -> str:
|
|
545
|
+
return f"<ItemAssetsStorageExtension ItemAssetDefinition={self.item_asset}>"
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
class SummariesStorageExtension(SummariesExtension):
|
|
549
|
+
"""A concrete implementation of :class:`~pystac.extensions.base.SummariesExtension`
|
|
550
|
+
that extends the ``summaries`` field of a :class:`~pystac.Collection` to include
|
|
551
|
+
properties defined in the :stac-ext:`Storage Extension <storage>`.
|
|
552
|
+
"""
|
|
553
|
+
|
|
554
|
+
@property
|
|
555
|
+
def schemes(self) -> list[dict[str, StorageScheme]] | None:
|
|
556
|
+
"""Get or sets the summary of :attr:`StorageScheme.platform` values
|
|
557
|
+
for this Collection.
|
|
558
|
+
"""
|
|
559
|
+
return map_opt(
|
|
560
|
+
lambda schemes: [
|
|
561
|
+
{k: StorageScheme(v) for k, v in x.items()} for x in schemes
|
|
562
|
+
],
|
|
563
|
+
self.summaries.get_list(SCHEMES_PROP),
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
@schemes.setter
|
|
567
|
+
def schemes(self, v: list[dict[str, StorageScheme]] | None) -> None:
|
|
568
|
+
self._set_summary(
|
|
569
|
+
SCHEMES_PROP,
|
|
570
|
+
map_opt(
|
|
571
|
+
lambda schemes: [
|
|
572
|
+
{k: c.to_dict() for k, c in x.items()} for x in schemes
|
|
573
|
+
],
|
|
574
|
+
v,
|
|
575
|
+
),
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
class StorageExtensionHooks(ExtensionHooks):
|
|
580
|
+
schema_uri: str = SCHEMA_URI_PATTERN.format(version=DEFAULT_VERSION)
|
|
581
|
+
prev_extension_ids = {
|
|
582
|
+
SCHEMA_URI_PATTERN.format(version=v)
|
|
583
|
+
for v in SUPPORTED_VERSIONS
|
|
584
|
+
if v != DEFAULT_VERSION
|
|
585
|
+
}
|
|
586
|
+
stac_object_types = {
|
|
587
|
+
pystac.STACObjectType.CATALOG,
|
|
588
|
+
pystac.STACObjectType.COLLECTION,
|
|
589
|
+
pystac.STACObjectType.ITEM,
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
# Mapping from v1.0.0 platform enum values to v2.0.0 type identifiers
|
|
593
|
+
# Only AWS and Azure have defined v2.0.0 platform definitions
|
|
594
|
+
_PLATFORM_TYPE_MAP: dict[str, str] = {
|
|
595
|
+
"AWS": "aws-s3",
|
|
596
|
+
"AZURE": "ms-azure",
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
# Mapping from v1.0.0 platform enum values to v2.0.0 platform URI templates
|
|
600
|
+
_PLATFORM_URI_MAP: dict[str, str] = {
|
|
601
|
+
"AWS": "https://{bucket}.s3.{region}.amazonaws.com",
|
|
602
|
+
"AZURE": "https://{account}.blob.core.windows.net",
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
# Mapping from v1.0.0 platform enum values to scheme key prefixes
|
|
606
|
+
_PLATFORM_KEY_PREFIX: dict[str, str] = {
|
|
607
|
+
"AWS": "aws",
|
|
608
|
+
"AZURE": "azure",
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
# Platforms that cannot be automatically migrated
|
|
612
|
+
_UNSUPPORTED_PLATFORMS: set[str] = {"GCP", "IBM", "ALIBABA", "ORACLE", "OTHER"}
|
|
613
|
+
|
|
614
|
+
# Regex patterns for parsing cloud storage URLs
|
|
615
|
+
_S3_URL_PATTERN = re.compile(r"^s3://([^/]+)/")
|
|
616
|
+
_AZURE_BLOB_PATTERN = re.compile(r"^https://([^.]+)\.blob\.core\.windows\.net/")
|
|
617
|
+
|
|
618
|
+
def migrate(
|
|
619
|
+
self, obj: dict[str, Any], version: STACVersionID, info: STACJSONDescription
|
|
620
|
+
) -> None:
|
|
621
|
+
if SCHEMA_URI_PATTERN.format(version="1.0.0") in info.extensions:
|
|
622
|
+
props = obj.get("properties", obj)
|
|
623
|
+
|
|
624
|
+
# v1 defined item level storage properties can
|
|
625
|
+
# be used across all assets
|
|
626
|
+
item_platform = props.get(PREFIX + "platform")
|
|
627
|
+
item_region = props.get(PREFIX + "region")
|
|
628
|
+
item_requester_pays = props.get(PREFIX + "requester_pays")
|
|
629
|
+
item_tier = props.get(PREFIX + "tier")
|
|
630
|
+
|
|
631
|
+
schemes: dict[str, dict[str, Any]] = {}
|
|
632
|
+
scheme_hash_to_key: dict[int, str] = {}
|
|
633
|
+
assets_with_tier: list[str] = []
|
|
634
|
+
assets_failed_parsing: list[str] = []
|
|
635
|
+
unsupported_platforms: set[str] = set()
|
|
636
|
+
migrated_assets: list[str] = []
|
|
637
|
+
|
|
638
|
+
for asset_key, asset in obj.get("assets", {}).items():
|
|
639
|
+
platform = asset.get(PREFIX + "platform", item_platform)
|
|
640
|
+
region = asset.get(PREFIX + "region", item_region)
|
|
641
|
+
requester_pays = asset.get(
|
|
642
|
+
PREFIX + "requester_pays", item_requester_pays
|
|
643
|
+
)
|
|
644
|
+
tier = asset.get(PREFIX + "tier", item_tier)
|
|
645
|
+
href = asset.get("href", "")
|
|
646
|
+
|
|
647
|
+
if tier is not None:
|
|
648
|
+
assets_with_tier.append(asset_key)
|
|
649
|
+
|
|
650
|
+
# cannot migrate assets without a platform
|
|
651
|
+
if platform is None:
|
|
652
|
+
continue
|
|
653
|
+
|
|
654
|
+
# cannot migrate assets with unsupported platforms
|
|
655
|
+
platform_upper = platform.upper()
|
|
656
|
+
if (
|
|
657
|
+
platform_upper in self._UNSUPPORTED_PLATFORMS
|
|
658
|
+
or platform_upper not in self._PLATFORM_TYPE_MAP
|
|
659
|
+
):
|
|
660
|
+
unsupported_platforms.add(platform_upper)
|
|
661
|
+
continue
|
|
662
|
+
|
|
663
|
+
scheme: dict[str, Any] = {
|
|
664
|
+
"type": self._PLATFORM_TYPE_MAP[platform_upper],
|
|
665
|
+
"platform": self._PLATFORM_URI_MAP[platform_upper],
|
|
666
|
+
}
|
|
667
|
+
if region is not None:
|
|
668
|
+
scheme["region"] = region
|
|
669
|
+
if requester_pays is not None:
|
|
670
|
+
scheme["requester_pays"] = requester_pays
|
|
671
|
+
|
|
672
|
+
# Parse bucket/account info from href
|
|
673
|
+
if platform_upper == "AWS":
|
|
674
|
+
if s3_match := self._S3_URL_PATTERN.match(href):
|
|
675
|
+
scheme["bucket"] = s3_match.group(1)
|
|
676
|
+
else:
|
|
677
|
+
assets_failed_parsing.append(asset_key)
|
|
678
|
+
continue
|
|
679
|
+
elif platform_upper == "AZURE":
|
|
680
|
+
if azure_match := self._AZURE_BLOB_PATTERN.match(href):
|
|
681
|
+
scheme["account"] = azure_match.group(1)
|
|
682
|
+
else:
|
|
683
|
+
assets_failed_parsing.append(asset_key)
|
|
684
|
+
continue
|
|
685
|
+
|
|
686
|
+
# Deduplicate schemes by content hash
|
|
687
|
+
scheme_hash = hash(frozenset(scheme.items()))
|
|
688
|
+
|
|
689
|
+
if scheme_hash in scheme_hash_to_key:
|
|
690
|
+
scheme_key = scheme_hash_to_key[scheme_hash]
|
|
691
|
+
else:
|
|
692
|
+
# Generate scheme key: provider-region or provider
|
|
693
|
+
# if key would collide, appends an int suffix
|
|
694
|
+
key_prefix = self._PLATFORM_KEY_PREFIX[platform_upper]
|
|
695
|
+
base_key = (
|
|
696
|
+
f"{key_prefix}-{region.lower()}" if region else key_prefix
|
|
697
|
+
)
|
|
698
|
+
|
|
699
|
+
scheme_key = base_key
|
|
700
|
+
counter = 1
|
|
701
|
+
|
|
702
|
+
while scheme_key in schemes:
|
|
703
|
+
scheme_key = f"{base_key}-{counter}"
|
|
704
|
+
counter += 1
|
|
705
|
+
|
|
706
|
+
schemes[scheme_key] = scheme
|
|
707
|
+
scheme_hash_to_key[scheme_hash] = scheme_key
|
|
708
|
+
|
|
709
|
+
asset.pop(PREFIX + "platform", None)
|
|
710
|
+
asset.pop(PREFIX + "region", None)
|
|
711
|
+
asset.pop(PREFIX + "requester_pays", None)
|
|
712
|
+
asset.pop(PREFIX + "tier", None)
|
|
713
|
+
asset[REFS_PROP] = [scheme_key]
|
|
714
|
+
migrated_assets.append(asset_key)
|
|
715
|
+
|
|
716
|
+
if assets_with_tier:
|
|
717
|
+
warnings.warn(
|
|
718
|
+
"storage:tier was removed in storage extension v2.0.0 and cannot "
|
|
719
|
+
f"be migrated. Property left in place for: {assets_with_tier}",
|
|
720
|
+
UserWarning,
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
if assets_failed_parsing:
|
|
724
|
+
warnings.warn(
|
|
725
|
+
"Could not parse bucket/account from href. "
|
|
726
|
+
f"The following assets were not migrated: {assets_failed_parsing}",
|
|
727
|
+
UserWarning,
|
|
728
|
+
)
|
|
729
|
+
|
|
730
|
+
if unsupported_platforms:
|
|
731
|
+
warnings.warn(
|
|
732
|
+
"The following platforms cannot be automatically migrated to "
|
|
733
|
+
f"storage extension v2.0.0: {unsupported_platforms}",
|
|
734
|
+
UserWarning,
|
|
735
|
+
)
|
|
736
|
+
|
|
737
|
+
# Only remove item-level properties if all assets were migrated
|
|
738
|
+
if (
|
|
739
|
+
migrated_assets
|
|
740
|
+
and not unsupported_platforms
|
|
741
|
+
and not assets_failed_parsing
|
|
742
|
+
):
|
|
743
|
+
props.pop(PREFIX + "platform", None)
|
|
744
|
+
props.pop(PREFIX + "region", None)
|
|
745
|
+
props.pop(PREFIX + "requester_pays", None)
|
|
746
|
+
|
|
747
|
+
if schemes:
|
|
748
|
+
props[SCHEMES_PROP] = schemes
|
|
749
|
+
|
|
750
|
+
super().migrate(obj, version, info)
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
STORAGE_EXTENSION_HOOKS: ExtensionHooks = StorageExtensionHooks()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pystac-ext-storage
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Storage extension for PySTAC
|
|
5
|
+
Project-URL: Documentation, https://pystac.readthedocs.io
|
|
6
|
+
Project-URL: Repository, https://github.com/stac-utils/pystac
|
|
7
|
+
Project-URL: Issues, https://github.com/stac-utils/pystac/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/stac-utils/pystac/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: Discussions, https://github.com/radiantearth/stac-spec/discussions/categories/stac-software
|
|
10
|
+
License: Apache-2.0
|
|
11
|
+
Keywords: STAC,catalog,imagery,pystac,raster,storage
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
|
+
Classifier: Natural Language :: English
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: pystac-core
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# pystac-ext-storage
|
|
26
|
+
|
|
27
|
+
[PySTAC](https://pypi.org/project/pystac/) extension package for the [Storage Extension](https://github.com/stac-extensions/storage).
|
|
28
|
+
This extension provides fields for describing cloud storage attributes of assets, including storage platform, region, tier, and requester-pays configuration.
|
|
29
|
+
|
|
30
|
+
## Supported versions
|
|
31
|
+
|
|
32
|
+
- [v2.0.0](https://stac-extensions.github.io/storage/v2.0.0/schema.json)
|
|
33
|
+
- [v1.0.0](https://stac-extensions.github.io/storage/v1.0.0/schema.json)
|
|
34
|
+
|
|
35
|
+
## Versioning
|
|
36
|
+
|
|
37
|
+
This package's version corresponds to the version of the extension specification it targets.
|
|
38
|
+
When we release updates to the package code without changing the target extension version, we use [post releases](https://packaging.python.org/en/latest/discussions/versioning/#post-releases), e.g. `2.0.0.post1`.
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
pystac/extensions/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
pystac/extensions/storage.py,sha256=g5oWwMrFnSYyPoRsZGQ3GTo3SyRzz7dnO4S7rMk1vCY,25518
|
|
3
|
+
pystac_ext_storage-2.0.0.dist-info/METADATA,sha256=UVh_AbUEeGylekkZzlLuqaT3HN0WjwSTg8tqTHAEWa4,1900
|
|
4
|
+
pystac_ext_storage-2.0.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
5
|
+
pystac_ext_storage-2.0.0.dist-info/RECORD,,
|