datacosmos 0.0.18__py3-none-any.whl → 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacosmos might be problematic. Click here for more details.
- datacosmos/auth/local_authenticator.py +4 -6
- datacosmos/auth/m2m_authenticator.py +3 -3
- datacosmos/datacosmos_client.py +11 -13
- datacosmos/exceptions/__init__.py +8 -0
- datacosmos/exceptions/{datacosmos_exception.py → datacosmos_error.py} +2 -2
- datacosmos/exceptions/stac_validation_error.py +8 -0
- datacosmos/stac/collection/collection_client.py +2 -2
- datacosmos/stac/item/item_client.py +76 -20
- datacosmos/stac/item/models/datacosmos_item.py +14 -6
- datacosmos/stac/storage/storage_base.py +54 -17
- datacosmos/stac/storage/uploader.py +57 -30
- datacosmos/utils/http_response/check_api_response.py +6 -6
- {datacosmos-0.0.18.dist-info → datacosmos-0.0.20.dist-info}/METADATA +1 -1
- {datacosmos-0.0.18.dist-info → datacosmos-0.0.20.dist-info}/RECORD +17 -16
- {datacosmos-0.0.18.dist-info → datacosmos-0.0.20.dist-info}/WHEEL +0 -0
- {datacosmos-0.0.18.dist-info → datacosmos-0.0.20.dist-info}/licenses/LICENSE.md +0 -0
- {datacosmos-0.0.18.dist-info → datacosmos-0.0.20.dist-info}/top_level.txt +0 -0
|
@@ -7,7 +7,7 @@ import requests
|
|
|
7
7
|
|
|
8
8
|
from datacosmos.auth.base_authenticator import AuthResult, BaseAuthenticator
|
|
9
9
|
from datacosmos.auth.local_token_fetcher import LocalTokenFetcher
|
|
10
|
-
from datacosmos.exceptions.
|
|
10
|
+
from datacosmos.exceptions.datacosmos_error import DatacosmosError
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
class LocalAuthenticator(BaseAuthenticator):
|
|
@@ -36,9 +36,7 @@ class LocalAuthenticator(BaseAuthenticator):
|
|
|
36
36
|
token_file=Path(auth.cache_file).expanduser(),
|
|
37
37
|
)
|
|
38
38
|
except Exception as e:
|
|
39
|
-
raise
|
|
40
|
-
f"Failed to initialize LocalTokenFetcher: {e}"
|
|
41
|
-
) from e
|
|
39
|
+
raise DatacosmosError(f"Failed to initialize LocalTokenFetcher: {e}") from e
|
|
42
40
|
|
|
43
41
|
def authenticate_and_build_session(self) -> AuthResult:
|
|
44
42
|
"""Builds an authenticated session using the local token fetcher."""
|
|
@@ -52,7 +50,7 @@ class LocalAuthenticator(BaseAuthenticator):
|
|
|
52
50
|
http_client=http_client, token=token, token_expiry=token_expiry
|
|
53
51
|
)
|
|
54
52
|
except Exception as e:
|
|
55
|
-
raise
|
|
53
|
+
raise DatacosmosError(f"Local authentication failed: {e}") from e
|
|
56
54
|
|
|
57
55
|
def refresh_token(self) -> AuthResult:
|
|
58
56
|
"""Refreshes the local token non-interactively."""
|
|
@@ -69,4 +67,4 @@ class LocalAuthenticator(BaseAuthenticator):
|
|
|
69
67
|
http_client=http_client, token=token, token_expiry=token_expiry
|
|
70
68
|
)
|
|
71
69
|
except Exception as e:
|
|
72
|
-
raise
|
|
70
|
+
raise DatacosmosError(f"Local token refresh failed: {e}") from e
|
|
@@ -13,7 +13,7 @@ from tenacity import (
|
|
|
13
13
|
)
|
|
14
14
|
|
|
15
15
|
from datacosmos.auth.base_authenticator import AuthResult, BaseAuthenticator
|
|
16
|
-
from datacosmos.exceptions.
|
|
16
|
+
from datacosmos.exceptions.datacosmos_error import DatacosmosError
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class M2MAuthenticator(BaseAuthenticator):
|
|
@@ -52,9 +52,9 @@ class M2MAuthenticator(BaseAuthenticator):
|
|
|
52
52
|
http_client=http_client, token=token, token_expiry=token_expiry
|
|
53
53
|
)
|
|
54
54
|
except (HTTPError, ConnectionError, Timeout) as e:
|
|
55
|
-
raise
|
|
55
|
+
raise DatacosmosError(f"M2M authentication failed: {e}") from e
|
|
56
56
|
except RequestException as e:
|
|
57
|
-
raise
|
|
57
|
+
raise DatacosmosError(
|
|
58
58
|
f"Unexpected request failure during M2M authentication: {e}"
|
|
59
59
|
) from e
|
|
60
60
|
|
datacosmos/datacosmos_client.py
CHANGED
|
@@ -20,7 +20,7 @@ from datacosmos.auth.base_authenticator import BaseAuthenticator
|
|
|
20
20
|
from datacosmos.auth.local_authenticator import LocalAuthenticator
|
|
21
21
|
from datacosmos.auth.m2m_authenticator import M2MAuthenticator
|
|
22
22
|
from datacosmos.config.config import Config
|
|
23
|
-
from datacosmos.exceptions.
|
|
23
|
+
from datacosmos.exceptions.datacosmos_error import DatacosmosError
|
|
24
24
|
|
|
25
25
|
_log = logging.getLogger(__name__)
|
|
26
26
|
|
|
@@ -75,9 +75,7 @@ class DatacosmosClient:
|
|
|
75
75
|
try:
|
|
76
76
|
return Config.model_validate(cfg)
|
|
77
77
|
except Exception as e:
|
|
78
|
-
raise
|
|
79
|
-
"Invalid config provided to DatacosmosClient"
|
|
80
|
-
) from e
|
|
78
|
+
raise DatacosmosError("Invalid config provided to DatacosmosClient") from e
|
|
81
79
|
|
|
82
80
|
def _init_with_injected_session(
|
|
83
81
|
self, http_session: requests.Session | OAuth2Session
|
|
@@ -88,7 +86,7 @@ class DatacosmosClient:
|
|
|
88
86
|
token_data = self._extract_token_data(http_session)
|
|
89
87
|
self.token = token_data.get("access_token")
|
|
90
88
|
if not self.token:
|
|
91
|
-
raise
|
|
89
|
+
raise DatacosmosError(
|
|
92
90
|
"Failed to extract access token from injected session"
|
|
93
91
|
)
|
|
94
92
|
self.token_expiry = self._compute_expiry(
|
|
@@ -103,11 +101,11 @@ class DatacosmosClient:
|
|
|
103
101
|
if isinstance(http_session, requests.Session):
|
|
104
102
|
auth_header = http_session.headers.get("Authorization", "")
|
|
105
103
|
if not auth_header.startswith("Bearer "):
|
|
106
|
-
raise
|
|
104
|
+
raise DatacosmosError(
|
|
107
105
|
"Injected requests.Session must include a 'Bearer' token in its headers"
|
|
108
106
|
)
|
|
109
107
|
return {"access_token": auth_header.split(" ", 1)[1]}
|
|
110
|
-
raise
|
|
108
|
+
raise DatacosmosError(f"Unsupported session type: {type(http_session)}")
|
|
111
109
|
|
|
112
110
|
def _compute_expiry(
|
|
113
111
|
self,
|
|
@@ -134,7 +132,7 @@ class DatacosmosClient:
|
|
|
134
132
|
elif auth_type == "local":
|
|
135
133
|
self._authenticator = LocalAuthenticator(self.config)
|
|
136
134
|
else:
|
|
137
|
-
raise
|
|
135
|
+
raise DatacosmosError(f"Unsupported authentication type: {auth_type}")
|
|
138
136
|
|
|
139
137
|
auth_result = self._authenticator.authenticate_and_build_session()
|
|
140
138
|
self.token = auth_result.token
|
|
@@ -166,7 +164,7 @@ class DatacosmosClient:
|
|
|
166
164
|
{"Authorization": f"Bearer {self.token}"}
|
|
167
165
|
)
|
|
168
166
|
else:
|
|
169
|
-
raise
|
|
167
|
+
raise DatacosmosError(
|
|
170
168
|
"Cannot refresh token, no authenticator initialized."
|
|
171
169
|
)
|
|
172
170
|
|
|
@@ -196,7 +194,7 @@ class DatacosmosClient:
|
|
|
196
194
|
requests.Response: The HTTP response.
|
|
197
195
|
|
|
198
196
|
Raises:
|
|
199
|
-
|
|
197
|
+
DatacosmosError: For any HTTP or request-related errors.
|
|
200
198
|
"""
|
|
201
199
|
self._refresh_token_if_needed()
|
|
202
200
|
|
|
@@ -228,16 +226,16 @@ class DatacosmosClient:
|
|
|
228
226
|
retry_response.raise_for_status()
|
|
229
227
|
return retry_response
|
|
230
228
|
except HTTPError as e:
|
|
231
|
-
raise
|
|
229
|
+
raise DatacosmosError(
|
|
232
230
|
f"HTTP error during {method.upper()} request to {url} after refresh",
|
|
233
231
|
response=e.response,
|
|
234
232
|
) from e
|
|
235
|
-
raise
|
|
233
|
+
raise DatacosmosError(
|
|
236
234
|
f"HTTP error during {method.upper()} request to {url}",
|
|
237
235
|
response=getattr(e, "response", None),
|
|
238
236
|
) from e
|
|
239
237
|
except RequestException as e:
|
|
240
|
-
raise
|
|
238
|
+
raise DatacosmosError(
|
|
241
239
|
f"Unexpected request failure during {method.upper()} request to {url}: {e}"
|
|
242
240
|
) from e
|
|
243
241
|
|
|
@@ -6,11 +6,11 @@ from requests import Response
|
|
|
6
6
|
from requests.exceptions import RequestException
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
class
|
|
9
|
+
class DatacosmosError(RequestException):
|
|
10
10
|
"""Base exception class for all Datacosmos SDK exceptions."""
|
|
11
11
|
|
|
12
12
|
def __init__(self, message: str, response: Optional[Response] = None):
|
|
13
|
-
"""Initialize
|
|
13
|
+
"""Initialize DatacosmosError.
|
|
14
14
|
|
|
15
15
|
Args:
|
|
16
16
|
message (str): The error message.
|
|
@@ -6,7 +6,7 @@ from pystac import Collection, Extent, SpatialExtent, TemporalExtent
|
|
|
6
6
|
from pystac.utils import str_to_datetime
|
|
7
7
|
|
|
8
8
|
from datacosmos.datacosmos_client import DatacosmosClient
|
|
9
|
-
from datacosmos.exceptions.
|
|
9
|
+
from datacosmos.exceptions.datacosmos_error import DatacosmosError
|
|
10
10
|
from datacosmos.stac.collection.models.collection_update import CollectionUpdate
|
|
11
11
|
from datacosmos.utils.http_response.check_api_response import check_api_response
|
|
12
12
|
|
|
@@ -147,7 +147,7 @@ class CollectionClient:
|
|
|
147
147
|
try:
|
|
148
148
|
return next_href.split("?")[1].split("=")[-1]
|
|
149
149
|
except (IndexError, AttributeError) as e:
|
|
150
|
-
raise
|
|
150
|
+
raise DatacosmosError(
|
|
151
151
|
f"Failed to parse pagination token from {next_href}",
|
|
152
152
|
response=e.response,
|
|
153
153
|
) from e
|
|
@@ -8,7 +8,7 @@ from typing import Generator, Optional
|
|
|
8
8
|
from pystac import Item
|
|
9
9
|
|
|
10
10
|
from datacosmos.datacosmos_client import DatacosmosClient
|
|
11
|
-
from datacosmos.exceptions
|
|
11
|
+
from datacosmos.exceptions import DatacosmosError, StacValidationError
|
|
12
12
|
from datacosmos.stac.item.models.catalog_search_parameters import (
|
|
13
13
|
CatalogSearchParameters,
|
|
14
14
|
)
|
|
@@ -72,17 +72,10 @@ class ItemClient:
|
|
|
72
72
|
|
|
73
73
|
Raises:
|
|
74
74
|
ValueError: If the item has no collection set.
|
|
75
|
+
StacValidationError: If the collection ID in the links doesn't match the item's collection field.
|
|
75
76
|
RequestError: If the API returns an error response.
|
|
76
77
|
"""
|
|
77
|
-
|
|
78
|
-
collection_id = item.collection_id or (
|
|
79
|
-
item.get_collection().id if item.get_collection() else None
|
|
80
|
-
)
|
|
81
|
-
else:
|
|
82
|
-
collection_id = item.collection
|
|
83
|
-
|
|
84
|
-
if not collection_id:
|
|
85
|
-
raise ValueError("Cannot create item: no collection_id found on item")
|
|
78
|
+
collection_id = self._get_validated_collection_id(item, method="create")
|
|
86
79
|
|
|
87
80
|
url = self.base_url.with_suffix(f"/collections/{collection_id}/items")
|
|
88
81
|
item_json: dict = item.to_dict()
|
|
@@ -99,17 +92,13 @@ class ItemClient:
|
|
|
99
92
|
|
|
100
93
|
Raises:
|
|
101
94
|
ValueError: If the item has no collection set.
|
|
95
|
+
StacValidationError: If the collection ID in the links doesn't match the item's collection field.
|
|
102
96
|
RequestError: If the API returns an error response.
|
|
103
97
|
"""
|
|
104
|
-
|
|
105
|
-
collection_id = item.collection_id or (
|
|
106
|
-
item.get_collection().id if item.get_collection() else None
|
|
107
|
-
)
|
|
108
|
-
else:
|
|
109
|
-
collection_id = item.collection
|
|
98
|
+
collection_id = self._get_validated_collection_id(item, method="add")
|
|
110
99
|
|
|
111
|
-
if not
|
|
112
|
-
raise ValueError("Cannot
|
|
100
|
+
if not item.id:
|
|
101
|
+
raise ValueError("Cannot add item: no item_id found on item")
|
|
113
102
|
|
|
114
103
|
url = self.base_url.with_suffix(f"/collections/{collection_id}/items/{item.id}")
|
|
115
104
|
item_json: dict = item.to_dict()
|
|
@@ -203,12 +192,79 @@ class ItemClient:
|
|
|
203
192
|
Optional[str]: The extracted token, or None if parsing fails.
|
|
204
193
|
|
|
205
194
|
Raises:
|
|
206
|
-
|
|
195
|
+
DatacosmosError: If pagination token extraction fails.
|
|
207
196
|
"""
|
|
208
197
|
try:
|
|
209
198
|
return next_href.split("?")[1].split("=")[-1]
|
|
210
199
|
except (IndexError, AttributeError) as e:
|
|
211
|
-
raise
|
|
200
|
+
raise DatacosmosError(
|
|
212
201
|
f"Failed to parse pagination token from {next_href}",
|
|
213
202
|
response=e.response,
|
|
214
203
|
) from e
|
|
204
|
+
|
|
205
|
+
def _get_validated_collection_id(
|
|
206
|
+
self, item: Item | DatacosmosItem, method: str
|
|
207
|
+
) -> str:
|
|
208
|
+
"""Resolves and validates the collection ID from an item, checking for link consistency.
|
|
209
|
+
|
|
210
|
+
Args:
|
|
211
|
+
item: The STAC item.
|
|
212
|
+
method: The client method calling this helper ("create" or "add").
|
|
213
|
+
|
|
214
|
+
Returns:
|
|
215
|
+
The validated collection_id.
|
|
216
|
+
|
|
217
|
+
Raises:
|
|
218
|
+
ValueError: If collection ID cannot be resolved.
|
|
219
|
+
StacValidationError: If the collection ID and parent link are inconsistent.
|
|
220
|
+
"""
|
|
221
|
+
if isinstance(item, Item):
|
|
222
|
+
collection_id = item.collection_id or (
|
|
223
|
+
item.get_collection().id if item.get_collection() else None
|
|
224
|
+
)
|
|
225
|
+
if collection_id and not self._is_collection_link_consistent_pystac(
|
|
226
|
+
item, collection_id
|
|
227
|
+
):
|
|
228
|
+
raise StacValidationError(
|
|
229
|
+
"Parent link in pystac.Item does not match its collection_id."
|
|
230
|
+
)
|
|
231
|
+
else:
|
|
232
|
+
collection_id = item.collection
|
|
233
|
+
if collection_id and not self._is_collection_link_consistent_datacosmos(
|
|
234
|
+
item
|
|
235
|
+
):
|
|
236
|
+
raise StacValidationError(
|
|
237
|
+
"Parent link in DatacosmosItem does not match its collection."
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
if not collection_id:
|
|
241
|
+
if method == "create":
|
|
242
|
+
raise ValueError("Cannot create item: no collection_id found on item")
|
|
243
|
+
else:
|
|
244
|
+
raise ValueError("Cannot add item: no collection_id found on item")
|
|
245
|
+
|
|
246
|
+
return collection_id
|
|
247
|
+
|
|
248
|
+
def _is_collection_link_consistent_pystac(
|
|
249
|
+
self, item: Item, collection_id: str
|
|
250
|
+
) -> bool:
|
|
251
|
+
"""Helper to check if the parent link matches the pystac item's collection_id."""
|
|
252
|
+
parent_link = next(
|
|
253
|
+
(link for link in item.get_links("parent") if link.rel == "parent"), None
|
|
254
|
+
)
|
|
255
|
+
if not parent_link:
|
|
256
|
+
return True
|
|
257
|
+
|
|
258
|
+
link_collection_id = parent_link.get_href().rstrip("/").split("/")[-1]
|
|
259
|
+
return link_collection_id == collection_id
|
|
260
|
+
|
|
261
|
+
def _is_collection_link_consistent_datacosmos(self, item: DatacosmosItem) -> bool:
|
|
262
|
+
"""Helper to check if the parent link matches the datacosmos item's collection field."""
|
|
263
|
+
if not item.collection:
|
|
264
|
+
return True
|
|
265
|
+
|
|
266
|
+
for link in item.links:
|
|
267
|
+
if link.get("rel") == "parent":
|
|
268
|
+
link_collection_id = link.get("href", "").rstrip("/").split("/")[-1]
|
|
269
|
+
return link_collection_id == item.collection
|
|
270
|
+
return True
|
|
@@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, field_validator, model_validator
|
|
|
8
8
|
from shapely.errors import ShapelyError
|
|
9
9
|
from shapely.geometry import Polygon, shape
|
|
10
10
|
|
|
11
|
-
from datacosmos.exceptions.
|
|
11
|
+
from datacosmos.exceptions.stac_validation_error import StacValidationError
|
|
12
12
|
from datacosmos.stac.enums.processing_level import ProcessingLevel
|
|
13
13
|
from datacosmos.stac.item.models.asset import Asset
|
|
14
14
|
|
|
@@ -48,7 +48,7 @@ class DatacosmosItem(BaseModel):
|
|
|
48
48
|
]
|
|
49
49
|
|
|
50
50
|
if missing_keys:
|
|
51
|
-
raise
|
|
51
|
+
raise StacValidationError(
|
|
52
52
|
f"Datacosmos-specific properties are missing: {', '.join(missing_keys)}."
|
|
53
53
|
)
|
|
54
54
|
return properties_data
|
|
@@ -62,7 +62,7 @@ class DatacosmosItem(BaseModel):
|
|
|
62
62
|
if geometry_data.get("type") != "Polygon" or not geometry_data.get(
|
|
63
63
|
"coordinates"
|
|
64
64
|
):
|
|
65
|
-
raise
|
|
65
|
+
raise StacValidationError("Geometry must be a Polygon with coordinates.")
|
|
66
66
|
|
|
67
67
|
try:
|
|
68
68
|
# Use shape() for robust GeoJSON parsing and validation
|
|
@@ -80,7 +80,7 @@ class DatacosmosItem(BaseModel):
|
|
|
80
80
|
)
|
|
81
81
|
|
|
82
82
|
except (KeyError, ShapelyError, ValueError) as e:
|
|
83
|
-
raise
|
|
83
|
+
raise StacValidationError(f"Invalid geometry data: {e}") from e
|
|
84
84
|
|
|
85
85
|
return geometry_data
|
|
86
86
|
|
|
@@ -97,12 +97,12 @@ class DatacosmosItem(BaseModel):
|
|
|
97
97
|
math.isclose(a, b, rel_tol=1e-9)
|
|
98
98
|
for a, b in zip(self.bbox, true_bbox)
|
|
99
99
|
):
|
|
100
|
-
raise
|
|
100
|
+
raise StacValidationError(
|
|
101
101
|
"Provided bbox does not match geometry bounds."
|
|
102
102
|
)
|
|
103
103
|
except Exception as e:
|
|
104
104
|
# Catch any errors from Shapely or the comparison
|
|
105
|
-
raise
|
|
105
|
+
raise StacValidationError(f"Invalid bbox or geometry: {e}") from e
|
|
106
106
|
return self
|
|
107
107
|
|
|
108
108
|
def get_property(self, key: str) -> Any | None:
|
|
@@ -137,3 +137,11 @@ class DatacosmosItem(BaseModel):
|
|
|
137
137
|
def to_dict(self) -> dict:
|
|
138
138
|
"""Converts the DatacosmosItem instance to a dictionary."""
|
|
139
139
|
return self.model_dump()
|
|
140
|
+
|
|
141
|
+
def has_self_link(self) -> bool:
|
|
142
|
+
"""Checks if the item has a 'self' link."""
|
|
143
|
+
return any(link.get("rel") == "self" for link in self.links)
|
|
144
|
+
|
|
145
|
+
def has_parent_link(self) -> bool:
|
|
146
|
+
"""Checks if the item has a 'parent' link."""
|
|
147
|
+
return any(link.get("rel") == "parent" for link in self.links)
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
"""Base class providing common storage helpers (threading, MIME guess, futures)."""
|
|
2
2
|
|
|
3
3
|
import mimetypes
|
|
4
|
-
from concurrent.futures import ThreadPoolExecutor, wait
|
|
4
|
+
from concurrent.futures import Future, ThreadPoolExecutor, wait
|
|
5
|
+
from typing import Any, Callable, Dict, Iterable, List, Tuple
|
|
5
6
|
|
|
6
7
|
from datacosmos.datacosmos_client import DatacosmosClient
|
|
8
|
+
from datacosmos.exceptions.datacosmos_error import DatacosmosError
|
|
7
9
|
|
|
8
10
|
|
|
9
11
|
class StorageBase:
|
|
@@ -18,23 +20,58 @@ class StorageBase:
|
|
|
18
20
|
mime, _ = mimetypes.guess_type(src)
|
|
19
21
|
return mime or "application/octet-stream"
|
|
20
22
|
|
|
21
|
-
def
|
|
23
|
+
def run_in_threads(
|
|
24
|
+
self,
|
|
25
|
+
fn: Callable[..., Any],
|
|
26
|
+
jobs: Iterable[Tuple[Any, ...]],
|
|
27
|
+
max_workers: int,
|
|
28
|
+
timeout: float,
|
|
29
|
+
) -> Tuple[List[Any], List[Dict[str, Any]]]:
|
|
22
30
|
"""Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
|
|
23
31
|
|
|
24
|
-
|
|
32
|
+
Collects successes and failures without aborting the batch on individual errors.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
fn: The function to execute.
|
|
36
|
+
jobs: An iterable of tuples, where each tuple is unpacked as fn(*args).
|
|
37
|
+
max_workers: Maximum number of threads to use.
|
|
38
|
+
timeout: Timeout for the entire batch.
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
A tuple containing (successes: List[Any], failures: List[Dict[str, Any]]).
|
|
42
|
+
Failures include the exception and job arguments.
|
|
43
|
+
|
|
44
|
+
Raises:
|
|
45
|
+
DatacosmosError: If the entire batch times out.
|
|
25
46
|
"""
|
|
26
|
-
futures = []
|
|
27
|
-
|
|
28
|
-
|
|
47
|
+
futures: List[Future] = []
|
|
48
|
+
|
|
49
|
+
executor = ThreadPoolExecutor(max_workers=max_workers)
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
for args in jobs:
|
|
29
53
|
futures.append(executor.submit(fn, *args))
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
54
|
+
|
|
55
|
+
# Wait until all futures are done or the timeout is reached
|
|
56
|
+
done, not_done = wait(futures, timeout=timeout)
|
|
57
|
+
|
|
58
|
+
successes = []
|
|
59
|
+
failures = []
|
|
60
|
+
|
|
61
|
+
for future in done:
|
|
62
|
+
try:
|
|
63
|
+
result = future.result()
|
|
64
|
+
except Exception as e:
|
|
65
|
+
failures.append({'error': str(e), 'exception': e})
|
|
66
|
+
else:
|
|
67
|
+
successes.append(result)
|
|
68
|
+
|
|
69
|
+
if not_done:
|
|
70
|
+
# The executor's shutdown wait must be skipped to allow cancellation
|
|
71
|
+
raise DatacosmosError("Batch processing failed: operation timed out.")
|
|
72
|
+
|
|
73
|
+
return successes, failures
|
|
74
|
+
finally:
|
|
75
|
+
# Shutdown without waiting to enable timeout handling
|
|
76
|
+
# The wait call already established which jobs finished
|
|
77
|
+
executor.shutdown(wait=False)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Handles uploading files to Datacosmos storage and registering STAC items."""
|
|
2
2
|
|
|
3
3
|
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
4
5
|
|
|
5
6
|
from pydantic import TypeAdapter
|
|
6
7
|
|
|
@@ -13,7 +14,7 @@ from datacosmos.stac.storage.storage_base import StorageBase
|
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
class Uploader(StorageBase):
|
|
16
|
-
"""Upload a STAC item and its assets to Datacosmos storage
|
|
17
|
+
"""Upload a STAC item and its assets to Datacosmos storage and register the item in the STAC API."""
|
|
17
18
|
|
|
18
19
|
def __init__(self, client: DatacosmosClient):
|
|
19
20
|
"""Initialize the uploader.
|
|
@@ -32,17 +33,25 @@ class Uploader(StorageBase):
|
|
|
32
33
|
included_assets: list[str] | bool = True,
|
|
33
34
|
max_workers: int = 4,
|
|
34
35
|
time_out: float = 60 * 60 * 1,
|
|
35
|
-
) -> DatacosmosItem:
|
|
36
|
-
"""Upload a STAC item (and optionally its assets) to Datacosmos.
|
|
36
|
+
) -> tuple[DatacosmosItem, list[str], list[dict[str, Any]]]:
|
|
37
|
+
"""Upload a STAC item (and optionally its assets) to Datacosmos in parallel threads.
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
39
|
+
Args:
|
|
40
|
+
item (DatacosmosItem | str):
|
|
41
|
+
- a DatacosmosItem instance, or
|
|
42
|
+
- the path to an item JSON file on disk.
|
|
43
|
+
project_id (str): The project ID to upload assets to.
|
|
44
|
+
assets_path (str | None): Base directory where local asset files are located.
|
|
45
|
+
included_assets (list[str] | bool):
|
|
46
|
+
- True → upload every asset in the item.
|
|
47
|
+
- list[str] → upload only the asset keys in that list.
|
|
48
|
+
- False → skip asset upload; just register the item.
|
|
49
|
+
max_workers (int): Maximum number of parallel threads for asset upload.
|
|
50
|
+
time_out (float): Timeout in seconds for the entire asset batch upload.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
tuple[DatacosmosItem, list[str], list[dict[str, Any]]]:
|
|
54
|
+
The updated DatacosmosItem, a list of asset keys that were uploaded successfully, and a list of upload failures.
|
|
46
55
|
"""
|
|
47
56
|
if not assets_path and not isinstance(item, str):
|
|
48
57
|
raise ValueError(
|
|
@@ -54,23 +63,45 @@ class Uploader(StorageBase):
|
|
|
54
63
|
item = self._load_item(item_filename)
|
|
55
64
|
assets_path = assets_path or str(Path(item_filename).parent)
|
|
56
65
|
|
|
66
|
+
if not isinstance(item, DatacosmosItem):
|
|
67
|
+
raise TypeError(f"item must be a DatacosmosItem, got {type(item).__name__}")
|
|
68
|
+
|
|
57
69
|
assets_path = assets_path or str(Path.cwd())
|
|
58
70
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
71
|
+
if included_assets is False:
|
|
72
|
+
upload_assets: list[str] = []
|
|
73
|
+
elif included_assets is True:
|
|
74
|
+
upload_assets = list(item.assets.keys())
|
|
75
|
+
elif isinstance(included_assets, list):
|
|
76
|
+
upload_assets = included_assets
|
|
77
|
+
else:
|
|
78
|
+
upload_assets = []
|
|
66
79
|
|
|
67
80
|
jobs = [
|
|
68
81
|
(item, asset_key, assets_path, project_id) for asset_key in upload_assets
|
|
69
82
|
]
|
|
70
|
-
self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
|
|
71
83
|
|
|
72
|
-
|
|
73
|
-
|
|
84
|
+
if not jobs:
|
|
85
|
+
self.item_client.add_item(item)
|
|
86
|
+
return item, [], []
|
|
87
|
+
|
|
88
|
+
successes, failures = self.run_in_threads(
|
|
89
|
+
self._upload_asset, jobs, max_workers, time_out
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# Register the item if the overall process didn't time out
|
|
93
|
+
# and there was at least one successful upload.
|
|
94
|
+
if successes:
|
|
95
|
+
self.item_client.add_item(item)
|
|
96
|
+
|
|
97
|
+
return item, successes, failures
|
|
98
|
+
|
|
99
|
+
@staticmethod
|
|
100
|
+
def _load_item(item_json_file_path: str) -> DatacosmosItem:
|
|
101
|
+
"""Load a DatacosmosItem from a JSON file on disk."""
|
|
102
|
+
with open(item_json_file_path, "rb") as file:
|
|
103
|
+
data = file.read().decode("utf-8")
|
|
104
|
+
return TypeAdapter(DatacosmosItem).validate_json(data)
|
|
74
105
|
|
|
75
106
|
def upload_from_file(
|
|
76
107
|
self, src: str, dst: str, mime_type: str | None = None
|
|
@@ -83,19 +114,13 @@ class Uploader(StorageBase):
|
|
|
83
114
|
response = self.client.put(url, data=f, headers=headers)
|
|
84
115
|
response.raise_for_status()
|
|
85
116
|
|
|
86
|
-
@staticmethod
|
|
87
|
-
def _load_item(item_json_file_path: str) -> DatacosmosItem:
|
|
88
|
-
"""Load a DatacosmosItem from a JSON file on disk."""
|
|
89
|
-
with open(item_json_file_path, "rb") as file:
|
|
90
|
-
data = file.read().decode("utf-8")
|
|
91
|
-
return TypeAdapter(DatacosmosItem).validate_json(data)
|
|
92
|
-
|
|
93
117
|
def _upload_asset(
|
|
94
118
|
self, item: DatacosmosItem, asset_key: str, assets_path: str, project_id: str
|
|
95
|
-
) ->
|
|
119
|
+
) -> str:
|
|
96
120
|
"""Upload a single asset file and update its href inside the item object.
|
|
97
121
|
|
|
98
|
-
|
|
122
|
+
Returns:
|
|
123
|
+
str: The asset_key upon successful upload.
|
|
99
124
|
"""
|
|
100
125
|
asset = item.assets[asset_key]
|
|
101
126
|
|
|
@@ -117,6 +142,8 @@ class Uploader(StorageBase):
|
|
|
117
142
|
self._update_asset_href(asset) # turn href into public URL
|
|
118
143
|
self.upload_from_file(src, str(upload_path), mime_type=asset.type)
|
|
119
144
|
|
|
145
|
+
return asset_key
|
|
146
|
+
|
|
120
147
|
def _update_asset_href(self, asset: Asset) -> None:
|
|
121
148
|
"""Convert the storage key to a public HTTPS URL."""
|
|
122
149
|
try:
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
"""Validates an API response and raises a
|
|
1
|
+
"""Validates an API response and raises a DatacosmosError if an error occurs."""
|
|
2
2
|
|
|
3
3
|
from pydantic import ValidationError
|
|
4
4
|
from requests import Response
|
|
5
5
|
|
|
6
|
-
from datacosmos.exceptions.
|
|
6
|
+
from datacosmos.exceptions.datacosmos_error import DatacosmosError
|
|
7
7
|
from datacosmos.utils.http_response.models.datacosmos_response import DatacosmosResponse
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def check_api_response(response: Response) -> None:
|
|
11
|
-
"""Validates an API response and raises a
|
|
11
|
+
"""Validates an API response and raises a DatacosmosError if an error occurs.
|
|
12
12
|
|
|
13
13
|
Args:
|
|
14
14
|
resp (requests.Response): The response object.
|
|
15
15
|
|
|
16
16
|
Raises:
|
|
17
|
-
|
|
17
|
+
DatacosmosError: If the response status code indicates an error.
|
|
18
18
|
"""
|
|
19
19
|
if 200 <= response.status_code < 400:
|
|
20
20
|
return
|
|
@@ -26,9 +26,9 @@ def check_api_response(response: Response) -> None:
|
|
|
26
26
|
msg = "\n * " + "\n * ".join(
|
|
27
27
|
error.human_readable() for error in response.errors
|
|
28
28
|
)
|
|
29
|
-
raise
|
|
29
|
+
raise DatacosmosError(msg, response=response)
|
|
30
30
|
|
|
31
31
|
except ValidationError:
|
|
32
|
-
raise
|
|
32
|
+
raise DatacosmosError(
|
|
33
33
|
f"HTTP {response.status_code}: {response.text}", response=response
|
|
34
34
|
)
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
datacosmos/__init__.py,sha256=dVHKpbz5FVtfoJAWHRdsUENG6H-vs4UrkuwnIvOGJr4,66
|
|
2
|
-
datacosmos/datacosmos_client.py,sha256=
|
|
2
|
+
datacosmos/datacosmos_client.py,sha256=hydaepbjoEHupRv1CUNUOUMF9t_uZNR4Refwzvzfl0s,10446
|
|
3
3
|
datacosmos/auth/__init__.py,sha256=ynCThS9QyLKV9miRdnjm8uF_breiGGiCcI0FaOSw_2o,45
|
|
4
4
|
datacosmos/auth/base_authenticator.py,sha256=bSlb-N-vIUTl4K9KnDd3Dz21MevN_nvpWuwxgifdWBE,1814
|
|
5
|
-
datacosmos/auth/local_authenticator.py,sha256=
|
|
5
|
+
datacosmos/auth/local_authenticator.py,sha256=ah2DWxHjPYP-aTqGR-kEMMPv1Geeh00weHCGpZWgSig,2864
|
|
6
6
|
datacosmos/auth/local_token_fetcher.py,sha256=E4MI2lTRHAmxIQA7qY6hmpAUZETTzXNO8MViBaXnxGs,5268
|
|
7
|
-
datacosmos/auth/m2m_authenticator.py,sha256=
|
|
7
|
+
datacosmos/auth/m2m_authenticator.py,sha256=_n5qpyK9ngqHec8uMDBOKL-U3k989geIQ-rzH9PMhZc,2621
|
|
8
8
|
datacosmos/auth/token.py,sha256=neSV9gnnFa-rxEwMAJlZe_cReV6g4PQf8mq4-1mZzB8,2558
|
|
9
9
|
datacosmos/config/__init__.py,sha256=KCsaTb9-ZgFui1GM8wZFIPLJy0D0O8l8Z1Sv3NRD9UM,140
|
|
10
10
|
datacosmos/config/config.py,sha256=JHWmS6Q_T32iMly7cvJncjPCvtIgamBigKJJcvjGH7g,3465
|
|
@@ -18,12 +18,13 @@ datacosmos/config/models/local_user_account_authentication_config.py,sha256=SJZR
|
|
|
18
18
|
datacosmos/config/models/m2m_authentication_config.py,sha256=4l3Mmgips73rYGX5l7FCoHAWpWSGQYYkzZYvQzbmRz0,782
|
|
19
19
|
datacosmos/config/models/no_authentication_config.py,sha256=x5xikSGPuqQbrf_S2oIWXo5XxAORci2sSE5KyJvZHVw,312
|
|
20
20
|
datacosmos/config/models/url.py,sha256=bBeulXQ2c-tLJyIoo3sTi9SPsZIyIDn_D2zmkCGWp9s,1597
|
|
21
|
-
datacosmos/exceptions/__init__.py,sha256=
|
|
22
|
-
datacosmos/exceptions/
|
|
21
|
+
datacosmos/exceptions/__init__.py,sha256=2nS68zE1_AysfbH2S7QtX2pJ984hPLNjSqI_qzod2rc,212
|
|
22
|
+
datacosmos/exceptions/datacosmos_error.py,sha256=6Y9Hzo__aYbjTIpgTaD_Vh6Z271lbxyhoed859LpCY4,915
|
|
23
|
+
datacosmos/exceptions/stac_validation_error.py,sha256=4wmLcJwRraZ6S0JPYxL7jDs8X8k68KmNbcVTl_z1hkI,237
|
|
23
24
|
datacosmos/stac/__init__.py,sha256=B4x_Mr4X7TzQoYtRC-VzI4W-fEON5WUOaz8cWJbk3Fc,214
|
|
24
25
|
datacosmos/stac/stac_client.py,sha256=S0HESbZhlIdS0x_VSCeOSuOFaB50U4CMnTOX_0zLjn8,730
|
|
25
26
|
datacosmos/stac/collection/__init__.py,sha256=VQMLnsU3sER5kh4YxHrHP7XCA3DG1y0n9yoSmvycOY0,212
|
|
26
|
-
datacosmos/stac/collection/collection_client.py,sha256
|
|
27
|
+
datacosmos/stac/collection/collection_client.py,sha256=2ritToTSAC2n1nJgd7HpEDNvzda_qc-zsUUV39rbCRs,6052
|
|
27
28
|
datacosmos/stac/collection/models/__init__.py,sha256=TQaihUS_CM9Eaekm4SbzFTNfv7BmabHv3Z-f37Py5Qs,40
|
|
28
29
|
datacosmos/stac/collection/models/collection_update.py,sha256=XC6-29nLz1VGWMxYAw7r1OuL8PdJ3b2oI-RPvnM-XXI,1657
|
|
29
30
|
datacosmos/stac/constants/__init__.py,sha256=dDRSsF7CKqNF44yIlNdE-PD1sp0Q5mhTEPT7hHIK7YE,26
|
|
@@ -33,29 +34,29 @@ datacosmos/stac/enums/processing_level.py,sha256=_k4FO818VlZWtlm-rULhg-CYkbewkOd
|
|
|
33
34
|
datacosmos/stac/enums/product_type.py,sha256=7lL0unJ1hxevW8Pepn9rmydUUWIORu2x4MEtp6rSFbA,196
|
|
34
35
|
datacosmos/stac/enums/season.py,sha256=QvUzXBYtPEfixhlbV0SAw2u_HK3tRFEnHKshJyIatdg,241
|
|
35
36
|
datacosmos/stac/item/__init__.py,sha256=lRuD_yp-JxoLqBA23q0XMkCNImf4T-X3BJnSw9u_3Yk,200
|
|
36
|
-
datacosmos/stac/item/item_client.py,sha256=
|
|
37
|
+
datacosmos/stac/item/item_client.py,sha256=o05_xHqndaNUrPvHmwkGh1cxe0jnk3tn2Itray7VLh0,10170
|
|
37
38
|
datacosmos/stac/item/models/__init__.py,sha256=bcOrOcIxGxGBrRVIyQVxSM3C3Xj_qzxIHgQeWo6f7Q8,34
|
|
38
39
|
datacosmos/stac/item/models/asset.py,sha256=mvg_fenYCGOTMGwXXpK2nyqBk5RMsUYxl6KhQTWW_b0,631
|
|
39
40
|
datacosmos/stac/item/models/catalog_search_parameters.py,sha256=3HrUm37VezujwuCR45jhMryS5m1FGc1XmX8-fdTy4jU,4870
|
|
40
|
-
datacosmos/stac/item/models/datacosmos_item.py,sha256=
|
|
41
|
+
datacosmos/stac/item/models/datacosmos_item.py,sha256=BzuqyHvcCq2ADYkg0F4rsu4Vzj4egPQnttSFw71xGFY,5352
|
|
41
42
|
datacosmos/stac/item/models/eo_band.py,sha256=YC3Scn_wFhIo51pIVcJeuJienF7JGWoEv39JngDM6rI,309
|
|
42
43
|
datacosmos/stac/item/models/item_update.py,sha256=_CpjQn9SsfedfuxlHSiGeptqY4M-p15t9YX__mBRueI,2088
|
|
43
44
|
datacosmos/stac/item/models/raster_band.py,sha256=CoEVs-YyPE5Fse0He9DdOs4dGZpzfCsCuVzOcdXa_UM,354
|
|
44
45
|
datacosmos/stac/storage/__init__.py,sha256=hivfSpOaoSwCAymgU0rTgvSk9LSPAn1cPLQQ9fLmFX0,151
|
|
45
|
-
datacosmos/stac/storage/storage_base.py,sha256=
|
|
46
|
+
datacosmos/stac/storage/storage_base.py,sha256=NpCKAA3qEI212WUNZ2-eG9XfSJKDMiJcu4pEmr10-JI,2843
|
|
46
47
|
datacosmos/stac/storage/storage_client.py,sha256=4boqQ3zVMrk9X2IXus-Cs429juLe0cUQ0XEzg_y3yOA,1205
|
|
47
|
-
datacosmos/stac/storage/uploader.py,sha256=
|
|
48
|
+
datacosmos/stac/storage/uploader.py,sha256=DawtNn4-uEtpUYPZS1fKv77wu-zNne9ltlGZiArzBFI,5919
|
|
48
49
|
datacosmos/stac/storage/dataclasses/__init__.py,sha256=IjcyA8Vod-z1_Gi1FMZhK58Owman0foL25Hs0YtkYYs,43
|
|
49
50
|
datacosmos/stac/storage/dataclasses/upload_path.py,sha256=gbpV67FECFNyXn-yGUSuLvGGWHtibbZq7Qu9yGod3C0,1398
|
|
50
51
|
datacosmos/utils/__init__.py,sha256=XQbAnoqJrPpnSpEzAbjh84yqYWw8cBM8mNp8ynTG-54,50
|
|
51
52
|
datacosmos/utils/url.py,sha256=iQwZr6mYRoePqUZg-k3KQSV9o2wju5ZuCa5WS_GyJo4,2114
|
|
52
53
|
datacosmos/utils/http_response/__init__.py,sha256=BvOWwC5coYqq_kFn8gIw5m54TLpdfJKlW9vgRkfhXiA,33
|
|
53
|
-
datacosmos/utils/http_response/check_api_response.py,sha256=
|
|
54
|
+
datacosmos/utils/http_response/check_api_response.py,sha256=l_yQiiekNcNbhFec_5Ue2mFY3YjwWEJS1gilhEJ3Luw,1158
|
|
54
55
|
datacosmos/utils/http_response/models/__init__.py,sha256=Wj8YT6dqw7rAz_rctllxo5Or_vv8DwopvQvBzwCTvpw,45
|
|
55
56
|
datacosmos/utils/http_response/models/datacosmos_error.py,sha256=Uqi2uM98nJPeCbM7zngV6vHSk97jEAb_nkdDEeUjiQM,740
|
|
56
57
|
datacosmos/utils/http_response/models/datacosmos_response.py,sha256=oV4n-sue7K1wwiIQeHpxdNU8vxeqF3okVPE2rydw5W0,336
|
|
57
|
-
datacosmos-0.0.
|
|
58
|
-
datacosmos-0.0.
|
|
59
|
-
datacosmos-0.0.
|
|
60
|
-
datacosmos-0.0.
|
|
61
|
-
datacosmos-0.0.
|
|
58
|
+
datacosmos-0.0.20.dist-info/licenses/LICENSE.md,sha256=vpbRI-UUbZVQfr3VG_CXt9HpRnL1b5kt8uTVbirxeyI,1486
|
|
59
|
+
datacosmos-0.0.20.dist-info/METADATA,sha256=WfpmuajQ7GSenjnr8X04rNXDg6U6NH3pmbFMtJYqFV4,1000
|
|
60
|
+
datacosmos-0.0.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
61
|
+
datacosmos-0.0.20.dist-info/top_level.txt,sha256=ueobs5CNeyDbPMgXPcVV0d0yNdm8CvGtDT3CaksRVtA,11
|
|
62
|
+
datacosmos-0.0.20.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|