datacosmos 0.0.8__py3-none-any.whl → 0.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacosmos might be problematic. Click here for more details.
- datacosmos/config/config.py +31 -2
- datacosmos/stac/item/item_client.py +43 -4
- datacosmos/stac/stac_client.py +5 -2
- datacosmos/stac/storage/__init__.py +5 -0
- datacosmos/stac/storage/storage_base.py +40 -0
- datacosmos/stac/storage/storage_client.py +31 -0
- datacosmos/stac/storage/uploader.py +98 -0
- {datacosmos-0.0.8.dist-info → datacosmos-0.0.9.dist-info}/METADATA +1 -1
- {datacosmos-0.0.8.dist-info → datacosmos-0.0.9.dist-info}/RECORD +14 -12
- datacosmos/uploader/__init__.py +0 -1
- datacosmos/uploader/datacosmos_uploader.py +0 -101
- /datacosmos/{uploader → stac/storage}/dataclasses/__init__.py +0 -0
- /datacosmos/{uploader → stac/storage}/dataclasses/upload_path.py +0 -0
- {datacosmos-0.0.8.dist-info → datacosmos-0.0.9.dist-info}/WHEEL +0 -0
- {datacosmos-0.0.8.dist-info → datacosmos-0.0.9.dist-info}/licenses/LICENSE.md +0 -0
- {datacosmos-0.0.8.dist-info → datacosmos-0.0.9.dist-info}/top_level.txt +0 -0
datacosmos/config/config.py
CHANGED
|
@@ -29,6 +29,7 @@ class Config(BaseSettings):
|
|
|
29
29
|
authentication: Optional[AuthenticationConfig] = None
|
|
30
30
|
stac: Optional[URL] = None
|
|
31
31
|
datacosmos_cloud_storage: Optional[URL] = None
|
|
32
|
+
datacosmos_public_cloud_storage: Optional[URL] = None
|
|
32
33
|
|
|
33
34
|
DEFAULT_AUTH_TYPE: ClassVar[str] = "m2m"
|
|
34
35
|
DEFAULT_AUTH_TOKEN_URL: ClassVar[str] = "https://login.open-cosmos.com/oauth/token"
|
|
@@ -86,12 +87,18 @@ class Config(BaseSettings):
|
|
|
86
87
|
path=os.getenv("DC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
|
|
87
88
|
)
|
|
88
89
|
|
|
90
|
+
datacosmos_public_cloud_storage_config = URL(
|
|
91
|
+
protocol=os.getenv("DC_PUBLIC_CLOUD_STORAGE_PROTOCOL", "https"),
|
|
92
|
+
host=os.getenv("DC_PUBLIC_CLOUD_STORAGE_HOST", "app.open-cosmos.com"),
|
|
93
|
+
port=int(os.getenv("DC_PUBLIC_CLOUD_STORAGE_PORT", "443")),
|
|
94
|
+
path=os.getenv("DC_PUBLIC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
|
|
95
|
+
)
|
|
96
|
+
|
|
89
97
|
return cls(
|
|
90
98
|
authentication=authentication_config,
|
|
91
99
|
stac=stac_config,
|
|
92
100
|
datacosmos_cloud_storage=datacosmos_cloud_storage_config,
|
|
93
|
-
|
|
94
|
-
environment=os.getenv("ENVIRONMENT", "test"),
|
|
101
|
+
datacosmos_public_cloud_storage=datacosmos_public_cloud_storage_config,
|
|
95
102
|
)
|
|
96
103
|
|
|
97
104
|
@field_validator("authentication", mode="after")
|
|
@@ -191,3 +198,25 @@ class Config(BaseSettings):
|
|
|
191
198
|
path="/api/data/v0/storage",
|
|
192
199
|
)
|
|
193
200
|
return datacosmos_cloud_storage_config
|
|
201
|
+
|
|
202
|
+
@field_validator("datacosmos_public_cloud_storage", mode="before")
|
|
203
|
+
@classmethod
|
|
204
|
+
def validate_datacosmos_public_cloud_storage(
|
|
205
|
+
cls, datacosmos_public_cloud_storage_config: Optional[URL]
|
|
206
|
+
) -> URL:
|
|
207
|
+
"""Ensure datacosmos cloud storage configuration has a default if not explicitly set.
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
datacosmos_public_cloud_storage_config (Optional[URL]): The datacosmos public cloud storage config to validate.
|
|
211
|
+
|
|
212
|
+
Returns:
|
|
213
|
+
URL: The validated datacosmos public cloud storage configuration.
|
|
214
|
+
"""
|
|
215
|
+
if datacosmos_public_cloud_storage_config is None:
|
|
216
|
+
return URL(
|
|
217
|
+
protocol="https",
|
|
218
|
+
host="app.open-cosmos.com",
|
|
219
|
+
port=443,
|
|
220
|
+
path="/api/data/v0/storage",
|
|
221
|
+
)
|
|
222
|
+
return datacosmos_public_cloud_storage_config
|
|
@@ -62,21 +62,60 @@ class ItemClient:
|
|
|
62
62
|
body = body | {"collections": parameters.collections}
|
|
63
63
|
return self._paginate_items(url, body)
|
|
64
64
|
|
|
65
|
-
def create_item(self,
|
|
66
|
-
"""Create a new STAC item in
|
|
65
|
+
def create_item(self, item: Item | DatacosmosItem) -> None:
|
|
66
|
+
"""Create a new STAC item in its own collection.
|
|
67
|
+
|
|
68
|
+
The collection ID is inferred from the item.
|
|
67
69
|
|
|
68
70
|
Args:
|
|
69
|
-
|
|
70
|
-
item (Item): The STAC Item to be created.
|
|
71
|
+
item (Item | DatacosmosItem): The STAC item to be created.
|
|
71
72
|
|
|
72
73
|
Raises:
|
|
74
|
+
ValueError: If the item has no collection set.
|
|
73
75
|
RequestError: If the API returns an error response.
|
|
74
76
|
"""
|
|
77
|
+
if isinstance(item, Item):
|
|
78
|
+
collection_id = item.collection_id or (
|
|
79
|
+
item.get_collection().id if item.get_collection() else None
|
|
80
|
+
)
|
|
81
|
+
else:
|
|
82
|
+
collection_id = item.collection
|
|
83
|
+
|
|
84
|
+
if not collection_id:
|
|
85
|
+
raise ValueError("Cannot create item: no collection_id found on item")
|
|
86
|
+
|
|
75
87
|
url = self.base_url.with_suffix(f"/collections/{collection_id}/items")
|
|
76
88
|
item_json: dict = item.to_dict()
|
|
77
89
|
response = self.client.post(url, json=item_json)
|
|
78
90
|
check_api_response(response)
|
|
79
91
|
|
|
92
|
+
def add_item(self, item: Item | DatacosmosItem) -> None:
|
|
93
|
+
"""Adds item to catalog.
|
|
94
|
+
|
|
95
|
+
The collection ID is inferred from the item.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
item (Item | DatacosmosItem): The STAC item to be created.
|
|
99
|
+
|
|
100
|
+
Raises:
|
|
101
|
+
ValueError: If the item has no collection set.
|
|
102
|
+
RequestError: If the API returns an error response.
|
|
103
|
+
"""
|
|
104
|
+
if isinstance(item, Item):
|
|
105
|
+
collection_id = item.collection_id or (
|
|
106
|
+
item.get_collection().id if item.get_collection() else None
|
|
107
|
+
)
|
|
108
|
+
else:
|
|
109
|
+
collection_id = item.collection
|
|
110
|
+
|
|
111
|
+
if not collection_id:
|
|
112
|
+
raise ValueError("Cannot create item: no collection_id found on item")
|
|
113
|
+
|
|
114
|
+
url = self.base_url.with_suffix(f"/collections/{collection_id}/items/{item.id}")
|
|
115
|
+
item_json: dict = item.to_dict()
|
|
116
|
+
response = self.client.put(url, json=item_json)
|
|
117
|
+
check_api_response(response)
|
|
118
|
+
|
|
80
119
|
def update_item(
|
|
81
120
|
self, item_id: str, collection_id: str, update_data: ItemUpdate
|
|
82
121
|
) -> None:
|
datacosmos/stac/stac_client.py
CHANGED
|
@@ -2,11 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from datacosmos.stac.collection.collection_client import CollectionClient
|
|
4
4
|
from datacosmos.stac.item.item_client import ItemClient
|
|
5
|
+
from datacosmos.stac.storage.storage_client import StorageClient
|
|
5
6
|
|
|
6
7
|
|
|
7
|
-
class STACClient(ItemClient, CollectionClient):
|
|
8
|
+
class STACClient(ItemClient, CollectionClient, StorageClient):
|
|
8
9
|
"""Unified interface for STAC API, combining Item & Collection operations."""
|
|
9
10
|
|
|
10
11
|
def __init__(self, client):
|
|
11
12
|
"""Initialize the STACClient with a DatacosmosClient."""
|
|
12
|
-
|
|
13
|
+
ItemClient.__init__(self, client)
|
|
14
|
+
CollectionClient.__init__(self, client)
|
|
15
|
+
StorageClient.__init__(self, client)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Base class providing common storage helpers (threading, MIME guess, futures)."""
|
|
2
|
+
|
|
3
|
+
import mimetypes
|
|
4
|
+
from concurrent.futures import ThreadPoolExecutor, wait
|
|
5
|
+
|
|
6
|
+
from datacosmos.datacosmos_client import DatacosmosClient
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class StorageBase:
|
|
10
|
+
"""Base class providing common storage helpers (threading, MIME guess, futures)."""
|
|
11
|
+
|
|
12
|
+
def __init__(self, client: DatacosmosClient):
|
|
13
|
+
"""Base class providing common storage helpers (threading, MIME guess, futures)."""
|
|
14
|
+
self.client = client
|
|
15
|
+
self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
|
|
16
|
+
|
|
17
|
+
def _guess_mime(self, src: str) -> str:
|
|
18
|
+
mime, _ = mimetypes.guess_type(src)
|
|
19
|
+
return mime or "application/octet-stream"
|
|
20
|
+
|
|
21
|
+
def _run_in_threads(self, fn, fn_args, max_workers: int, timeout: float):
|
|
22
|
+
"""Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
|
|
23
|
+
|
|
24
|
+
`jobs` should be a list of tuples, each tuple unpacked as fn(*args).
|
|
25
|
+
"""
|
|
26
|
+
futures = []
|
|
27
|
+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
28
|
+
for args in fn_args:
|
|
29
|
+
futures.append(executor.submit(fn, *args))
|
|
30
|
+
done, not_done = wait(futures, timeout=timeout)
|
|
31
|
+
errors = []
|
|
32
|
+
for future in done:
|
|
33
|
+
try:
|
|
34
|
+
future.result()
|
|
35
|
+
except Exception as e:
|
|
36
|
+
errors.append(e)
|
|
37
|
+
for future in not_done:
|
|
38
|
+
future.cancel()
|
|
39
|
+
if errors:
|
|
40
|
+
raise errors[0]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Generic StorageClient for all storage operations (upload, download, etc.)."""
|
|
2
|
+
|
|
3
|
+
from datacosmos.datacosmos_client import DatacosmosClient
|
|
4
|
+
from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
|
|
5
|
+
from datacosmos.stac.storage.uploader import Uploader
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class StorageClient:
|
|
9
|
+
"""Generic StorageClient for all storage operations (upload, download, etc.)."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, client: DatacosmosClient):
|
|
12
|
+
"""Generic StorageClient for all storage operations (upload, download, etc.)."""
|
|
13
|
+
self.client = client
|
|
14
|
+
self.uploader = Uploader(client)
|
|
15
|
+
|
|
16
|
+
def upload_item(
|
|
17
|
+
self,
|
|
18
|
+
item: DatacosmosItem,
|
|
19
|
+
assets_path: str | None = None,
|
|
20
|
+
included_assets: list[str] | bool = True,
|
|
21
|
+
max_workers: int = 4,
|
|
22
|
+
time_out: float = 60 * 60 * 1,
|
|
23
|
+
) -> DatacosmosItem:
|
|
24
|
+
"""Proxy to Uploader.upload_item, without needing to pass client each call."""
|
|
25
|
+
return self.uploader.upload_item(
|
|
26
|
+
item=item,
|
|
27
|
+
assets_path=assets_path,
|
|
28
|
+
included_assets=included_assets,
|
|
29
|
+
max_workers=max_workers,
|
|
30
|
+
time_out=time_out,
|
|
31
|
+
)
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Handles uploading files to Datacosmos storage and registering STAC items."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from pydantic import TypeAdapter
|
|
6
|
+
|
|
7
|
+
from datacosmos.datacosmos_client import DatacosmosClient
|
|
8
|
+
from datacosmos.stac.item.item_client import ItemClient
|
|
9
|
+
from datacosmos.stac.item.models.asset import Asset
|
|
10
|
+
from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
|
|
11
|
+
from datacosmos.stac.storage.dataclasses.upload_path import UploadPath
|
|
12
|
+
from datacosmos.stac.storage.storage_base import StorageBase
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Uploader(StorageBase):
|
|
16
|
+
"""Handles uploading files to Datacosmos storage and registering STAC items."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, client: DatacosmosClient):
|
|
19
|
+
"""Handles uploading files to Datacosmos storage and registering STAC items."""
|
|
20
|
+
super().__init__(client)
|
|
21
|
+
self.item_client = ItemClient(client)
|
|
22
|
+
|
|
23
|
+
def upload_item(
|
|
24
|
+
self,
|
|
25
|
+
item: DatacosmosItem,
|
|
26
|
+
assets_path: str | None = None,
|
|
27
|
+
included_assets: list[str] | bool = True,
|
|
28
|
+
max_workers: int = 4,
|
|
29
|
+
time_out: float = 60 * 60 * 1,
|
|
30
|
+
) -> DatacosmosItem:
|
|
31
|
+
"""Upload a STAC item and its assets to Datacosmos."""
|
|
32
|
+
if not assets_path and not isinstance(item, str):
|
|
33
|
+
raise ValueError(
|
|
34
|
+
"assets_path must be provided if item is not the path to an item file."
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
if isinstance(item, str):
|
|
38
|
+
item_filename = item
|
|
39
|
+
item = self._load_item(item_filename)
|
|
40
|
+
if not assets_path:
|
|
41
|
+
assets_path = str(Path(item_filename).parent)
|
|
42
|
+
|
|
43
|
+
assets_path = assets_path or str(Path.cwd())
|
|
44
|
+
|
|
45
|
+
upload_assets = (
|
|
46
|
+
included_assets
|
|
47
|
+
if isinstance(included_assets, list)
|
|
48
|
+
else item.assets.keys()
|
|
49
|
+
if included_assets is True
|
|
50
|
+
else []
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
jobs = [(item, asset_key, assets_path) for asset_key in upload_assets]
|
|
54
|
+
|
|
55
|
+
self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
|
|
56
|
+
|
|
57
|
+
self.item_client.add_item(item)
|
|
58
|
+
|
|
59
|
+
return item
|
|
60
|
+
|
|
61
|
+
def upload_from_file(
|
|
62
|
+
self, src: str, dst: str, mime_type: str | None = None
|
|
63
|
+
) -> None:
|
|
64
|
+
"""Uploads a single file to the specified destination path."""
|
|
65
|
+
url = self.base_url.with_suffix(dst)
|
|
66
|
+
mime = mime_type or self._guess_mime(src)
|
|
67
|
+
headers = {"Content-Type": mime}
|
|
68
|
+
with open(src, "rb") as f:
|
|
69
|
+
response = self.client.put(url, data=f, headers=headers)
|
|
70
|
+
response.raise_for_status()
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
def _load_item(item_json_file_path: str) -> DatacosmosItem:
|
|
74
|
+
with open(item_json_file_path, "rb") as file:
|
|
75
|
+
data = file.read().decode("utf-8")
|
|
76
|
+
return TypeAdapter(DatacosmosItem).validate_json(data)
|
|
77
|
+
|
|
78
|
+
def _upload_asset(
|
|
79
|
+
self, item: DatacosmosItem, asset_key: str, assets_path: str
|
|
80
|
+
) -> None:
|
|
81
|
+
asset = item.assets[asset_key]
|
|
82
|
+
upload_path = UploadPath.from_item_path(item, "", Path(asset.href).name)
|
|
83
|
+
local_src = Path(assets_path) / asset.href
|
|
84
|
+
if local_src.exists():
|
|
85
|
+
src = str(local_src)
|
|
86
|
+
asset.href = f"file:///{upload_path}"
|
|
87
|
+
else:
|
|
88
|
+
src = str(Path(assets_path) / Path(asset.href).name)
|
|
89
|
+
self._update_asset_href(asset)
|
|
90
|
+
self.upload_from_file(src, str(upload_path), mime_type=asset.type)
|
|
91
|
+
|
|
92
|
+
def _update_asset_href(self, asset: Asset) -> None:
|
|
93
|
+
try:
|
|
94
|
+
url = self.client.config.datacosmos_public_cloud_storage.as_domain_url()
|
|
95
|
+
new_href = url.with_base(asset.href) # type: ignore
|
|
96
|
+
asset.href = str(new_href)
|
|
97
|
+
except ValueError:
|
|
98
|
+
pass
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
datacosmos/__init__.py,sha256=dVHKpbz5FVtfoJAWHRdsUENG6H-vs4UrkuwnIvOGJr4,66
|
|
2
2
|
datacosmos/datacosmos_client.py,sha256=3BurTz1fPk1Dzp8B5xt5gZZrFiqk1AT5oaqKeYmXPec,6517
|
|
3
3
|
datacosmos/config/__init__.py,sha256=KCsaTb9-ZgFui1GM8wZFIPLJy0D0O8l8Z1Sv3NRD9UM,140
|
|
4
|
-
datacosmos/config/config.py,sha256=
|
|
4
|
+
datacosmos/config/config.py,sha256=3iet6ou0vrcreaJjIFn0s59fyXHnolf026r1qe9PEvY,8487
|
|
5
5
|
datacosmos/config/models/__init__.py,sha256=r3lThPkyKjBjUZXRNscFzOrmn_-m_i9DvG3RePfCFYc,41
|
|
6
6
|
datacosmos/config/models/authentication_config.py,sha256=01Q90-yupbJ5orYDtatZIm9EaL7roQ-oUMoZfFMRzIM,499
|
|
7
7
|
datacosmos/config/models/local_user_account_authentication_config.py,sha256=8WApn720MBXMKQa6w7bCd7Z37GRmYR-I7mBUgUI20lQ,701
|
|
@@ -11,7 +11,7 @@ datacosmos/config/models/url.py,sha256=fwr2C06e_RDS8AWxOV_orVxMWhc57bzYoWSjFxQbk
|
|
|
11
11
|
datacosmos/exceptions/__init__.py,sha256=Crz8W7mOvPUXYcfDVotvjUt_3HKawBpmJA_-uel9UJk,45
|
|
12
12
|
datacosmos/exceptions/datacosmos_exception.py,sha256=rKjJvQDvCEbxXWWccxB5GI_sth662bW8Yml0hX-vRw4,923
|
|
13
13
|
datacosmos/stac/__init__.py,sha256=B4x_Mr4X7TzQoYtRC-VzI4W-fEON5WUOaz8cWJbk3Fc,214
|
|
14
|
-
datacosmos/stac/stac_client.py,sha256=
|
|
14
|
+
datacosmos/stac/stac_client.py,sha256=J4k4aJdakwVK1sorBxeK8KbPtYvjIGa68iqKA_itSgU,654
|
|
15
15
|
datacosmos/stac/collection/__init__.py,sha256=VQMLnsU3sER5kh4YxHrHP7XCA3DG1y0n9yoSmvycOY0,212
|
|
16
16
|
datacosmos/stac/collection/collection_client.py,sha256=-Nn3yqL4mQS05YAMd0IUmv03hdHKYBtVG2_EqoaAQWc,6064
|
|
17
17
|
datacosmos/stac/collection/models/__init__.py,sha256=TQaihUS_CM9Eaekm4SbzFTNfv7BmabHv3Z-f37Py5Qs,40
|
|
@@ -23,7 +23,7 @@ datacosmos/stac/enums/processing_level.py,sha256=5gHG-0kG5rCUxmXYwF3t94ALKk6zUqg
|
|
|
23
23
|
datacosmos/stac/enums/product_type.py,sha256=7lL0unJ1hxevW8Pepn9rmydUUWIORu2x4MEtp6rSFbA,196
|
|
24
24
|
datacosmos/stac/enums/season.py,sha256=QvUzXBYtPEfixhlbV0SAw2u_HK3tRFEnHKshJyIatdg,241
|
|
25
25
|
datacosmos/stac/item/__init__.py,sha256=lRuD_yp-JxoLqBA23q0XMkCNImf4T-X3BJnSw9u_3Yk,200
|
|
26
|
-
datacosmos/stac/item/item_client.py,sha256=
|
|
26
|
+
datacosmos/stac/item/item_client.py,sha256=HCHl3cHp0u2qxbwLxPk0xkujC1D4uwIBIFI-flpLXfQ,7783
|
|
27
27
|
datacosmos/stac/item/models/__init__.py,sha256=bcOrOcIxGxGBrRVIyQVxSM3C3Xj_qzxIHgQeWo6f7Q8,34
|
|
28
28
|
datacosmos/stac/item/models/asset.py,sha256=mvg_fenYCGOTMGwXXpK2nyqBk5RMsUYxl6KhQTWW_b0,631
|
|
29
29
|
datacosmos/stac/item/models/catalog_search_parameters.py,sha256=3HrUm37VezujwuCR45jhMryS5m1FGc1XmX8-fdTy4jU,4870
|
|
@@ -31,10 +31,12 @@ datacosmos/stac/item/models/datacosmos_item.py,sha256=AImz0GRxrpZfIETdzzNfaKX35w
|
|
|
31
31
|
datacosmos/stac/item/models/eo_band.py,sha256=YC3Scn_wFhIo51pIVcJeuJienF7JGWoEv39JngDM6rI,309
|
|
32
32
|
datacosmos/stac/item/models/item_update.py,sha256=_CpjQn9SsfedfuxlHSiGeptqY4M-p15t9YX__mBRueI,2088
|
|
33
33
|
datacosmos/stac/item/models/raster_band.py,sha256=CoEVs-YyPE5Fse0He9DdOs4dGZpzfCsCuVzOcdXa_UM,354
|
|
34
|
-
datacosmos/
|
|
35
|
-
datacosmos/
|
|
36
|
-
datacosmos/
|
|
37
|
-
datacosmos/
|
|
34
|
+
datacosmos/stac/storage/__init__.py,sha256=hivfSpOaoSwCAymgU0rTgvSk9LSPAn1cPLQQ9fLmFX0,151
|
|
35
|
+
datacosmos/stac/storage/storage_base.py,sha256=5ioMKbEltPEWr4dkhZQiUhdBFEhe7ajIYUd9z3K8elU,1483
|
|
36
|
+
datacosmos/stac/storage/storage_client.py,sha256=GeWJoa8ALqelZHvmnop_sSuyU7ntFNFXMFQfplIo0kU,1145
|
|
37
|
+
datacosmos/stac/storage/uploader.py,sha256=5W4Wcx2yzdkU9sg93jnwYP0TiZcuxQXB9owfjL2NsBg,3630
|
|
38
|
+
datacosmos/stac/storage/dataclasses/__init__.py,sha256=IjcyA8Vod-z1_Gi1FMZhK58Owman0foL25Hs0YtkYYs,43
|
|
39
|
+
datacosmos/stac/storage/dataclasses/upload_path.py,sha256=5QadynHxkJrnOk1lyPtLyiVAHdzBshEuhjA9hwVF0NI,1903
|
|
38
40
|
datacosmos/utils/__init__.py,sha256=XQbAnoqJrPpnSpEzAbjh84yqYWw8cBM8mNp8ynTG-54,50
|
|
39
41
|
datacosmos/utils/url.py,sha256=iQwZr6mYRoePqUZg-k3KQSV9o2wju5ZuCa5WS_GyJo4,2114
|
|
40
42
|
datacosmos/utils/http_response/__init__.py,sha256=BvOWwC5coYqq_kFn8gIw5m54TLpdfJKlW9vgRkfhXiA,33
|
|
@@ -42,8 +44,8 @@ datacosmos/utils/http_response/check_api_response.py,sha256=dKWW01jn2_lWV0xpOBAB
|
|
|
42
44
|
datacosmos/utils/http_response/models/__init__.py,sha256=Wj8YT6dqw7rAz_rctllxo5Or_vv8DwopvQvBzwCTvpw,45
|
|
43
45
|
datacosmos/utils/http_response/models/datacosmos_error.py,sha256=Uqi2uM98nJPeCbM7zngV6vHSk97jEAb_nkdDEeUjiQM,740
|
|
44
46
|
datacosmos/utils/http_response/models/datacosmos_response.py,sha256=oV4n-sue7K1wwiIQeHpxdNU8vxeqF3okVPE2rydw5W0,336
|
|
45
|
-
datacosmos-0.0.
|
|
46
|
-
datacosmos-0.0.
|
|
47
|
-
datacosmos-0.0.
|
|
48
|
-
datacosmos-0.0.
|
|
49
|
-
datacosmos-0.0.
|
|
47
|
+
datacosmos-0.0.9.dist-info/licenses/LICENSE.md,sha256=vpbRI-UUbZVQfr3VG_CXt9HpRnL1b5kt8uTVbirxeyI,1486
|
|
48
|
+
datacosmos-0.0.9.dist-info/METADATA,sha256=1vpXTtgxF7gG1CdXSg0nsprKapv5LocXA2lvXaSZFp0,896
|
|
49
|
+
datacosmos-0.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
50
|
+
datacosmos-0.0.9.dist-info/top_level.txt,sha256=ueobs5CNeyDbPMgXPcVV0d0yNdm8CvGtDT3CaksRVtA,11
|
|
51
|
+
datacosmos-0.0.9.dist-info/RECORD,,
|
datacosmos/uploader/__init__.py
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
"""Uploader package for interacting with the Uploader API, providing upload functionalities to the datacosmos cloud storage."""
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
"""Module for uploading files to Datacosmos cloud storage and registering STAC items."""
|
|
2
|
-
|
|
3
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
from pydantic import TypeAdapter
|
|
7
|
-
|
|
8
|
-
from datacosmos.datacosmos_client import DatacosmosClient
|
|
9
|
-
from datacosmos.stac.item.item_client import ItemClient
|
|
10
|
-
from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
|
|
11
|
-
from datacosmos.uploader.dataclasses.upload_path import UploadPath
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class DatacosmosUploader:
|
|
15
|
-
"""Handles uploading files to Datacosmos storage and registering STAC items."""
|
|
16
|
-
|
|
17
|
-
def __init__(self, client: DatacosmosClient):
|
|
18
|
-
"""Initialize the uploader with DatacosmosClient."""
|
|
19
|
-
self.environment = client.config.environment
|
|
20
|
-
|
|
21
|
-
self.datacosmos_client = client
|
|
22
|
-
self.item_client = ItemClient(client)
|
|
23
|
-
self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
|
|
24
|
-
|
|
25
|
-
def upload_and_register_item(self, item_json_file_path: str) -> None:
|
|
26
|
-
"""Uploads files to Datacosmos storage and registers a STAC item.
|
|
27
|
-
|
|
28
|
-
Args:
|
|
29
|
-
item_json_file_path (str): Path to the STAC item JSON file.
|
|
30
|
-
"""
|
|
31
|
-
item = self._load_item(item_json_file_path)
|
|
32
|
-
collection_id, item_id = item.collection, item.id
|
|
33
|
-
dirname = str(Path(item_json_file_path).parent / Path(item_json_file_path).stem)
|
|
34
|
-
|
|
35
|
-
self._delete_existing_item(collection_id, item_id)
|
|
36
|
-
upload_path = self._get_upload_path(item)
|
|
37
|
-
self.upload_from_folder(dirname, upload_path)
|
|
38
|
-
|
|
39
|
-
self._update_item_assets(item)
|
|
40
|
-
|
|
41
|
-
self.item_client.create_item(collection_id, item)
|
|
42
|
-
|
|
43
|
-
def upload_file(self, src: str, dst: str) -> None:
|
|
44
|
-
"""Uploads a single file to the specified destination path."""
|
|
45
|
-
url = self.base_url.with_suffix(dst)
|
|
46
|
-
|
|
47
|
-
with open(src, "rb") as f:
|
|
48
|
-
response = self.datacosmos_client.put(url, data=f)
|
|
49
|
-
response.raise_for_status()
|
|
50
|
-
|
|
51
|
-
def upload_from_folder(self, src: str, dst: UploadPath, workers: int = 4) -> None:
|
|
52
|
-
"""Uploads all files from a folder to the destination path in parallel."""
|
|
53
|
-
if Path(dst.path).is_file():
|
|
54
|
-
raise ValueError(f"Destination path should not be a file path {dst}")
|
|
55
|
-
|
|
56
|
-
if Path(src).is_file():
|
|
57
|
-
raise ValueError(f"Source path should not be a file path {src}")
|
|
58
|
-
|
|
59
|
-
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
60
|
-
futures = []
|
|
61
|
-
for file in Path(src).rglob("*"):
|
|
62
|
-
if file.is_file():
|
|
63
|
-
dst = UploadPath(
|
|
64
|
-
mission=dst.mission,
|
|
65
|
-
level=dst.level,
|
|
66
|
-
day=dst.day,
|
|
67
|
-
month=dst.month,
|
|
68
|
-
year=dst.year,
|
|
69
|
-
id=dst.id,
|
|
70
|
-
path=str(file.relative_to(src)),
|
|
71
|
-
)
|
|
72
|
-
futures.append(executor.submit(self.upload_file, str(file), dst))
|
|
73
|
-
for future in futures:
|
|
74
|
-
future.result()
|
|
75
|
-
|
|
76
|
-
@staticmethod
|
|
77
|
-
def _load_item(item_json_file_path: str) -> DatacosmosItem:
|
|
78
|
-
"""Loads and validates the STAC item from a JSON file."""
|
|
79
|
-
with open(item_json_file_path, "rb") as file:
|
|
80
|
-
data = file.read().decode("utf-8")
|
|
81
|
-
return TypeAdapter(DatacosmosItem).validate_json(data)
|
|
82
|
-
|
|
83
|
-
def _delete_existing_item(self, collection_id: str, item_id: str) -> None:
|
|
84
|
-
"""Deletes an existing item if it already exists."""
|
|
85
|
-
try:
|
|
86
|
-
self.item_client.delete_item(item_id, collection_id)
|
|
87
|
-
except Exception: # nosec
|
|
88
|
-
pass # Ignore if item doesn't exist
|
|
89
|
-
|
|
90
|
-
def _get_upload_path(self, item: DatacosmosItem, mission_name: str = "") -> str:
|
|
91
|
-
"""Constructs the storage upload path based on the item and mission name."""
|
|
92
|
-
return UploadPath.from_item_path(item, mission_name, "")
|
|
93
|
-
|
|
94
|
-
def _update_item_assets(self, item: DatacosmosItem) -> None:
|
|
95
|
-
"""Updates the item's assets with uploaded file URLs."""
|
|
96
|
-
for asset in item.assets.values():
|
|
97
|
-
try:
|
|
98
|
-
url = self.base_url
|
|
99
|
-
asset.href = url.with_base(asset.href) # type: ignore
|
|
100
|
-
except ValueError:
|
|
101
|
-
pass
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|