datacosmos 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of datacosmos might be problematic. Click here for more details.

@@ -29,6 +29,7 @@ class Config(BaseSettings):
29
29
  authentication: Optional[AuthenticationConfig] = None
30
30
  stac: Optional[URL] = None
31
31
  datacosmos_cloud_storage: Optional[URL] = None
32
+ datacosmos_public_cloud_storage: Optional[URL] = None
32
33
 
33
34
  DEFAULT_AUTH_TYPE: ClassVar[str] = "m2m"
34
35
  DEFAULT_AUTH_TOKEN_URL: ClassVar[str] = "https://login.open-cosmos.com/oauth/token"
@@ -86,12 +87,18 @@ class Config(BaseSettings):
86
87
  path=os.getenv("DC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
87
88
  )
88
89
 
90
+ datacosmos_public_cloud_storage_config = URL(
91
+ protocol=os.getenv("DC_PUBLIC_CLOUD_STORAGE_PROTOCOL", "https"),
92
+ host=os.getenv("DC_PUBLIC_CLOUD_STORAGE_HOST", "app.open-cosmos.com"),
93
+ port=int(os.getenv("DC_PUBLIC_CLOUD_STORAGE_PORT", "443")),
94
+ path=os.getenv("DC_PUBLIC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
95
+ )
96
+
89
97
  return cls(
90
98
  authentication=authentication_config,
91
99
  stac=stac_config,
92
100
  datacosmos_cloud_storage=datacosmos_cloud_storage_config,
93
- mission_id=int(os.getenv("MISSION_ID", "0")),
94
- environment=os.getenv("ENVIRONMENT", "test"),
101
+ datacosmos_public_cloud_storage=datacosmos_public_cloud_storage_config,
95
102
  )
96
103
 
97
104
  @field_validator("authentication", mode="after")
@@ -191,3 +198,25 @@ class Config(BaseSettings):
191
198
  path="/api/data/v0/storage",
192
199
  )
193
200
  return datacosmos_cloud_storage_config
201
+
202
+ @field_validator("datacosmos_public_cloud_storage", mode="before")
203
+ @classmethod
204
+ def validate_datacosmos_public_cloud_storage(
205
+ cls, datacosmos_public_cloud_storage_config: Optional[URL]
206
+ ) -> URL:
207
+ """Ensure datacosmos cloud storage configuration has a default if not explicitly set.
208
+
209
+ Args:
210
+ datacosmos_public_cloud_storage_config (Optional[URL]): The datacosmos public cloud storage config to validate.
211
+
212
+ Returns:
213
+ URL: The validated datacosmos public cloud storage configuration.
214
+ """
215
+ if datacosmos_public_cloud_storage_config is None:
216
+ return URL(
217
+ protocol="https",
218
+ host="app.open-cosmos.com",
219
+ port=443,
220
+ path="/api/data/v0/storage",
221
+ )
222
+ return datacosmos_public_cloud_storage_config
@@ -6,10 +6,11 @@ from enum import Enum
6
6
  class ProcessingLevel(Enum):
7
7
  """Enum class for the processing levels of the data."""
8
8
 
9
- L0 = "L0"
10
- L1A = "L1A"
11
- L2A = "L2A"
12
- L1B = "L1B"
13
- L1C = "L1C"
14
- L1D = "L1D"
15
- L3 = "L3"
9
+ RAW = "RAW"
10
+ L0 = "l0"
11
+ L1A = "l1A"
12
+ L2A = "l2A"
13
+ L1B = "l1B"
14
+ L1C = "l1C"
15
+ L1D = "l1D"
16
+ L3 = "l3"
@@ -62,21 +62,60 @@ class ItemClient:
62
62
  body = body | {"collections": parameters.collections}
63
63
  return self._paginate_items(url, body)
64
64
 
65
- def create_item(self, collection_id: str, item: Item | DatacosmosItem) -> None:
66
- """Create a new STAC item in a specified collection.
65
+ def create_item(self, item: Item | DatacosmosItem) -> None:
66
+ """Create a new STAC item in its own collection.
67
+
68
+ The collection ID is inferred from the item.
67
69
 
68
70
  Args:
69
- collection_id (str): The ID of the collection where the item will be created.
70
- item (Item): The STAC Item to be created.
71
+ item (Item | DatacosmosItem): The STAC item to be created.
71
72
 
72
73
  Raises:
74
+ ValueError: If the item has no collection set.
73
75
  RequestError: If the API returns an error response.
74
76
  """
77
+ if isinstance(item, Item):
78
+ collection_id = item.collection_id or (
79
+ item.get_collection().id if item.get_collection() else None
80
+ )
81
+ else:
82
+ collection_id = item.collection
83
+
84
+ if not collection_id:
85
+ raise ValueError("Cannot create item: no collection_id found on item")
86
+
75
87
  url = self.base_url.with_suffix(f"/collections/{collection_id}/items")
76
88
  item_json: dict = item.to_dict()
77
89
  response = self.client.post(url, json=item_json)
78
90
  check_api_response(response)
79
91
 
92
+ def add_item(self, item: Item | DatacosmosItem) -> None:
93
+ """Adds item to catalog.
94
+
95
+ The collection ID is inferred from the item.
96
+
97
+ Args:
98
+ item (Item | DatacosmosItem): The STAC item to be created.
99
+
100
+ Raises:
101
+ ValueError: If the item has no collection set.
102
+ RequestError: If the API returns an error response.
103
+ """
104
+ if isinstance(item, Item):
105
+ collection_id = item.collection_id or (
106
+ item.get_collection().id if item.get_collection() else None
107
+ )
108
+ else:
109
+ collection_id = item.collection
110
+
111
+ if not collection_id:
112
+ raise ValueError("Cannot create item: no collection_id found on item")
113
+
114
+ url = self.base_url.with_suffix(f"/collections/{collection_id}/items/{item.id}")
115
+ item_json: dict = item.to_dict()
116
+ response = self.client.put(url, json=item_json)
117
+ check_api_response(response)
118
+
80
119
  def update_item(
81
120
  self, item_id: str, collection_id: str, update_data: ItemUpdate
82
121
  ) -> None:
@@ -2,11 +2,14 @@
2
2
 
3
3
  from datacosmos.stac.collection.collection_client import CollectionClient
4
4
  from datacosmos.stac.item.item_client import ItemClient
5
+ from datacosmos.stac.storage.storage_client import StorageClient
5
6
 
6
7
 
7
- class STACClient(ItemClient, CollectionClient):
8
+ class STACClient(ItemClient, CollectionClient, StorageClient):
8
9
  """Unified interface for STAC API, combining Item & Collection operations."""
9
10
 
10
11
  def __init__(self, client):
11
12
  """Initialize the STACClient with a DatacosmosClient."""
12
- super().__init__(client)
13
+ ItemClient.__init__(self, client)
14
+ CollectionClient.__init__(self, client)
15
+ StorageClient.__init__(self, client)
@@ -0,0 +1,5 @@
1
+ """Facade module for all storage-related operations."""
2
+
3
+ from datacosmos.stac.storage.storage_client import StorageClient
4
+
5
+ __all__ = ["StorageClient"]
@@ -37,7 +37,7 @@ class UploadPath:
37
37
  dt = datetime.strptime(item.properties["datetime"], "%Y-%m-%dT%H:%M:%SZ")
38
38
  path = UploadPath(
39
39
  mission=mission,
40
- level=ProcessingLevel(item.properties["processing:level"].upper()),
40
+ level=ProcessingLevel(item.properties["processing:level"]),
41
41
  day=dt.day,
42
42
  month=dt.month,
43
43
  year=dt.year,
@@ -0,0 +1,40 @@
1
+ """Base class providing common storage helpers (threading, MIME guess, futures)."""
2
+
3
+ import mimetypes
4
+ from concurrent.futures import ThreadPoolExecutor, wait
5
+
6
+ from datacosmos.datacosmos_client import DatacosmosClient
7
+
8
+
9
+ class StorageBase:
10
+ """Base class providing common storage helpers (threading, MIME guess, futures)."""
11
+
12
+ def __init__(self, client: DatacosmosClient):
13
+ """Base class providing common storage helpers (threading, MIME guess, futures)."""
14
+ self.client = client
15
+ self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
16
+
17
+ def _guess_mime(self, src: str) -> str:
18
+ mime, _ = mimetypes.guess_type(src)
19
+ return mime or "application/octet-stream"
20
+
21
+ def _run_in_threads(self, fn, fn_args, max_workers: int, timeout: float):
22
+ """Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
23
+
24
+ `jobs` should be a list of tuples, each tuple unpacked as fn(*args).
25
+ """
26
+ futures = []
27
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
28
+ for args in fn_args:
29
+ futures.append(executor.submit(fn, *args))
30
+ done, not_done = wait(futures, timeout=timeout)
31
+ errors = []
32
+ for future in done:
33
+ try:
34
+ future.result()
35
+ except Exception as e:
36
+ errors.append(e)
37
+ for future in not_done:
38
+ future.cancel()
39
+ if errors:
40
+ raise errors[0]
@@ -0,0 +1,31 @@
1
+ """Generic StorageClient for all storage operations (upload, download, etc.)."""
2
+
3
+ from datacosmos.datacosmos_client import DatacosmosClient
4
+ from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
5
+ from datacosmos.stac.storage.uploader import Uploader
6
+
7
+
8
+ class StorageClient:
9
+ """Generic StorageClient for all storage operations (upload, download, etc.)."""
10
+
11
+ def __init__(self, client: DatacosmosClient):
12
+ """Generic StorageClient for all storage operations (upload, download, etc.)."""
13
+ self.client = client
14
+ self.uploader = Uploader(client)
15
+
16
+ def upload_item(
17
+ self,
18
+ item: DatacosmosItem,
19
+ assets_path: str | None = None,
20
+ included_assets: list[str] | bool = True,
21
+ max_workers: int = 4,
22
+ time_out: float = 60 * 60 * 1,
23
+ ) -> DatacosmosItem:
24
+ """Proxy to Uploader.upload_item, without needing to pass client each call."""
25
+ return self.uploader.upload_item(
26
+ item=item,
27
+ assets_path=assets_path,
28
+ included_assets=included_assets,
29
+ max_workers=max_workers,
30
+ time_out=time_out,
31
+ )
@@ -0,0 +1,98 @@
1
+ """Handles uploading files to Datacosmos storage and registering STAC items."""
2
+
3
+ from pathlib import Path
4
+
5
+ from pydantic import TypeAdapter
6
+
7
+ from datacosmos.datacosmos_client import DatacosmosClient
8
+ from datacosmos.stac.item.item_client import ItemClient
9
+ from datacosmos.stac.item.models.asset import Asset
10
+ from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
11
+ from datacosmos.stac.storage.dataclasses.upload_path import UploadPath
12
+ from datacosmos.stac.storage.storage_base import StorageBase
13
+
14
+
15
+ class Uploader(StorageBase):
16
+ """Handles uploading files to Datacosmos storage and registering STAC items."""
17
+
18
+ def __init__(self, client: DatacosmosClient):
19
+ """Handles uploading files to Datacosmos storage and registering STAC items."""
20
+ super().__init__(client)
21
+ self.item_client = ItemClient(client)
22
+
23
+ def upload_item(
24
+ self,
25
+ item: DatacosmosItem,
26
+ assets_path: str | None = None,
27
+ included_assets: list[str] | bool = True,
28
+ max_workers: int = 4,
29
+ time_out: float = 60 * 60 * 1,
30
+ ) -> DatacosmosItem:
31
+ """Upload a STAC item and its assets to Datacosmos."""
32
+ if not assets_path and not isinstance(item, str):
33
+ raise ValueError(
34
+ "assets_path must be provided if item is not the path to an item file."
35
+ )
36
+
37
+ if isinstance(item, str):
38
+ item_filename = item
39
+ item = self._load_item(item_filename)
40
+ if not assets_path:
41
+ assets_path = str(Path(item_filename).parent)
42
+
43
+ assets_path = assets_path or str(Path.cwd())
44
+
45
+ upload_assets = (
46
+ included_assets
47
+ if isinstance(included_assets, list)
48
+ else item.assets.keys()
49
+ if included_assets is True
50
+ else []
51
+ )
52
+
53
+ jobs = [(item, asset_key, assets_path) for asset_key in upload_assets]
54
+
55
+ self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
56
+
57
+ self.item_client.add_item(item)
58
+
59
+ return item
60
+
61
+ def upload_from_file(
62
+ self, src: str, dst: str, mime_type: str | None = None
63
+ ) -> None:
64
+ """Uploads a single file to the specified destination path."""
65
+ url = self.base_url.with_suffix(dst)
66
+ mime = mime_type or self._guess_mime(src)
67
+ headers = {"Content-Type": mime}
68
+ with open(src, "rb") as f:
69
+ response = self.client.put(url, data=f, headers=headers)
70
+ response.raise_for_status()
71
+
72
+ @staticmethod
73
+ def _load_item(item_json_file_path: str) -> DatacosmosItem:
74
+ with open(item_json_file_path, "rb") as file:
75
+ data = file.read().decode("utf-8")
76
+ return TypeAdapter(DatacosmosItem).validate_json(data)
77
+
78
+ def _upload_asset(
79
+ self, item: DatacosmosItem, asset_key: str, assets_path: str
80
+ ) -> None:
81
+ asset = item.assets[asset_key]
82
+ upload_path = UploadPath.from_item_path(item, "", Path(asset.href).name)
83
+ local_src = Path(assets_path) / asset.href
84
+ if local_src.exists():
85
+ src = str(local_src)
86
+ asset.href = f"file:///{upload_path}"
87
+ else:
88
+ src = str(Path(assets_path) / Path(asset.href).name)
89
+ self._update_asset_href(asset)
90
+ self.upload_from_file(src, str(upload_path), mime_type=asset.type)
91
+
92
+ def _update_asset_href(self, asset: Asset) -> None:
93
+ try:
94
+ url = self.client.config.datacosmos_public_cloud_storage.as_domain_url()
95
+ new_href = url.with_base(asset.href) # type: ignore
96
+ asset.href = str(new_href)
97
+ except ValueError:
98
+ pass
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datacosmos
3
- Version: 0.0.8
3
+ Version: 0.0.10
4
4
  Summary: A library for interacting with DataCosmos from Python code
5
5
  Author-email: Open Cosmos <support@open-cosmos.com>
6
6
  Classifier: Programming Language :: Python :: 3
@@ -1,7 +1,7 @@
1
1
  datacosmos/__init__.py,sha256=dVHKpbz5FVtfoJAWHRdsUENG6H-vs4UrkuwnIvOGJr4,66
2
2
  datacosmos/datacosmos_client.py,sha256=3BurTz1fPk1Dzp8B5xt5gZZrFiqk1AT5oaqKeYmXPec,6517
3
3
  datacosmos/config/__init__.py,sha256=KCsaTb9-ZgFui1GM8wZFIPLJy0D0O8l8Z1Sv3NRD9UM,140
4
- datacosmos/config/config.py,sha256=qj9e68enxX61Iw8lqEXUUCNJ2O7vy-1j9nAoBIB-4Ao,7219
4
+ datacosmos/config/config.py,sha256=3iet6ou0vrcreaJjIFn0s59fyXHnolf026r1qe9PEvY,8487
5
5
  datacosmos/config/models/__init__.py,sha256=r3lThPkyKjBjUZXRNscFzOrmn_-m_i9DvG3RePfCFYc,41
6
6
  datacosmos/config/models/authentication_config.py,sha256=01Q90-yupbJ5orYDtatZIm9EaL7roQ-oUMoZfFMRzIM,499
7
7
  datacosmos/config/models/local_user_account_authentication_config.py,sha256=8WApn720MBXMKQa6w7bCd7Z37GRmYR-I7mBUgUI20lQ,701
@@ -11,7 +11,7 @@ datacosmos/config/models/url.py,sha256=fwr2C06e_RDS8AWxOV_orVxMWhc57bzYoWSjFxQbk
11
11
  datacosmos/exceptions/__init__.py,sha256=Crz8W7mOvPUXYcfDVotvjUt_3HKawBpmJA_-uel9UJk,45
12
12
  datacosmos/exceptions/datacosmos_exception.py,sha256=rKjJvQDvCEbxXWWccxB5GI_sth662bW8Yml0hX-vRw4,923
13
13
  datacosmos/stac/__init__.py,sha256=B4x_Mr4X7TzQoYtRC-VzI4W-fEON5WUOaz8cWJbk3Fc,214
14
- datacosmos/stac/stac_client.py,sha256=Cz_p96RmAgWX8t7Sye4OJRanQpCLihKStvfEw7IgYZc,472
14
+ datacosmos/stac/stac_client.py,sha256=J4k4aJdakwVK1sorBxeK8KbPtYvjIGa68iqKA_itSgU,654
15
15
  datacosmos/stac/collection/__init__.py,sha256=VQMLnsU3sER5kh4YxHrHP7XCA3DG1y0n9yoSmvycOY0,212
16
16
  datacosmos/stac/collection/collection_client.py,sha256=-Nn3yqL4mQS05YAMd0IUmv03hdHKYBtVG2_EqoaAQWc,6064
17
17
  datacosmos/stac/collection/models/__init__.py,sha256=TQaihUS_CM9Eaekm4SbzFTNfv7BmabHv3Z-f37Py5Qs,40
@@ -19,11 +19,11 @@ datacosmos/stac/collection/models/collection_update.py,sha256=XC6-29nLz1VGWMxYAw
19
19
  datacosmos/stac/constants/__init__.py,sha256=dDRSsF7CKqNF44yIlNdE-PD1sp0Q5mhTEPT7hHIK7YE,26
20
20
  datacosmos/stac/constants/satellite_name_mapping.py,sha256=EJqNdO9uW5B-sIeDF72AjnW7va5BM9mm4oNwijtl51w,575
21
21
  datacosmos/stac/enums/__init__.py,sha256=GUEL2xGtdjsrszrxivs0X6daxkaZs2JsTu2JoBtsvB4,22
22
- datacosmos/stac/enums/processing_level.py,sha256=5gHG-0kG5rCUxmXYwF3t94ALKk6zUqguOdyTL-jwgps,247
22
+ datacosmos/stac/enums/processing_level.py,sha256=kWGhpMXgzoyTzwR4yoSFd3UQ5IBw3cUf3TaM1FaOtUg,263
23
23
  datacosmos/stac/enums/product_type.py,sha256=7lL0unJ1hxevW8Pepn9rmydUUWIORu2x4MEtp6rSFbA,196
24
24
  datacosmos/stac/enums/season.py,sha256=QvUzXBYtPEfixhlbV0SAw2u_HK3tRFEnHKshJyIatdg,241
25
25
  datacosmos/stac/item/__init__.py,sha256=lRuD_yp-JxoLqBA23q0XMkCNImf4T-X3BJnSw9u_3Yk,200
26
- datacosmos/stac/item/item_client.py,sha256=mFcbXqV1Ascd5hUSlZFzNgni_DncHAIyIvhtUHpgHI0,6457
26
+ datacosmos/stac/item/item_client.py,sha256=HCHl3cHp0u2qxbwLxPk0xkujC1D4uwIBIFI-flpLXfQ,7783
27
27
  datacosmos/stac/item/models/__init__.py,sha256=bcOrOcIxGxGBrRVIyQVxSM3C3Xj_qzxIHgQeWo6f7Q8,34
28
28
  datacosmos/stac/item/models/asset.py,sha256=mvg_fenYCGOTMGwXXpK2nyqBk5RMsUYxl6KhQTWW_b0,631
29
29
  datacosmos/stac/item/models/catalog_search_parameters.py,sha256=3HrUm37VezujwuCR45jhMryS5m1FGc1XmX8-fdTy4jU,4870
@@ -31,10 +31,12 @@ datacosmos/stac/item/models/datacosmos_item.py,sha256=AImz0GRxrpZfIETdzzNfaKX35w
31
31
  datacosmos/stac/item/models/eo_band.py,sha256=YC3Scn_wFhIo51pIVcJeuJienF7JGWoEv39JngDM6rI,309
32
32
  datacosmos/stac/item/models/item_update.py,sha256=_CpjQn9SsfedfuxlHSiGeptqY4M-p15t9YX__mBRueI,2088
33
33
  datacosmos/stac/item/models/raster_band.py,sha256=CoEVs-YyPE5Fse0He9DdOs4dGZpzfCsCuVzOcdXa_UM,354
34
- datacosmos/uploader/__init__.py,sha256=ZtfCVJ_pWKKh2F1r_NArnbG3_JtpcEiXcA_tmSwSKmQ,128
35
- datacosmos/uploader/datacosmos_uploader.py,sha256=FDBUSZ9mpieXi3dms3RM17aIrnRuAKd0rFkD7Jwl1pY,4195
36
- datacosmos/uploader/dataclasses/__init__.py,sha256=IjcyA8Vod-z1_Gi1FMZhK58Owman0foL25Hs0YtkYYs,43
37
- datacosmos/uploader/dataclasses/upload_path.py,sha256=5QadynHxkJrnOk1lyPtLyiVAHdzBshEuhjA9hwVF0NI,1903
34
+ datacosmos/stac/storage/__init__.py,sha256=hivfSpOaoSwCAymgU0rTgvSk9LSPAn1cPLQQ9fLmFX0,151
35
+ datacosmos/stac/storage/storage_base.py,sha256=5ioMKbEltPEWr4dkhZQiUhdBFEhe7ajIYUd9z3K8elU,1483
36
+ datacosmos/stac/storage/storage_client.py,sha256=GeWJoa8ALqelZHvmnop_sSuyU7ntFNFXMFQfplIo0kU,1145
37
+ datacosmos/stac/storage/uploader.py,sha256=5W4Wcx2yzdkU9sg93jnwYP0TiZcuxQXB9owfjL2NsBg,3630
38
+ datacosmos/stac/storage/dataclasses/__init__.py,sha256=IjcyA8Vod-z1_Gi1FMZhK58Owman0foL25Hs0YtkYYs,43
39
+ datacosmos/stac/storage/dataclasses/upload_path.py,sha256=2Nvk51j4s6Cyse9y9sKmN3rQLV7oIMNtokpnt4_qTaw,1895
38
40
  datacosmos/utils/__init__.py,sha256=XQbAnoqJrPpnSpEzAbjh84yqYWw8cBM8mNp8ynTG-54,50
39
41
  datacosmos/utils/url.py,sha256=iQwZr6mYRoePqUZg-k3KQSV9o2wju5ZuCa5WS_GyJo4,2114
40
42
  datacosmos/utils/http_response/__init__.py,sha256=BvOWwC5coYqq_kFn8gIw5m54TLpdfJKlW9vgRkfhXiA,33
@@ -42,8 +44,8 @@ datacosmos/utils/http_response/check_api_response.py,sha256=dKWW01jn2_lWV0xpOBAB
42
44
  datacosmos/utils/http_response/models/__init__.py,sha256=Wj8YT6dqw7rAz_rctllxo5Or_vv8DwopvQvBzwCTvpw,45
43
45
  datacosmos/utils/http_response/models/datacosmos_error.py,sha256=Uqi2uM98nJPeCbM7zngV6vHSk97jEAb_nkdDEeUjiQM,740
44
46
  datacosmos/utils/http_response/models/datacosmos_response.py,sha256=oV4n-sue7K1wwiIQeHpxdNU8vxeqF3okVPE2rydw5W0,336
45
- datacosmos-0.0.8.dist-info/licenses/LICENSE.md,sha256=vpbRI-UUbZVQfr3VG_CXt9HpRnL1b5kt8uTVbirxeyI,1486
46
- datacosmos-0.0.8.dist-info/METADATA,sha256=tK8nQIG9_5zJ-gCQPQB7S2q6z7HabdgYdpPLfkinP9g,896
47
- datacosmos-0.0.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
48
- datacosmos-0.0.8.dist-info/top_level.txt,sha256=ueobs5CNeyDbPMgXPcVV0d0yNdm8CvGtDT3CaksRVtA,11
49
- datacosmos-0.0.8.dist-info/RECORD,,
47
+ datacosmos-0.0.10.dist-info/licenses/LICENSE.md,sha256=vpbRI-UUbZVQfr3VG_CXt9HpRnL1b5kt8uTVbirxeyI,1486
48
+ datacosmos-0.0.10.dist-info/METADATA,sha256=9LtN2v3bq397TTrDhL4MZ-euAt6S8YixPT6nXqy2UZs,897
49
+ datacosmos-0.0.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
50
+ datacosmos-0.0.10.dist-info/top_level.txt,sha256=ueobs5CNeyDbPMgXPcVV0d0yNdm8CvGtDT3CaksRVtA,11
51
+ datacosmos-0.0.10.dist-info/RECORD,,
@@ -1 +0,0 @@
1
- """Uploader package for interacting with the Uploader API, providing upload functionalities to the datacosmos cloud storage."""
@@ -1,101 +0,0 @@
1
- """Module for uploading files to Datacosmos cloud storage and registering STAC items."""
2
-
3
- from concurrent.futures import ThreadPoolExecutor
4
- from pathlib import Path
5
-
6
- from pydantic import TypeAdapter
7
-
8
- from datacosmos.datacosmos_client import DatacosmosClient
9
- from datacosmos.stac.item.item_client import ItemClient
10
- from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
11
- from datacosmos.uploader.dataclasses.upload_path import UploadPath
12
-
13
-
14
- class DatacosmosUploader:
15
- """Handles uploading files to Datacosmos storage and registering STAC items."""
16
-
17
- def __init__(self, client: DatacosmosClient):
18
- """Initialize the uploader with DatacosmosClient."""
19
- self.environment = client.config.environment
20
-
21
- self.datacosmos_client = client
22
- self.item_client = ItemClient(client)
23
- self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
24
-
25
- def upload_and_register_item(self, item_json_file_path: str) -> None:
26
- """Uploads files to Datacosmos storage and registers a STAC item.
27
-
28
- Args:
29
- item_json_file_path (str): Path to the STAC item JSON file.
30
- """
31
- item = self._load_item(item_json_file_path)
32
- collection_id, item_id = item.collection, item.id
33
- dirname = str(Path(item_json_file_path).parent / Path(item_json_file_path).stem)
34
-
35
- self._delete_existing_item(collection_id, item_id)
36
- upload_path = self._get_upload_path(item)
37
- self.upload_from_folder(dirname, upload_path)
38
-
39
- self._update_item_assets(item)
40
-
41
- self.item_client.create_item(collection_id, item)
42
-
43
- def upload_file(self, src: str, dst: str) -> None:
44
- """Uploads a single file to the specified destination path."""
45
- url = self.base_url.with_suffix(dst)
46
-
47
- with open(src, "rb") as f:
48
- response = self.datacosmos_client.put(url, data=f)
49
- response.raise_for_status()
50
-
51
- def upload_from_folder(self, src: str, dst: UploadPath, workers: int = 4) -> None:
52
- """Uploads all files from a folder to the destination path in parallel."""
53
- if Path(dst.path).is_file():
54
- raise ValueError(f"Destination path should not be a file path {dst}")
55
-
56
- if Path(src).is_file():
57
- raise ValueError(f"Source path should not be a file path {src}")
58
-
59
- with ThreadPoolExecutor(max_workers=workers) as executor:
60
- futures = []
61
- for file in Path(src).rglob("*"):
62
- if file.is_file():
63
- dst = UploadPath(
64
- mission=dst.mission,
65
- level=dst.level,
66
- day=dst.day,
67
- month=dst.month,
68
- year=dst.year,
69
- id=dst.id,
70
- path=str(file.relative_to(src)),
71
- )
72
- futures.append(executor.submit(self.upload_file, str(file), dst))
73
- for future in futures:
74
- future.result()
75
-
76
- @staticmethod
77
- def _load_item(item_json_file_path: str) -> DatacosmosItem:
78
- """Loads and validates the STAC item from a JSON file."""
79
- with open(item_json_file_path, "rb") as file:
80
- data = file.read().decode("utf-8")
81
- return TypeAdapter(DatacosmosItem).validate_json(data)
82
-
83
- def _delete_existing_item(self, collection_id: str, item_id: str) -> None:
84
- """Deletes an existing item if it already exists."""
85
- try:
86
- self.item_client.delete_item(item_id, collection_id)
87
- except Exception: # nosec
88
- pass # Ignore if item doesn't exist
89
-
90
- def _get_upload_path(self, item: DatacosmosItem, mission_name: str = "") -> str:
91
- """Constructs the storage upload path based on the item and mission name."""
92
- return UploadPath.from_item_path(item, mission_name, "")
93
-
94
- def _update_item_assets(self, item: DatacosmosItem) -> None:
95
- """Updates the item's assets with uploaded file URLs."""
96
- for asset in item.assets.values():
97
- try:
98
- url = self.base_url
99
- asset.href = url.with_base(asset.href) # type: ignore
100
- except ValueError:
101
- pass