datacosmos 0.0.8__tar.gz → 0.0.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of datacosmos might be problematic. Click here for more details.

Files changed (59) hide show
  1. {datacosmos-0.0.8 → datacosmos-0.0.10}/PKG-INFO +1 -1
  2. {datacosmos-0.0.8 → datacosmos-0.0.10}/README.md +66 -34
  3. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/config/config.py +31 -2
  4. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/enums/processing_level.py +8 -7
  5. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/item_client.py +43 -4
  6. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/stac_client.py +5 -2
  7. datacosmos-0.0.10/datacosmos/stac/storage/__init__.py +5 -0
  8. {datacosmos-0.0.8/datacosmos/uploader → datacosmos-0.0.10/datacosmos/stac/storage}/dataclasses/upload_path.py +1 -1
  9. datacosmos-0.0.10/datacosmos/stac/storage/storage_base.py +40 -0
  10. datacosmos-0.0.10/datacosmos/stac/storage/storage_client.py +31 -0
  11. datacosmos-0.0.10/datacosmos/stac/storage/uploader.py +98 -0
  12. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos.egg-info/PKG-INFO +1 -1
  13. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos.egg-info/SOURCES.txt +6 -4
  14. {datacosmos-0.0.8 → datacosmos-0.0.10}/pyproject.toml +1 -1
  15. datacosmos-0.0.8/datacosmos/uploader/__init__.py +0 -1
  16. datacosmos-0.0.8/datacosmos/uploader/datacosmos_uploader.py +0 -101
  17. {datacosmos-0.0.8 → datacosmos-0.0.10}/LICENSE.md +0 -0
  18. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/__init__.py +0 -0
  19. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/config/__init__.py +0 -0
  20. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/config/models/__init__.py +0 -0
  21. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/config/models/authentication_config.py +0 -0
  22. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/config/models/local_user_account_authentication_config.py +0 -0
  23. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/config/models/m2m_authentication_config.py +0 -0
  24. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/config/models/no_authentication_config.py +0 -0
  25. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/config/models/url.py +0 -0
  26. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/datacosmos_client.py +0 -0
  27. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/exceptions/__init__.py +0 -0
  28. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/exceptions/datacosmos_exception.py +0 -0
  29. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/__init__.py +0 -0
  30. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/collection/__init__.py +0 -0
  31. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/collection/collection_client.py +0 -0
  32. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/collection/models/__init__.py +0 -0
  33. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/collection/models/collection_update.py +0 -0
  34. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/constants/__init__.py +0 -0
  35. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/constants/satellite_name_mapping.py +0 -0
  36. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/enums/__init__.py +0 -0
  37. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/enums/product_type.py +0 -0
  38. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/enums/season.py +0 -0
  39. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/__init__.py +0 -0
  40. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/models/__init__.py +0 -0
  41. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/models/asset.py +0 -0
  42. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/models/catalog_search_parameters.py +0 -0
  43. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/models/datacosmos_item.py +0 -0
  44. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/models/eo_band.py +0 -0
  45. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/models/item_update.py +0 -0
  46. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/stac/item/models/raster_band.py +0 -0
  47. {datacosmos-0.0.8/datacosmos/uploader → datacosmos-0.0.10/datacosmos/stac/storage}/dataclasses/__init__.py +0 -0
  48. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/utils/__init__.py +0 -0
  49. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/utils/http_response/__init__.py +0 -0
  50. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/utils/http_response/check_api_response.py +0 -0
  51. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/utils/http_response/models/__init__.py +0 -0
  52. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/utils/http_response/models/datacosmos_error.py +0 -0
  53. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/utils/http_response/models/datacosmos_response.py +0 -0
  54. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos/utils/url.py +0 -0
  55. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos.egg-info/dependency_links.txt +0 -0
  56. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos.egg-info/requires.txt +0 -0
  57. {datacosmos-0.0.8 → datacosmos-0.0.10}/datacosmos.egg-info/top_level.txt +0 -0
  58. {datacosmos-0.0.8 → datacosmos-0.0.10}/setup.cfg +0 -0
  59. {datacosmos-0.0.8 → datacosmos-0.0.10}/tests/test_pass.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datacosmos
3
- Version: 0.0.8
3
+ Version: 0.0.10
4
4
  Summary: A library for interacting with DataCosmos from Python code
5
5
  Author-email: Open Cosmos <support@open-cosmos.com>
6
6
  Classifier: Programming Language :: Python :: 3
@@ -83,8 +83,10 @@ client = DatacosmosClient(config=config)
83
83
  | `datacosmos_cloud_storage.host` | `app.open-cosmos.com` | YAML / ENV |
84
84
  | `datacosmos_cloud_storage.port` | `443` | YAML / ENV |
85
85
  | `datacosmos_cloud_storage.path` | `/api/data/v0/storage` | YAML / ENV |
86
- | `mission_id` | `0` | YAML / ENV |
87
- | `environment` | `test` | YAML / ENV |
86
+ | `datacosmos_public_cloud_storage.protocol` | `https` | YAML / ENV |
87
+ | `datacosmos_public_cloud_storage.host` | `app.open-cosmos.com` | YAML / ENV |
88
+ | `datacosmos_public_cloud_storage.port` | `443` | YAML / ENV |
89
+ | `datacosmos_public_cloud_storage.path` | `/api/data/v0/storage` | YAML / ENV |
88
90
 
89
91
  ## STAC Client
90
92
 
@@ -163,7 +165,41 @@ stac_item.add_asset(
163
165
  )
164
166
  )
165
167
 
166
- stac_client.create_item(collection_id="example-collection", item=stac_item)
168
+ # this will raise an error if the stac item already exists. Use the add_item method in that case
169
+ stac_client.create_item(item=stac_item)
170
+ ```
171
+
172
+ #### 4. **Add a STAC Item**
173
+ ```python
174
+ from pystac import Item, Asset
175
+ from datetime import datetime
176
+
177
+ from datacosmos.datacosmos_client import DatacosmosClient
178
+ from datacosmos.stac.stac_client import STACClient
179
+
180
+ client = DatacosmosClient()
181
+ stac_client = STACClient(client)
182
+
183
+ stac_item = Item(
184
+ id="new-item",
185
+ geometry={"type": "Point", "coordinates": [102.0, 0.5]},
186
+ bbox=[101.0, 0.0, 103.0, 1.0],
187
+ datetime=datetime.utcnow(),
188
+ properties={"datetime": datetime.utcnow(), "processing:level": "example-processing-level"},
189
+ collection="example-collection"
190
+ )
191
+
192
+ stac_item.add_asset(
193
+ "image",
194
+ Asset(
195
+ href="https://example.com/sample-image.tiff",
196
+ media_type="image/tiff",
197
+ roles=["data"],
198
+ title="Sample Image"
199
+ )
200
+ )
201
+
202
+ stac_client.add_item(item=stac_item)
167
203
  ```
168
204
 
169
205
  #### 4. **Update an Existing STAC Item**
@@ -298,48 +334,44 @@ stac_client.delete_collection("test-collection")
298
334
 
299
335
  ## Uploading Files and Registering STAC Items
300
336
 
301
- You can use the `DatacosmosUploader` class to upload files to the DataCosmos cloud storage and register a STAC item. The `upload_and_register_item` method will take care of both uploading files and creating the STAC item.
337
+ You can use the `STACClient` class to upload files to the DataCosmos cloud storage and register a STAC item.
302
338
 
303
- ### **Upload Files and Register STAC Item**
304
-
305
- 1. Make sure you have a directory with the same name as your STAC item JSON file (this directory should contain the files you want to upload).
306
- 2. Call the `upload_and_register_item` method, providing the path to the STAC item JSON file.
339
+ ### **Upload and add STAC Item**
307
340
 
308
341
  ```python
309
- from datacosmos.datacosmos_client import DatacosmosClient
310
- from datacosmos.uploader.datacosmos_uploader import DatacosmosUploader
342
+ from pystac import Item, Asset
311
343
 
312
- # Initialize the client with the configuration
313
- client = DatacosmosClient(config=config)
344
+ from datacosmos.datacosmos_client import DatacosmosClient
345
+ from datacosmos.stac.stac_client import STACClient
314
346
 
315
- # Create the uploader instance
316
- uploader = DatacosmosUploader(client)
347
+ client = DatacosmosClient()
317
348
 
318
- # Path to your STAC item JSON file
319
- item_json_file_path = "/home/peres/repos/datacosmos-sdk/MENUT_L1A_000001943_20250304134812_20250304134821_49435814.json"
349
+ stac_client = STACClient(client)
320
350
 
321
- # Upload the item and its assets, and register it in the STAC API
322
- uploader.upload_and_register_item(item_json_file_path)
323
- ```
351
+ stac_item = Item(
352
+ id="new-item",
353
+ geometry={"type": "Point", "coordinates": [102.0, 0.5]},
354
+ bbox=[101.0, 0.0, 103.0, 1.0],
355
+ datetime=datetime.utcnow(),
356
+ properties={"datetime": datetime.utcnow(), "processing:level": "example-processing-level"},
357
+ collection="example-collection"
358
+ )
324
359
 
325
- ### **Folder Structure**
360
+ stac_item.add_asset(
361
+ "image",
362
+ Asset(
363
+ href="https://example.com/sample-image.tiff",
364
+ media_type="image/tiff",
365
+ roles=["data"],
366
+ title="Sample Image"
367
+ )
368
+ )
326
369
 
327
- For the `upload_and_register_item` method to work correctly, ensure that the directory structure matches the name of the STAC item JSON file. For example:
370
+ assets_path = "path/to/assets"
328
371
 
372
+ # Upload the item and its assets, and register it in the STAC API
373
+ stac_client.upload_item(stac_item, assets_path)
329
374
  ```
330
- /home/peres/repos/datacosmos-sdk/MENUT_L1A_000001943_20250304134812_20250304134821_49435814.json
331
- /home/peres/repos/datacosmos-sdk/MENUT_L1A_000001943_20250304134812_20250304134821_49435814/
332
- ├── asset1.tiff
333
- ├── asset2.tiff
334
- └── ...
335
- ```
336
-
337
- The folder `MENUT_L1A_000001943_20250304134812_20250304134821_49435814` should contain the assets (files) for upload.
338
-
339
- The `upload_and_register_item` method will:
340
- 1. Delete any existing item with the same ID (if it exists).
341
- 2. Upload the assets in the folder to DataCosmos cloud storage.
342
- 3. Register the item in the STAC API.
343
375
 
344
376
  ## Error Handling
345
377
 
@@ -29,6 +29,7 @@ class Config(BaseSettings):
29
29
  authentication: Optional[AuthenticationConfig] = None
30
30
  stac: Optional[URL] = None
31
31
  datacosmos_cloud_storage: Optional[URL] = None
32
+ datacosmos_public_cloud_storage: Optional[URL] = None
32
33
 
33
34
  DEFAULT_AUTH_TYPE: ClassVar[str] = "m2m"
34
35
  DEFAULT_AUTH_TOKEN_URL: ClassVar[str] = "https://login.open-cosmos.com/oauth/token"
@@ -86,12 +87,18 @@ class Config(BaseSettings):
86
87
  path=os.getenv("DC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
87
88
  )
88
89
 
90
+ datacosmos_public_cloud_storage_config = URL(
91
+ protocol=os.getenv("DC_PUBLIC_CLOUD_STORAGE_PROTOCOL", "https"),
92
+ host=os.getenv("DC_PUBLIC_CLOUD_STORAGE_HOST", "app.open-cosmos.com"),
93
+ port=int(os.getenv("DC_PUBLIC_CLOUD_STORAGE_PORT", "443")),
94
+ path=os.getenv("DC_PUBLIC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
95
+ )
96
+
89
97
  return cls(
90
98
  authentication=authentication_config,
91
99
  stac=stac_config,
92
100
  datacosmos_cloud_storage=datacosmos_cloud_storage_config,
93
- mission_id=int(os.getenv("MISSION_ID", "0")),
94
- environment=os.getenv("ENVIRONMENT", "test"),
101
+ datacosmos_public_cloud_storage=datacosmos_public_cloud_storage_config,
95
102
  )
96
103
 
97
104
  @field_validator("authentication", mode="after")
@@ -191,3 +198,25 @@ class Config(BaseSettings):
191
198
  path="/api/data/v0/storage",
192
199
  )
193
200
  return datacosmos_cloud_storage_config
201
+
202
+ @field_validator("datacosmos_public_cloud_storage", mode="before")
203
+ @classmethod
204
+ def validate_datacosmos_public_cloud_storage(
205
+ cls, datacosmos_public_cloud_storage_config: Optional[URL]
206
+ ) -> URL:
207
+ """Ensure datacosmos cloud storage configuration has a default if not explicitly set.
208
+
209
+ Args:
210
+ datacosmos_public_cloud_storage_config (Optional[URL]): The datacosmos public cloud storage config to validate.
211
+
212
+ Returns:
213
+ URL: The validated datacosmos public cloud storage configuration.
214
+ """
215
+ if datacosmos_public_cloud_storage_config is None:
216
+ return URL(
217
+ protocol="https",
218
+ host="app.open-cosmos.com",
219
+ port=443,
220
+ path="/api/data/v0/storage",
221
+ )
222
+ return datacosmos_public_cloud_storage_config
@@ -6,10 +6,11 @@ from enum import Enum
6
6
  class ProcessingLevel(Enum):
7
7
  """Enum class for the processing levels of the data."""
8
8
 
9
- L0 = "L0"
10
- L1A = "L1A"
11
- L2A = "L2A"
12
- L1B = "L1B"
13
- L1C = "L1C"
14
- L1D = "L1D"
15
- L3 = "L3"
9
+ RAW = "RAW"
10
+ L0 = "l0"
11
+ L1A = "l1A"
12
+ L2A = "l2A"
13
+ L1B = "l1B"
14
+ L1C = "l1C"
15
+ L1D = "l1D"
16
+ L3 = "l3"
@@ -62,21 +62,60 @@ class ItemClient:
62
62
  body = body | {"collections": parameters.collections}
63
63
  return self._paginate_items(url, body)
64
64
 
65
- def create_item(self, collection_id: str, item: Item | DatacosmosItem) -> None:
66
- """Create a new STAC item in a specified collection.
65
+ def create_item(self, item: Item | DatacosmosItem) -> None:
66
+ """Create a new STAC item in its own collection.
67
+
68
+ The collection ID is inferred from the item.
67
69
 
68
70
  Args:
69
- collection_id (str): The ID of the collection where the item will be created.
70
- item (Item): The STAC Item to be created.
71
+ item (Item | DatacosmosItem): The STAC item to be created.
71
72
 
72
73
  Raises:
74
+ ValueError: If the item has no collection set.
73
75
  RequestError: If the API returns an error response.
74
76
  """
77
+ if isinstance(item, Item):
78
+ collection_id = item.collection_id or (
79
+ item.get_collection().id if item.get_collection() else None
80
+ )
81
+ else:
82
+ collection_id = item.collection
83
+
84
+ if not collection_id:
85
+ raise ValueError("Cannot create item: no collection_id found on item")
86
+
75
87
  url = self.base_url.with_suffix(f"/collections/{collection_id}/items")
76
88
  item_json: dict = item.to_dict()
77
89
  response = self.client.post(url, json=item_json)
78
90
  check_api_response(response)
79
91
 
92
+ def add_item(self, item: Item | DatacosmosItem) -> None:
93
+ """Adds item to catalog.
94
+
95
+ The collection ID is inferred from the item.
96
+
97
+ Args:
98
+ item (Item | DatacosmosItem): The STAC item to be created.
99
+
100
+ Raises:
101
+ ValueError: If the item has no collection set.
102
+ RequestError: If the API returns an error response.
103
+ """
104
+ if isinstance(item, Item):
105
+ collection_id = item.collection_id or (
106
+ item.get_collection().id if item.get_collection() else None
107
+ )
108
+ else:
109
+ collection_id = item.collection
110
+
111
+ if not collection_id:
112
+ raise ValueError("Cannot create item: no collection_id found on item")
113
+
114
+ url = self.base_url.with_suffix(f"/collections/{collection_id}/items/{item.id}")
115
+ item_json: dict = item.to_dict()
116
+ response = self.client.put(url, json=item_json)
117
+ check_api_response(response)
118
+
80
119
  def update_item(
81
120
  self, item_id: str, collection_id: str, update_data: ItemUpdate
82
121
  ) -> None:
@@ -2,11 +2,14 @@
2
2
 
3
3
  from datacosmos.stac.collection.collection_client import CollectionClient
4
4
  from datacosmos.stac.item.item_client import ItemClient
5
+ from datacosmos.stac.storage.storage_client import StorageClient
5
6
 
6
7
 
7
- class STACClient(ItemClient, CollectionClient):
8
+ class STACClient(ItemClient, CollectionClient, StorageClient):
8
9
  """Unified interface for STAC API, combining Item & Collection operations."""
9
10
 
10
11
  def __init__(self, client):
11
12
  """Initialize the STACClient with a DatacosmosClient."""
12
- super().__init__(client)
13
+ ItemClient.__init__(self, client)
14
+ CollectionClient.__init__(self, client)
15
+ StorageClient.__init__(self, client)
@@ -0,0 +1,5 @@
1
+ """Facade module for all storage-related operations."""
2
+
3
+ from datacosmos.stac.storage.storage_client import StorageClient
4
+
5
+ __all__ = ["StorageClient"]
@@ -37,7 +37,7 @@ class UploadPath:
37
37
  dt = datetime.strptime(item.properties["datetime"], "%Y-%m-%dT%H:%M:%SZ")
38
38
  path = UploadPath(
39
39
  mission=mission,
40
- level=ProcessingLevel(item.properties["processing:level"].upper()),
40
+ level=ProcessingLevel(item.properties["processing:level"]),
41
41
  day=dt.day,
42
42
  month=dt.month,
43
43
  year=dt.year,
@@ -0,0 +1,40 @@
1
+ """Base class providing common storage helpers (threading, MIME guess, futures)."""
2
+
3
+ import mimetypes
4
+ from concurrent.futures import ThreadPoolExecutor, wait
5
+
6
+ from datacosmos.datacosmos_client import DatacosmosClient
7
+
8
+
9
+ class StorageBase:
10
+ """Base class providing common storage helpers (threading, MIME guess, futures)."""
11
+
12
+ def __init__(self, client: DatacosmosClient):
13
+ """Base class providing common storage helpers (threading, MIME guess, futures)."""
14
+ self.client = client
15
+ self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
16
+
17
+ def _guess_mime(self, src: str) -> str:
18
+ mime, _ = mimetypes.guess_type(src)
19
+ return mime or "application/octet-stream"
20
+
21
+ def _run_in_threads(self, fn, fn_args, max_workers: int, timeout: float):
22
+ """Run the callable `fn(*args)` over the iterable of jobs in parallel threads.
23
+
24
+ `jobs` should be a list of tuples, each tuple unpacked as fn(*args).
25
+ """
26
+ futures = []
27
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
28
+ for args in fn_args:
29
+ futures.append(executor.submit(fn, *args))
30
+ done, not_done = wait(futures, timeout=timeout)
31
+ errors = []
32
+ for future in done:
33
+ try:
34
+ future.result()
35
+ except Exception as e:
36
+ errors.append(e)
37
+ for future in not_done:
38
+ future.cancel()
39
+ if errors:
40
+ raise errors[0]
@@ -0,0 +1,31 @@
1
+ """Generic StorageClient for all storage operations (upload, download, etc.)."""
2
+
3
+ from datacosmos.datacosmos_client import DatacosmosClient
4
+ from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
5
+ from datacosmos.stac.storage.uploader import Uploader
6
+
7
+
8
+ class StorageClient:
9
+ """Generic StorageClient for all storage operations (upload, download, etc.)."""
10
+
11
+ def __init__(self, client: DatacosmosClient):
12
+ """Generic StorageClient for all storage operations (upload, download, etc.)."""
13
+ self.client = client
14
+ self.uploader = Uploader(client)
15
+
16
+ def upload_item(
17
+ self,
18
+ item: DatacosmosItem,
19
+ assets_path: str | None = None,
20
+ included_assets: list[str] | bool = True,
21
+ max_workers: int = 4,
22
+ time_out: float = 60 * 60 * 1,
23
+ ) -> DatacosmosItem:
24
+ """Proxy to Uploader.upload_item, without needing to pass client each call."""
25
+ return self.uploader.upload_item(
26
+ item=item,
27
+ assets_path=assets_path,
28
+ included_assets=included_assets,
29
+ max_workers=max_workers,
30
+ time_out=time_out,
31
+ )
@@ -0,0 +1,98 @@
1
+ """Handles uploading files to Datacosmos storage and registering STAC items."""
2
+
3
+ from pathlib import Path
4
+
5
+ from pydantic import TypeAdapter
6
+
7
+ from datacosmos.datacosmos_client import DatacosmosClient
8
+ from datacosmos.stac.item.item_client import ItemClient
9
+ from datacosmos.stac.item.models.asset import Asset
10
+ from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
11
+ from datacosmos.stac.storage.dataclasses.upload_path import UploadPath
12
+ from datacosmos.stac.storage.storage_base import StorageBase
13
+
14
+
15
+ class Uploader(StorageBase):
16
+ """Handles uploading files to Datacosmos storage and registering STAC items."""
17
+
18
+ def __init__(self, client: DatacosmosClient):
19
+ """Handles uploading files to Datacosmos storage and registering STAC items."""
20
+ super().__init__(client)
21
+ self.item_client = ItemClient(client)
22
+
23
+ def upload_item(
24
+ self,
25
+ item: DatacosmosItem,
26
+ assets_path: str | None = None,
27
+ included_assets: list[str] | bool = True,
28
+ max_workers: int = 4,
29
+ time_out: float = 60 * 60 * 1,
30
+ ) -> DatacosmosItem:
31
+ """Upload a STAC item and its assets to Datacosmos."""
32
+ if not assets_path and not isinstance(item, str):
33
+ raise ValueError(
34
+ "assets_path must be provided if item is not the path to an item file."
35
+ )
36
+
37
+ if isinstance(item, str):
38
+ item_filename = item
39
+ item = self._load_item(item_filename)
40
+ if not assets_path:
41
+ assets_path = str(Path(item_filename).parent)
42
+
43
+ assets_path = assets_path or str(Path.cwd())
44
+
45
+ upload_assets = (
46
+ included_assets
47
+ if isinstance(included_assets, list)
48
+ else item.assets.keys()
49
+ if included_assets is True
50
+ else []
51
+ )
52
+
53
+ jobs = [(item, asset_key, assets_path) for asset_key in upload_assets]
54
+
55
+ self._run_in_threads(self._upload_asset, jobs, max_workers, time_out)
56
+
57
+ self.item_client.add_item(item)
58
+
59
+ return item
60
+
61
+ def upload_from_file(
62
+ self, src: str, dst: str, mime_type: str | None = None
63
+ ) -> None:
64
+ """Uploads a single file to the specified destination path."""
65
+ url = self.base_url.with_suffix(dst)
66
+ mime = mime_type or self._guess_mime(src)
67
+ headers = {"Content-Type": mime}
68
+ with open(src, "rb") as f:
69
+ response = self.client.put(url, data=f, headers=headers)
70
+ response.raise_for_status()
71
+
72
+ @staticmethod
73
+ def _load_item(item_json_file_path: str) -> DatacosmosItem:
74
+ with open(item_json_file_path, "rb") as file:
75
+ data = file.read().decode("utf-8")
76
+ return TypeAdapter(DatacosmosItem).validate_json(data)
77
+
78
+ def _upload_asset(
79
+ self, item: DatacosmosItem, asset_key: str, assets_path: str
80
+ ) -> None:
81
+ asset = item.assets[asset_key]
82
+ upload_path = UploadPath.from_item_path(item, "", Path(asset.href).name)
83
+ local_src = Path(assets_path) / asset.href
84
+ if local_src.exists():
85
+ src = str(local_src)
86
+ asset.href = f"file:///{upload_path}"
87
+ else:
88
+ src = str(Path(assets_path) / Path(asset.href).name)
89
+ self._update_asset_href(asset)
90
+ self.upload_from_file(src, str(upload_path), mime_type=asset.type)
91
+
92
+ def _update_asset_href(self, asset: Asset) -> None:
93
+ try:
94
+ url = self.client.config.datacosmos_public_cloud_storage.as_domain_url()
95
+ new_href = url.with_base(asset.href) # type: ignore
96
+ asset.href = str(new_href)
97
+ except ValueError:
98
+ pass
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: datacosmos
3
- Version: 0.0.8
3
+ Version: 0.0.10
4
4
  Summary: A library for interacting with DataCosmos from Python code
5
5
  Author-email: Open Cosmos <support@open-cosmos.com>
6
6
  Classifier: Programming Language :: Python :: 3
@@ -39,10 +39,12 @@ datacosmos/stac/item/models/datacosmos_item.py
39
39
  datacosmos/stac/item/models/eo_band.py
40
40
  datacosmos/stac/item/models/item_update.py
41
41
  datacosmos/stac/item/models/raster_band.py
42
- datacosmos/uploader/__init__.py
43
- datacosmos/uploader/datacosmos_uploader.py
44
- datacosmos/uploader/dataclasses/__init__.py
45
- datacosmos/uploader/dataclasses/upload_path.py
42
+ datacosmos/stac/storage/__init__.py
43
+ datacosmos/stac/storage/storage_base.py
44
+ datacosmos/stac/storage/storage_client.py
45
+ datacosmos/stac/storage/uploader.py
46
+ datacosmos/stac/storage/dataclasses/__init__.py
47
+ datacosmos/stac/storage/dataclasses/upload_path.py
46
48
  datacosmos/utils/__init__.py
47
49
  datacosmos/utils/url.py
48
50
  datacosmos/utils/http_response/__init__.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "datacosmos"
7
- version = "0.0.8"
7
+ version = "0.0.10"
8
8
  authors = [
9
9
  { name="Open Cosmos", email="support@open-cosmos.com" },
10
10
  ]
@@ -1 +0,0 @@
1
- """Uploader package for interacting with the Uploader API, providing upload functionalities to the datacosmos cloud storage."""
@@ -1,101 +0,0 @@
1
- """Module for uploading files to Datacosmos cloud storage and registering STAC items."""
2
-
3
- from concurrent.futures import ThreadPoolExecutor
4
- from pathlib import Path
5
-
6
- from pydantic import TypeAdapter
7
-
8
- from datacosmos.datacosmos_client import DatacosmosClient
9
- from datacosmos.stac.item.item_client import ItemClient
10
- from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
11
- from datacosmos.uploader.dataclasses.upload_path import UploadPath
12
-
13
-
14
- class DatacosmosUploader:
15
- """Handles uploading files to Datacosmos storage and registering STAC items."""
16
-
17
- def __init__(self, client: DatacosmosClient):
18
- """Initialize the uploader with DatacosmosClient."""
19
- self.environment = client.config.environment
20
-
21
- self.datacosmos_client = client
22
- self.item_client = ItemClient(client)
23
- self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
24
-
25
- def upload_and_register_item(self, item_json_file_path: str) -> None:
26
- """Uploads files to Datacosmos storage and registers a STAC item.
27
-
28
- Args:
29
- item_json_file_path (str): Path to the STAC item JSON file.
30
- """
31
- item = self._load_item(item_json_file_path)
32
- collection_id, item_id = item.collection, item.id
33
- dirname = str(Path(item_json_file_path).parent / Path(item_json_file_path).stem)
34
-
35
- self._delete_existing_item(collection_id, item_id)
36
- upload_path = self._get_upload_path(item)
37
- self.upload_from_folder(dirname, upload_path)
38
-
39
- self._update_item_assets(item)
40
-
41
- self.item_client.create_item(collection_id, item)
42
-
43
- def upload_file(self, src: str, dst: str) -> None:
44
- """Uploads a single file to the specified destination path."""
45
- url = self.base_url.with_suffix(dst)
46
-
47
- with open(src, "rb") as f:
48
- response = self.datacosmos_client.put(url, data=f)
49
- response.raise_for_status()
50
-
51
- def upload_from_folder(self, src: str, dst: UploadPath, workers: int = 4) -> None:
52
- """Uploads all files from a folder to the destination path in parallel."""
53
- if Path(dst.path).is_file():
54
- raise ValueError(f"Destination path should not be a file path {dst}")
55
-
56
- if Path(src).is_file():
57
- raise ValueError(f"Source path should not be a file path {src}")
58
-
59
- with ThreadPoolExecutor(max_workers=workers) as executor:
60
- futures = []
61
- for file in Path(src).rglob("*"):
62
- if file.is_file():
63
- dst = UploadPath(
64
- mission=dst.mission,
65
- level=dst.level,
66
- day=dst.day,
67
- month=dst.month,
68
- year=dst.year,
69
- id=dst.id,
70
- path=str(file.relative_to(src)),
71
- )
72
- futures.append(executor.submit(self.upload_file, str(file), dst))
73
- for future in futures:
74
- future.result()
75
-
76
- @staticmethod
77
- def _load_item(item_json_file_path: str) -> DatacosmosItem:
78
- """Loads and validates the STAC item from a JSON file."""
79
- with open(item_json_file_path, "rb") as file:
80
- data = file.read().decode("utf-8")
81
- return TypeAdapter(DatacosmosItem).validate_json(data)
82
-
83
- def _delete_existing_item(self, collection_id: str, item_id: str) -> None:
84
- """Deletes an existing item if it already exists."""
85
- try:
86
- self.item_client.delete_item(item_id, collection_id)
87
- except Exception: # nosec
88
- pass # Ignore if item doesn't exist
89
-
90
- def _get_upload_path(self, item: DatacosmosItem, mission_name: str = "") -> str:
91
- """Constructs the storage upload path based on the item and mission name."""
92
- return UploadPath.from_item_path(item, mission_name, "")
93
-
94
- def _update_item_assets(self, item: DatacosmosItem) -> None:
95
- """Updates the item's assets with uploaded file URLs."""
96
- for asset in item.assets.values():
97
- try:
98
- url = self.base_url
99
- asset.href = url.with_base(asset.href) # type: ignore
100
- except ValueError:
101
- pass
File without changes
File without changes