datacosmos 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of datacosmos might be problematic. Click here for more details.
- config/config.py +56 -28
- datacosmos/stac/enums/__init__.py +1 -0
- datacosmos/stac/enums/level.py +15 -0
- datacosmos/stac/item/item_client.py +2 -1
- datacosmos/stac/item/models/asset.py +23 -0
- datacosmos/stac/item/models/datacosmos_item.py +55 -0
- datacosmos/stac/item/models/eo_band.py +15 -0
- datacosmos/stac/item/models/raster_band.py +17 -0
- datacosmos/uploader/__init__.py +1 -0
- datacosmos/uploader/dataclasses/__init__.py +1 -0
- datacosmos/uploader/dataclasses/upload_path.py +93 -0
- datacosmos/uploader/datacosmos_uploader.py +106 -0
- datacosmos/utils/constants.py +16 -0
- datacosmos/utils/missions.py +27 -0
- datacosmos/utils/url.py +23 -0
- {datacosmos-0.0.2.dist-info → datacosmos-0.0.3.dist-info}/METADATA +3 -2
- {datacosmos-0.0.2.dist-info → datacosmos-0.0.3.dist-info}/RECORD +20 -8
- {datacosmos-0.0.2.dist-info → datacosmos-0.0.3.dist-info}/WHEEL +1 -1
- {datacosmos-0.0.2.dist-info → datacosmos-0.0.3.dist-info/licenses}/LICENSE.md +0 -0
- {datacosmos-0.0.2.dist-info → datacosmos-0.0.3.dist-info}/top_level.txt +0 -0
config/config.py
CHANGED
|
@@ -6,7 +6,7 @@ and supports environment variable-based overrides.
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import os
|
|
9
|
-
from typing import ClassVar, Optional
|
|
9
|
+
from typing import ClassVar, Literal, Optional
|
|
10
10
|
|
|
11
11
|
import yaml
|
|
12
12
|
from pydantic import field_validator
|
|
@@ -27,6 +27,9 @@ class Config(BaseSettings):
|
|
|
27
27
|
|
|
28
28
|
authentication: Optional[M2MAuthenticationConfig] = None
|
|
29
29
|
stac: Optional[URL] = None
|
|
30
|
+
datacosmos_cloud_storage: Optional[URL] = None
|
|
31
|
+
mission_id: int = 0
|
|
32
|
+
environment: Literal["local", "test", "prod"] = "test"
|
|
30
33
|
|
|
31
34
|
DEFAULT_AUTH_TYPE: ClassVar[str] = "m2m"
|
|
32
35
|
DEFAULT_AUTH_TOKEN_URL: ClassVar[str] = "https://login.open-cosmos.com/oauth/token"
|
|
@@ -77,7 +80,20 @@ class Config(BaseSettings):
|
|
|
77
80
|
path=os.getenv("OC_STAC_PATH", "/api/data/v0/stac"),
|
|
78
81
|
)
|
|
79
82
|
|
|
80
|
-
|
|
83
|
+
datacosmos_cloud_storage_config = URL(
|
|
84
|
+
protocol=os.getenv("DC_CLOUD_STORAGE_PROTOCOL", "https"),
|
|
85
|
+
host=os.getenv("DC_CLOUD_STORAGE_HOST", "app.open-cosmos.com"),
|
|
86
|
+
port=int(os.getenv("DC_CLOUD_STORAGE_PORT", "443")),
|
|
87
|
+
path=os.getenv("DC_CLOUD_STORAGE_PATH", "/api/data/v0/storage"),
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
return cls(
|
|
91
|
+
authentication=authentication_config,
|
|
92
|
+
stac=stac_config,
|
|
93
|
+
datacosmos_cloud_storage=datacosmos_cloud_storage_config,
|
|
94
|
+
mission_id=int(os.getenv("MISSION_ID", "0")),
|
|
95
|
+
environment=os.getenv("ENVIRONMENT", "test"),
|
|
96
|
+
)
|
|
81
97
|
|
|
82
98
|
@field_validator("authentication", mode="before")
|
|
83
99
|
@classmethod
|
|
@@ -96,7 +112,7 @@ class Config(BaseSettings):
|
|
|
96
112
|
ValueError: If authentication is missing or required fields are not set.
|
|
97
113
|
"""
|
|
98
114
|
if not auth_data:
|
|
99
|
-
cls.
|
|
115
|
+
return cls.apply_auth_defaults(M2MAuthenticationConfig())
|
|
100
116
|
|
|
101
117
|
auth = cls.parse_auth_config(auth_data)
|
|
102
118
|
auth = cls.apply_auth_defaults(auth)
|
|
@@ -105,34 +121,24 @@ class Config(BaseSettings):
|
|
|
105
121
|
return auth
|
|
106
122
|
|
|
107
123
|
@staticmethod
|
|
108
|
-
def
|
|
109
|
-
"""Raise an error when authentication is missing."""
|
|
110
|
-
raise ValueError(
|
|
111
|
-
"M2M authentication is required. Provide it via:\n"
|
|
112
|
-
"1. Explicit instantiation (Config(authentication=...))\n"
|
|
113
|
-
"2. A YAML config file (config.yaml)\n"
|
|
114
|
-
"3. Environment variables (OC_AUTH_CLIENT_ID, OC_AUTH_CLIENT_SECRET, etc.)"
|
|
115
|
-
)
|
|
116
|
-
|
|
117
|
-
@staticmethod
|
|
118
|
-
def parse_auth_config(auth_data: dict) -> M2MAuthenticationConfig:
|
|
119
|
-
"""Convert dictionary input to M2MAuthenticationConfig object."""
|
|
120
|
-
return (
|
|
121
|
-
M2MAuthenticationConfig(**auth_data)
|
|
122
|
-
if isinstance(auth_data, dict)
|
|
123
|
-
else auth_data
|
|
124
|
-
)
|
|
125
|
-
|
|
126
|
-
@classmethod
|
|
127
|
-
def apply_auth_defaults(
|
|
128
|
-
cls, auth: M2MAuthenticationConfig
|
|
129
|
-
) -> M2MAuthenticationConfig:
|
|
124
|
+
def apply_auth_defaults(auth: M2MAuthenticationConfig) -> M2MAuthenticationConfig:
|
|
130
125
|
"""Apply default authentication values if they are missing."""
|
|
131
|
-
auth.type = auth.type or
|
|
132
|
-
auth.token_url = auth.token_url or
|
|
133
|
-
auth.audience = auth.audience or
|
|
126
|
+
auth.type = auth.type or Config.DEFAULT_AUTH_TYPE
|
|
127
|
+
auth.token_url = auth.token_url or Config.DEFAULT_AUTH_TOKEN_URL
|
|
128
|
+
auth.audience = auth.audience or Config.DEFAULT_AUTH_AUDIENCE
|
|
134
129
|
return auth
|
|
135
130
|
|
|
131
|
+
@classmethod
|
|
132
|
+
def parse_auth_config(cls, auth_data: dict) -> M2MAuthenticationConfig:
|
|
133
|
+
"""Parse authentication config from a dictionary."""
|
|
134
|
+
return M2MAuthenticationConfig(
|
|
135
|
+
type=auth_data.get("type", cls.DEFAULT_AUTH_TYPE),
|
|
136
|
+
token_url=auth_data.get("token_url", cls.DEFAULT_AUTH_TOKEN_URL),
|
|
137
|
+
audience=auth_data.get("audience", cls.DEFAULT_AUTH_AUDIENCE),
|
|
138
|
+
client_id=auth_data.get("client_id"),
|
|
139
|
+
client_secret=auth_data.get("client_secret"),
|
|
140
|
+
)
|
|
141
|
+
|
|
136
142
|
@staticmethod
|
|
137
143
|
def check_required_auth_fields(auth: M2MAuthenticationConfig):
|
|
138
144
|
"""Ensure required fields (client_id, client_secret) are provided."""
|
|
@@ -165,3 +171,25 @@ class Config(BaseSettings):
|
|
|
165
171
|
path="/api/data/v0/stac",
|
|
166
172
|
)
|
|
167
173
|
return stac_config
|
|
174
|
+
|
|
175
|
+
@field_validator("datacosmos_cloud_storage", mode="before")
|
|
176
|
+
@classmethod
|
|
177
|
+
def validate_datacosmos_cloud_storage(
|
|
178
|
+
cls, datacosmos_cloud_storage_config: Optional[URL]
|
|
179
|
+
) -> URL:
|
|
180
|
+
"""Ensure datacosmos cloud storage configuration has a default if not explicitly set.
|
|
181
|
+
|
|
182
|
+
Args:
|
|
183
|
+
datacosmos_cloud_storage_config (Optional[URL]): The datacosmos cloud storage config to validate.
|
|
184
|
+
|
|
185
|
+
Returns:
|
|
186
|
+
URL: The validated datacosmos cloud storage configuration.
|
|
187
|
+
"""
|
|
188
|
+
if datacosmos_cloud_storage_config is None:
|
|
189
|
+
return URL(
|
|
190
|
+
protocol="https",
|
|
191
|
+
host="app.open-cosmos.com",
|
|
192
|
+
port=443,
|
|
193
|
+
path="/api/data/v0/storage",
|
|
194
|
+
)
|
|
195
|
+
return datacosmos_cloud_storage_config
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Enums for STAC."""
|
|
@@ -9,6 +9,7 @@ from pystac import Item
|
|
|
9
9
|
|
|
10
10
|
from datacosmos.datacosmos_client import DatacosmosClient
|
|
11
11
|
from datacosmos.exceptions.datacosmos_exception import DatacosmosException
|
|
12
|
+
from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
|
|
12
13
|
from datacosmos.stac.item.models.item_update import ItemUpdate
|
|
13
14
|
from datacosmos.stac.item.models.search_parameters import SearchParameters
|
|
14
15
|
from datacosmos.utils.http_response.check_api_response import check_api_response
|
|
@@ -71,7 +72,7 @@ class ItemClient:
|
|
|
71
72
|
body = parameters.model_dump(by_alias=True, exclude_none=True)
|
|
72
73
|
return self._paginate_items(url, body)
|
|
73
74
|
|
|
74
|
-
def create_item(self, collection_id: str, item: Item) -> None:
|
|
75
|
+
def create_item(self, collection_id: str, item: Item | DatacosmosItem) -> None:
|
|
75
76
|
"""Create a new STAC item in a specified collection.
|
|
76
77
|
|
|
77
78
|
Args:
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Model representing a datacosmos item asset."""
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
from datacosmos.stac.item.models.eo_band import EoBand
|
|
6
|
+
from datacosmos.stac.item.models.raster_band import RasterBand
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Asset(BaseModel):
|
|
10
|
+
"""Model representing a datacosmos item asset."""
|
|
11
|
+
|
|
12
|
+
href: str
|
|
13
|
+
title: str
|
|
14
|
+
description: str
|
|
15
|
+
type: str
|
|
16
|
+
roles: list[str] | None
|
|
17
|
+
eo_bands: list[EoBand] | None = Field(default=None, alias="eo:bands")
|
|
18
|
+
raster_bands: list[RasterBand] | None = Field(default=None, alias="raster:bands")
|
|
19
|
+
|
|
20
|
+
class Config:
|
|
21
|
+
"""Pydantic configuration."""
|
|
22
|
+
|
|
23
|
+
populate_by_name = True
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Model representing a datacosmos item."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel
|
|
6
|
+
|
|
7
|
+
from datacosmos.stac.enums.level import Level
|
|
8
|
+
from datacosmos.stac.item.models.asset import Asset
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DatacosmosItem(BaseModel):
|
|
12
|
+
"""Model representing a datacosmos item."""
|
|
13
|
+
|
|
14
|
+
id: str
|
|
15
|
+
type: str
|
|
16
|
+
stac_version: str
|
|
17
|
+
stac_extensions: list | None
|
|
18
|
+
geometry: dict
|
|
19
|
+
properties: dict
|
|
20
|
+
links: list
|
|
21
|
+
assets: dict[str, Asset]
|
|
22
|
+
collection: str
|
|
23
|
+
bbox: tuple[float, float, float, float]
|
|
24
|
+
|
|
25
|
+
def get_property(self, key: str) -> str | None:
|
|
26
|
+
"""Get a property value from the Datacosmos item."""
|
|
27
|
+
return self.properties.get(key)
|
|
28
|
+
|
|
29
|
+
def get_asset(self, key: str) -> Asset | None:
|
|
30
|
+
"""Get an asset from the Datacosmos item."""
|
|
31
|
+
return self.assets.get(key)
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def datetime(self) -> datetime:
|
|
35
|
+
"""Get the datetime of the Datacosmos item."""
|
|
36
|
+
return datetime.strptime(self.properties["datetime"], "%Y-%m-%dT%H:%M:%SZ")
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def level(self) -> Level:
|
|
40
|
+
"""Get the processing level of the Datacosmos item."""
|
|
41
|
+
return Level(self.properties["processing:level"].lower())
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def sat_int_designator(self) -> str:
|
|
45
|
+
"""Get the satellite international designator of the Datacosmos item."""
|
|
46
|
+
property = self.get_property("sat:platform_international_designator")
|
|
47
|
+
if property is None:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
"sat:platform_international_designator is missing in STAC item"
|
|
50
|
+
)
|
|
51
|
+
return property
|
|
52
|
+
|
|
53
|
+
def to_dict(self) -> dict:
|
|
54
|
+
"""Converts the DatacosmosItem instance to a dictionary."""
|
|
55
|
+
return self.model_dump()
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Model representing an EO band."""
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class EoBand(BaseModel):
|
|
9
|
+
"""Model representing an EO band."""
|
|
10
|
+
|
|
11
|
+
name: str
|
|
12
|
+
common_name: str
|
|
13
|
+
center_wavelength: float
|
|
14
|
+
full_width_half_max: float
|
|
15
|
+
solar_illumination: Optional[float] = None
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Model representing a raster band."""
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class RasterBand(BaseModel):
|
|
7
|
+
"""Model representing a raster band."""
|
|
8
|
+
|
|
9
|
+
gain: float = Field(alias="scale")
|
|
10
|
+
bias: float = Field(alias="offset")
|
|
11
|
+
nodata: int
|
|
12
|
+
unit: str
|
|
13
|
+
|
|
14
|
+
class Config:
|
|
15
|
+
"""Pydantic configuration."""
|
|
16
|
+
|
|
17
|
+
populate_by_name = True
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Uploader package for interacting with the Uploader API, providing upload functionalities to the datacosmos cloud storage."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Dataclasses for the uploader module."""
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Dataclass for retrieving the upload path of a file."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import structlog
|
|
8
|
+
|
|
9
|
+
from datacosmos.stac.enums.level import Level
|
|
10
|
+
from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
|
|
11
|
+
from datacosmos.utils.missions import get_mission_id
|
|
12
|
+
|
|
13
|
+
logger = structlog.get_logger()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class UploadPath:
|
|
18
|
+
"""Dataclass for retrieving the upload path of a file."""
|
|
19
|
+
|
|
20
|
+
mission: str
|
|
21
|
+
level: Level
|
|
22
|
+
day: int
|
|
23
|
+
month: int
|
|
24
|
+
year: int
|
|
25
|
+
id: str
|
|
26
|
+
path: str
|
|
27
|
+
|
|
28
|
+
def __str__(self):
|
|
29
|
+
"""Return a human-readable string representation of the Path."""
|
|
30
|
+
path = f"full/{self.mission.lower()}/{self.level.value.lower()}/{self.year:02}/{self.month:02}/{self.day:02}/{self.id}/{self.path}"
|
|
31
|
+
return path.removesuffix("/")
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def from_item_path(
|
|
35
|
+
cls, item: DatacosmosItem, mission: str, item_path: str
|
|
36
|
+
) -> "Path":
|
|
37
|
+
"""Create a Path instance from a DatacosmosItem and a path."""
|
|
38
|
+
for asset in item.assets.values():
|
|
39
|
+
if mission == "":
|
|
40
|
+
mission = cls._get_mission_name(asset.href)
|
|
41
|
+
else:
|
|
42
|
+
break
|
|
43
|
+
dt = datetime.strptime(item.properties["datetime"], "%Y-%m-%dT%H:%M:%SZ")
|
|
44
|
+
path = UploadPath(
|
|
45
|
+
mission=mission,
|
|
46
|
+
level=Level(item.properties["processing:level"].lower()),
|
|
47
|
+
day=dt.day,
|
|
48
|
+
month=dt.month,
|
|
49
|
+
year=dt.year,
|
|
50
|
+
id=item.id,
|
|
51
|
+
path=item_path,
|
|
52
|
+
)
|
|
53
|
+
return cls(**path.__dict__)
|
|
54
|
+
|
|
55
|
+
@classmethod
|
|
56
|
+
def from_path(cls, path: str) -> "Path":
|
|
57
|
+
"""Create a Path instance from a string path."""
|
|
58
|
+
parts = path.split("/")
|
|
59
|
+
if len(parts) < 7:
|
|
60
|
+
raise ValueError(f"Invalid path {path}")
|
|
61
|
+
return cls(
|
|
62
|
+
mission=parts[0],
|
|
63
|
+
level=Level(parts[1]),
|
|
64
|
+
day=int(parts[4]),
|
|
65
|
+
month=int(parts[3]),
|
|
66
|
+
year=int(parts[2]),
|
|
67
|
+
id=parts[5],
|
|
68
|
+
path="/".join(parts[6:]),
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
@classmethod
|
|
72
|
+
def _get_mission_name(cls, href: str) -> str:
|
|
73
|
+
mission = ""
|
|
74
|
+
# bruteforce mission name from asset path
|
|
75
|
+
# traverse the path and check if any part is a mission name (generates a mission id)
|
|
76
|
+
href_parts = href.split("/")
|
|
77
|
+
for idx, part in enumerate(href_parts):
|
|
78
|
+
try:
|
|
79
|
+
# when an id is found, then the mission name is valid
|
|
80
|
+
get_mission_id(
|
|
81
|
+
part, "test"
|
|
82
|
+
) # using test as it is more wide and anything on prod should exists on test
|
|
83
|
+
except KeyError:
|
|
84
|
+
continue
|
|
85
|
+
# validate the mission name by checking if the path is correct
|
|
86
|
+
# using the same logic as the __str__ method
|
|
87
|
+
mission = part.lower()
|
|
88
|
+
h = "/".join(["full", *href_parts[idx:]])
|
|
89
|
+
p = UploadPath.from_path("/".join([mission, *href_parts[idx + 1 :]]))
|
|
90
|
+
if str(p) != h:
|
|
91
|
+
raise ValueError(f"Could not find mission name in asset path {href}")
|
|
92
|
+
break
|
|
93
|
+
return mission
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Module for uploading files to Datacosmos cloud storage and registering STAC items."""
|
|
2
|
+
|
|
3
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from pydantic import TypeAdapter
|
|
7
|
+
|
|
8
|
+
from datacosmos.datacosmos_client import DatacosmosClient
|
|
9
|
+
from datacosmos.stac.item.item_client import ItemClient
|
|
10
|
+
from datacosmos.stac.item.models.datacosmos_item import DatacosmosItem
|
|
11
|
+
from datacosmos.uploader.dataclasses.upload_path import UploadPath
|
|
12
|
+
from datacosmos.utils.missions import get_mission_name
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class DatacosmosUploader:
|
|
16
|
+
"""Handles uploading files to Datacosmos storage and registering STAC items."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, client: DatacosmosClient):
|
|
19
|
+
"""Initialize the uploader with DatacosmosClient."""
|
|
20
|
+
mission_id = client.config.mission_id
|
|
21
|
+
environment = client.config.environment
|
|
22
|
+
|
|
23
|
+
self.datacosmos_client = client
|
|
24
|
+
self.item_client = ItemClient(client)
|
|
25
|
+
self.mission_name = (
|
|
26
|
+
get_mission_name(mission_id, environment) if mission_id != 0 else ""
|
|
27
|
+
)
|
|
28
|
+
self.base_url = client.config.datacosmos_cloud_storage.as_domain_url()
|
|
29
|
+
|
|
30
|
+
def upload_and_register_item(self, item_json_file_path: str) -> None:
|
|
31
|
+
"""Uploads files to Datacosmos storage and registers a STAC item.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
item_json_file_path (str): Path to the STAC item JSON file.
|
|
35
|
+
"""
|
|
36
|
+
item = self._load_item(item_json_file_path)
|
|
37
|
+
collection_id, item_id = item.collection, item.id
|
|
38
|
+
dirname = str(Path(item_json_file_path).parent / Path(item_json_file_path).stem)
|
|
39
|
+
|
|
40
|
+
self._delete_existing_item(collection_id, item_id)
|
|
41
|
+
upload_path = self._get_upload_path(item)
|
|
42
|
+
self.upload_from_folder(dirname, upload_path)
|
|
43
|
+
|
|
44
|
+
self._update_item_assets(item)
|
|
45
|
+
|
|
46
|
+
self.item_client.create_item(collection_id, item)
|
|
47
|
+
|
|
48
|
+
def upload_file(self, src: str, dst: str) -> None:
|
|
49
|
+
"""Uploads a single file to the specified destination path."""
|
|
50
|
+
url = self.base_url.with_suffix(str(dst))
|
|
51
|
+
|
|
52
|
+
with open(src, "rb") as f:
|
|
53
|
+
response = self.datacosmos_client.put(url, data=f)
|
|
54
|
+
response.raise_for_status()
|
|
55
|
+
|
|
56
|
+
def upload_from_folder(self, src: str, dst: UploadPath, workers: int = 4) -> None:
|
|
57
|
+
"""Uploads all files from a folder to the destination path in parallel."""
|
|
58
|
+
if Path(dst.path).is_file():
|
|
59
|
+
raise ValueError(f"Destination path should not be a file path {dst}")
|
|
60
|
+
|
|
61
|
+
if Path(src).is_file():
|
|
62
|
+
raise ValueError(f"Source path should not be a file path {src}")
|
|
63
|
+
|
|
64
|
+
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
65
|
+
futures = []
|
|
66
|
+
for file in Path(src).rglob("*"):
|
|
67
|
+
if file.is_file():
|
|
68
|
+
dst = UploadPath(
|
|
69
|
+
mission=dst.mission,
|
|
70
|
+
level=dst.level,
|
|
71
|
+
day=dst.day,
|
|
72
|
+
month=dst.month,
|
|
73
|
+
year=dst.year,
|
|
74
|
+
id=dst.id,
|
|
75
|
+
path=str(file.relative_to(src)),
|
|
76
|
+
)
|
|
77
|
+
futures.append(executor.submit(self.upload_file, str(file), dst))
|
|
78
|
+
for future in futures:
|
|
79
|
+
future.result()
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def _load_item(item_json_file_path: str) -> DatacosmosItem:
|
|
83
|
+
"""Loads and validates the STAC item from a JSON file."""
|
|
84
|
+
with open(item_json_file_path, "rb") as file:
|
|
85
|
+
data = file.read().decode("utf-8")
|
|
86
|
+
return TypeAdapter(DatacosmosItem).validate_json(data)
|
|
87
|
+
|
|
88
|
+
def _delete_existing_item(self, collection_id: str, item_id: str) -> None:
|
|
89
|
+
"""Deletes an existing item if it already exists."""
|
|
90
|
+
try:
|
|
91
|
+
self.item_client.delete_item(item_id, collection_id)
|
|
92
|
+
except Exception: # nosec
|
|
93
|
+
pass # Ignore if item doesn't exist
|
|
94
|
+
|
|
95
|
+
def _get_upload_path(self, item: DatacosmosItem) -> str:
|
|
96
|
+
"""Constructs the storage upload path based on the item and mission name."""
|
|
97
|
+
return UploadPath.from_item_path(item, self.mission_name, "")
|
|
98
|
+
|
|
99
|
+
def _update_item_assets(self, item: DatacosmosItem) -> None:
|
|
100
|
+
"""Updates the item's assets with uploaded file URLs."""
|
|
101
|
+
for asset in item.assets.values():
|
|
102
|
+
try:
|
|
103
|
+
url = self.base_url
|
|
104
|
+
asset.href = url.with_base(asset.href) # type: ignore
|
|
105
|
+
except ValueError:
|
|
106
|
+
pass
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Package for storing constants."""
|
|
2
|
+
|
|
3
|
+
TEST_MISSION_NAMES = {
|
|
4
|
+
55: "MENUT",
|
|
5
|
+
56: "PHISAT-2",
|
|
6
|
+
57: "HAMMER",
|
|
7
|
+
63: "MANTIS",
|
|
8
|
+
64: "PLATERO",
|
|
9
|
+
}
|
|
10
|
+
PROD_MISSION_NAMES = {
|
|
11
|
+
23: "MENUT",
|
|
12
|
+
29: "MANTIS",
|
|
13
|
+
35: "PHISAT-2",
|
|
14
|
+
37: "PLATERO",
|
|
15
|
+
48: "HAMMER",
|
|
16
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Package for storing mission specific information."""
|
|
2
|
+
|
|
3
|
+
from datacosmos.utils.constants import PROD_MISSION_NAMES, TEST_MISSION_NAMES
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_mission_name(mission: int, env: str) -> str:
|
|
7
|
+
"""Get the mission name from the mission number."""
|
|
8
|
+
if env == "test" or env == "local":
|
|
9
|
+
return TEST_MISSION_NAMES[mission]
|
|
10
|
+
elif env == "prod":
|
|
11
|
+
return PROD_MISSION_NAMES[mission]
|
|
12
|
+
else:
|
|
13
|
+
raise ValueError(f"Unsupported environment: {env}")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def get_mission_id(mission_name: str, env: str) -> int:
|
|
17
|
+
"""Get the mission number from the mission name."""
|
|
18
|
+
if env == "test" or env == "local":
|
|
19
|
+
return {v.upper(): k for k, v in TEST_MISSION_NAMES.items()}[
|
|
20
|
+
mission_name.upper()
|
|
21
|
+
]
|
|
22
|
+
elif env == "prod":
|
|
23
|
+
return {v.upper(): k for k, v in PROD_MISSION_NAMES.items()}[
|
|
24
|
+
mission_name.upper()
|
|
25
|
+
]
|
|
26
|
+
else:
|
|
27
|
+
raise ValueError(f"Unsupported environment: {env}")
|
datacosmos/utils/url.py
CHANGED
|
@@ -35,3 +35,26 @@ class URL:
|
|
|
35
35
|
"""
|
|
36
36
|
base = self.string()
|
|
37
37
|
return f"{base.rstrip('/')}/{suffix.lstrip('/')}"
|
|
38
|
+
|
|
39
|
+
def with_base(self, url: str) -> str:
|
|
40
|
+
"""Replaces the base of the url with the base stored in the URL object. (migrates url from one base to another).
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
url (str): url to migrate to the base of the URL object
|
|
44
|
+
|
|
45
|
+
Returns (str):
|
|
46
|
+
url with the base of the URL object
|
|
47
|
+
"""
|
|
48
|
+
split_url = url.split("/")
|
|
49
|
+
if len(split_url) < 3 or url.find("://") == -1:
|
|
50
|
+
raise ValueError(f"URL '{url}' does not meet the minimum requirements")
|
|
51
|
+
# get the whole path
|
|
52
|
+
url_path = "/".join(split_url[3:])
|
|
53
|
+
# simple case, matching self.base at url
|
|
54
|
+
b = self.base.lstrip("/")
|
|
55
|
+
if (base_pos := url_path.find(b)) != -1:
|
|
56
|
+
# remove the base from the url
|
|
57
|
+
url_suffix = url_path[len(b) + base_pos :]
|
|
58
|
+
else:
|
|
59
|
+
url_suffix = url_path
|
|
60
|
+
return self.with_suffix(url_suffix)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: datacosmos
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.3
|
|
4
4
|
Summary: A library for interacting with DataCosmos from Python code
|
|
5
5
|
Author-email: Open Cosmos <support@open-cosmos.com>
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -20,3 +20,4 @@ Requires-Dist: pytest==7.2.0; extra == "dev"
|
|
|
20
20
|
Requires-Dist: bandit[toml]==1.7.4; extra == "dev"
|
|
21
21
|
Requires-Dist: isort==5.11.4; extra == "dev"
|
|
22
22
|
Requires-Dist: pydocstyle==6.1.1; extra == "dev"
|
|
23
|
+
Dynamic: license-file
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
config/__init__.py,sha256=KCsaTb9-ZgFui1GM8wZFIPLJy0D0O8l8Z1Sv3NRD9UM,140
|
|
2
|
-
config/config.py,sha256=
|
|
2
|
+
config/config.py,sha256=h5XwNSA6QFBCDyennyFDNMAmbQOdtg8DsFAvjHlSEx4,7233
|
|
3
3
|
config/models/__init__.py,sha256=r3lThPkyKjBjUZXRNscFzOrmn_-m_i9DvG3RePfCFYc,41
|
|
4
4
|
config/models/m2m_authentication_config.py,sha256=1eJ_9df7Twn5WeWPbqMuR63ZdxhnTpBtKzqAPMnzP_k,565
|
|
5
5
|
config/models/url.py,sha256=fwr2C06e_RDS8AWxOV_orVxMWhc57bzYoWSjFxQbkwg,835
|
|
@@ -13,20 +13,32 @@ datacosmos/stac/collection/__init__.py,sha256=VQMLnsU3sER5kh4YxHrHP7XCA3DG1y0n9y
|
|
|
13
13
|
datacosmos/stac/collection/collection_client.py,sha256=XTO2s309-cktJosvnwnFFXHDVmJc4vjvbEsZjpsCDmY,5904
|
|
14
14
|
datacosmos/stac/collection/models/__init__.py,sha256=TQaihUS_CM9Eaekm4SbzFTNfv7BmabHv3Z-f37Py5Qs,40
|
|
15
15
|
datacosmos/stac/collection/models/collection_update.py,sha256=Tqmfg4H4UQj5jsgy1dpKJCR59NSfWeiCSi9y8CY8-Cg,1656
|
|
16
|
+
datacosmos/stac/enums/__init__.py,sha256=GUEL2xGtdjsrszrxivs0X6daxkaZs2JsTu2JoBtsvB4,22
|
|
17
|
+
datacosmos/stac/enums/level.py,sha256=dqrkSRtoutMTWatGyRRUz3uNKVlNXn3qa_ubXNbw618,237
|
|
16
18
|
datacosmos/stac/item/__init__.py,sha256=lRuD_yp-JxoLqBA23q0XMkCNImf4T-X3BJnSw9u_3Yk,200
|
|
17
|
-
datacosmos/stac/item/item_client.py,sha256=
|
|
19
|
+
datacosmos/stac/item/item_client.py,sha256=E6zHf3ANzVXd5Di_u05mLen5-EOKTdCs0VKcXXJ6lUc,6826
|
|
18
20
|
datacosmos/stac/item/models/__init__.py,sha256=bcOrOcIxGxGBrRVIyQVxSM3C3Xj_qzxIHgQeWo6f7Q8,34
|
|
21
|
+
datacosmos/stac/item/models/asset.py,sha256=mvg_fenYCGOTMGwXXpK2nyqBk5RMsUYxl6KhQTWW_b0,631
|
|
22
|
+
datacosmos/stac/item/models/datacosmos_item.py,sha256=jHuOkNvbVXUHdpplClPnA5mR4mcrfYQNm51EgQaVtNk,1704
|
|
23
|
+
datacosmos/stac/item/models/eo_band.py,sha256=YC3Scn_wFhIo51pIVcJeuJienF7JGWoEv39JngDM6rI,309
|
|
19
24
|
datacosmos/stac/item/models/item_update.py,sha256=_CpjQn9SsfedfuxlHSiGeptqY4M-p15t9YX__mBRueI,2088
|
|
25
|
+
datacosmos/stac/item/models/raster_band.py,sha256=CoEVs-YyPE5Fse0He9DdOs4dGZpzfCsCuVzOcdXa_UM,354
|
|
20
26
|
datacosmos/stac/item/models/search_parameters.py,sha256=yMmcb-Tr2as8585MD5wuZLWcqzwtRRkj07WBkootVS0,2022
|
|
27
|
+
datacosmos/uploader/__init__.py,sha256=ZtfCVJ_pWKKh2F1r_NArnbG3_JtpcEiXcA_tmSwSKmQ,128
|
|
28
|
+
datacosmos/uploader/datacosmos_uploader.py,sha256=LUtBDvAjZI7AYxKnC9TZQDP4z6lV2aHusz92XqivFGw,4398
|
|
29
|
+
datacosmos/uploader/dataclasses/__init__.py,sha256=IjcyA8Vod-z1_Gi1FMZhK58Owman0foL25Hs0YtkYYs,43
|
|
30
|
+
datacosmos/uploader/dataclasses/upload_path.py,sha256=WPl9u-oB-ti07ssKNDjL4vRQXhlOmLCgjt8MxFGrf3A,3153
|
|
21
31
|
datacosmos/utils/__init__.py,sha256=XQbAnoqJrPpnSpEzAbjh84yqYWw8cBM8mNp8ynTG-54,50
|
|
22
|
-
datacosmos/utils/
|
|
32
|
+
datacosmos/utils/constants.py,sha256=f7pOqCpdXk7WFGoaTyuCpr65jb-TtfhoVGuYTz3_T6Y,272
|
|
33
|
+
datacosmos/utils/missions.py,sha256=7GOnrjxB8V11C_Jr3HHI4vpXifgkOSeirNjIDx17C58,940
|
|
34
|
+
datacosmos/utils/url.py,sha256=iQwZr6mYRoePqUZg-k3KQSV9o2wju5ZuCa5WS_GyJo4,2114
|
|
23
35
|
datacosmos/utils/http_response/__init__.py,sha256=BvOWwC5coYqq_kFn8gIw5m54TLpdfJKlW9vgRkfhXiA,33
|
|
24
36
|
datacosmos/utils/http_response/check_api_response.py,sha256=dKWW01jn2_lWV0xpOBABhEP42CFSsx9dP0iSxykbN54,1186
|
|
25
37
|
datacosmos/utils/http_response/models/__init__.py,sha256=Wj8YT6dqw7rAz_rctllxo5Or_vv8DwopvQvBzwCTvpw,45
|
|
26
38
|
datacosmos/utils/http_response/models/datacosmos_error.py,sha256=Uqi2uM98nJPeCbM7zngV6vHSk97jEAb_nkdDEeUjiQM,740
|
|
27
39
|
datacosmos/utils/http_response/models/datacosmos_response.py,sha256=oV4n-sue7K1wwiIQeHpxdNU8vxeqF3okVPE2rydw5W0,336
|
|
28
|
-
datacosmos-0.0.
|
|
29
|
-
datacosmos-0.0.
|
|
30
|
-
datacosmos-0.0.
|
|
31
|
-
datacosmos-0.0.
|
|
32
|
-
datacosmos-0.0.
|
|
40
|
+
datacosmos-0.0.3.dist-info/licenses/LICENSE.md,sha256=vpbRI-UUbZVQfr3VG_CXt9HpRnL1b5kt8uTVbirxeyI,1486
|
|
41
|
+
datacosmos-0.0.3.dist-info/METADATA,sha256=ejuFFFnmdaInVXwXQtO6m35BQdTyotYYMLDVT-oWec4,843
|
|
42
|
+
datacosmos-0.0.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
|
43
|
+
datacosmos-0.0.3.dist-info/top_level.txt,sha256=Iu5b533Fmdfz0rFKTnuBPjSUOQL2lEkTfHxsokP72s4,18
|
|
44
|
+
datacosmos-0.0.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|