cubexpress 0.1.11__py3-none-any.whl → 0.1.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cubexpress might be problematic. Click here for more details.
- cubexpress/__init__.py +2 -2
- cubexpress/cloud_utils.py +68 -61
- cubexpress/cube.py +30 -62
- cubexpress/downloader.py +53 -61
- cubexpress/geospatial.py +0 -2
- cubexpress/geotyping.py +0 -5
- cubexpress/request.py +10 -11
- {cubexpress-0.1.11.dist-info → cubexpress-0.1.12.dist-info}/METADATA +2 -2
- cubexpress-0.1.12.dist-info/RECORD +13 -0
- cubexpress-0.1.11.dist-info/RECORD +0 -13
- {cubexpress-0.1.11.dist-info → cubexpress-0.1.12.dist-info}/LICENSE +0 -0
- {cubexpress-0.1.11.dist-info → cubexpress-0.1.12.dist-info}/WHEEL +0 -0
cubexpress/__init__.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from cubexpress.conversion import lonlat2rt, geo2utm
|
|
2
2
|
from cubexpress.geotyping import RasterTransform, Request, RequestSet, GeotransformDict
|
|
3
|
-
from cubexpress.cloud_utils import
|
|
3
|
+
from cubexpress.cloud_utils import s2_table
|
|
4
4
|
from cubexpress.cube import get_cube
|
|
5
5
|
from cubexpress.request import table_to_requestset
|
|
6
6
|
|
|
@@ -16,7 +16,7 @@ __all__ = [
|
|
|
16
16
|
"RequestSet",
|
|
17
17
|
"geo2utm",
|
|
18
18
|
"get_cube",
|
|
19
|
-
"
|
|
19
|
+
"s2_table",
|
|
20
20
|
"table_to_requestset"
|
|
21
21
|
]
|
|
22
22
|
|
cubexpress/cloud_utils.py
CHANGED
|
@@ -15,9 +15,11 @@ from __future__ import annotations
|
|
|
15
15
|
import datetime as dt
|
|
16
16
|
import ee
|
|
17
17
|
import pandas as pd
|
|
18
|
-
|
|
19
18
|
from cubexpress.cache import _cache_key
|
|
19
|
+
import datetime as dt
|
|
20
20
|
from cubexpress.geospatial import _square_roi
|
|
21
|
+
import warnings
|
|
22
|
+
warnings.filterwarnings('ignore', category=DeprecationWarning)
|
|
21
23
|
|
|
22
24
|
|
|
23
25
|
def _cloud_table_single_range(
|
|
@@ -55,61 +57,64 @@ def _cloud_table_single_range(
|
|
|
55
57
|
|
|
56
58
|
center = ee.Geometry.Point([lon, lat])
|
|
57
59
|
roi = _square_roi(lon, lat, edge_size, 10)
|
|
58
|
-
|
|
60
|
+
|
|
59
61
|
s2 = (
|
|
60
|
-
ee.ImageCollection("COPERNICUS/
|
|
62
|
+
ee.ImageCollection("COPERNICUS/S2_SR_HARMONIZED")
|
|
61
63
|
.filterBounds(roi)
|
|
62
64
|
.filterDate(start, end)
|
|
63
65
|
)
|
|
64
|
-
|
|
65
|
-
csp = ee.ImageCollection("GOOGLE/CLOUD_SCORE_PLUS/V1/S2_HARMONIZED")
|
|
66
|
-
|
|
67
66
|
ic = (
|
|
68
67
|
s2
|
|
69
|
-
.linkCollection(
|
|
68
|
+
.linkCollection(
|
|
69
|
+
ee.ImageCollection("GOOGLE/CLOUD_SCORE_PLUS/V1/S2_HARMONIZED"),
|
|
70
|
+
["cs_cdf"]
|
|
71
|
+
)
|
|
70
72
|
.select(["cs_cdf"])
|
|
71
73
|
)
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
74
|
+
ids_inside = (
|
|
75
|
+
ic
|
|
76
|
+
.map(
|
|
77
|
+
lambda img: img.set(
|
|
78
|
+
'roi_inside_scene',
|
|
79
|
+
img.geometry().contains(roi, maxError=10)
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
.filter(ee.Filter.eq('roi_inside_scene', True))
|
|
83
|
+
.aggregate_array('system:index')
|
|
84
|
+
.getInfo()
|
|
85
|
+
)
|
|
86
|
+
|
|
81
87
|
try:
|
|
82
|
-
raw = ic.getRegion(
|
|
88
|
+
raw = ic.getRegion(
|
|
89
|
+
geometry=center,
|
|
90
|
+
scale=(edge_size) * 11
|
|
91
|
+
).getInfo()
|
|
83
92
|
except ee.ee_exception.EEException as e:
|
|
84
93
|
if "No bands in collection" in str(e):
|
|
85
94
|
return pd.DataFrame(
|
|
86
|
-
columns=["id", "
|
|
95
|
+
columns=["id", "longitude", "latitude", "time", "cs_cdf", "inside"]
|
|
87
96
|
)
|
|
88
|
-
raise
|
|
89
|
-
|
|
90
|
-
df_raw =
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
df = (
|
|
94
|
-
df_ids
|
|
95
|
-
.merge(df_raw, on="id", how="left")
|
|
97
|
+
raise e
|
|
98
|
+
|
|
99
|
+
df_raw = (
|
|
100
|
+
pd.DataFrame(raw[1:], columns=raw[0])
|
|
101
|
+
.drop(columns=["longitude", "latitude"])
|
|
96
102
|
.assign(
|
|
97
|
-
date=lambda d: pd.to_datetime(d["id"].str[:8], format="%Y%m%d").dt.strftime("%Y-%m-%d")
|
|
98
|
-
null_flag=lambda d: d["cs_cdf"].isna().astype(int),
|
|
103
|
+
date=lambda d: pd.to_datetime(d["id"].str[:8], format="%Y%m%d").dt.strftime("%Y-%m-%d")
|
|
99
104
|
)
|
|
100
|
-
.drop(columns=["longitude", "latitude", "time"])
|
|
101
105
|
)
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
106
|
+
df_raw["inside"] = df_raw["id"].isin(set(ids_inside)).astype(int)
|
|
107
|
+
df_raw['cs_cdf'] = df_raw.groupby('date').apply(
|
|
108
|
+
lambda group: group['cs_cdf'].transform(
|
|
109
|
+
lambda _: group[group['inside'] == 1]['cs_cdf'].iloc[0]
|
|
110
|
+
if (group['inside'] == 1).any()
|
|
111
|
+
else group['cs_cdf'].mean()
|
|
112
|
+
)
|
|
113
|
+
).reset_index(drop=True)
|
|
114
|
+
|
|
115
|
+
return df_raw
|
|
116
|
+
|
|
117
|
+
def s2_table(
|
|
113
118
|
lon: float,
|
|
114
119
|
lat: float,
|
|
115
120
|
edge_size: int,
|
|
@@ -117,8 +122,7 @@ def s2_cloud_table(
|
|
|
117
122
|
end: str,
|
|
118
123
|
max_cscore: float = 1.0,
|
|
119
124
|
min_cscore: float = 0.0,
|
|
120
|
-
cache: bool = False
|
|
121
|
-
verbose: bool = True,
|
|
125
|
+
cache: bool = False
|
|
122
126
|
) -> pd.DataFrame:
|
|
123
127
|
"""Build (and cache) a per-day cloud-table for the requested ROI.
|
|
124
128
|
|
|
@@ -147,9 +151,7 @@ def s2_cloud_table(
|
|
|
147
151
|
Downstream path hint stored in ``result.attrs``; not used internally.
|
|
148
152
|
cache
|
|
149
153
|
Toggle parquet caching.
|
|
150
|
-
|
|
151
|
-
If *True* prints cache info/progress.
|
|
152
|
-
|
|
154
|
+
|
|
153
155
|
Returns
|
|
154
156
|
-------
|
|
155
157
|
pandas.DataFrame
|
|
@@ -161,10 +163,9 @@ def s2_cloud_table(
|
|
|
161
163
|
scale = 10
|
|
162
164
|
cache_file = _cache_key(lon, lat, edge_size, scale, collection)
|
|
163
165
|
|
|
164
|
-
#
|
|
166
|
+
# Load cached data if present
|
|
165
167
|
if cache and cache_file.exists():
|
|
166
|
-
|
|
167
|
-
print("📂 Loading cached metadata …")
|
|
168
|
+
print("📂 Loading cached metadata …")
|
|
168
169
|
df_cached = pd.read_parquet(cache_file)
|
|
169
170
|
have_idx = pd.to_datetime(df_cached["date"], errors="coerce").dropna()
|
|
170
171
|
|
|
@@ -175,8 +176,7 @@ def s2_cloud_table(
|
|
|
175
176
|
dt.date.fromisoformat(start) >= cached_start
|
|
176
177
|
and dt.date.fromisoformat(end) <= cached_end
|
|
177
178
|
):
|
|
178
|
-
|
|
179
|
-
print("✅ Served entirely from metadata.")
|
|
179
|
+
print("✅ Served entirely from metadata.")
|
|
180
180
|
df_full = df_cached
|
|
181
181
|
else:
|
|
182
182
|
# Identify missing segments and fetch only those.
|
|
@@ -185,14 +185,22 @@ def s2_cloud_table(
|
|
|
185
185
|
a1, b1 = start, cached_start.isoformat()
|
|
186
186
|
df_new_parts.append(
|
|
187
187
|
_cloud_table_single_range(
|
|
188
|
-
lon,
|
|
188
|
+
lon=lon,
|
|
189
|
+
lat=lat,
|
|
190
|
+
edge_size=edge_size,
|
|
191
|
+
start=a1,
|
|
192
|
+
end=b1
|
|
189
193
|
)
|
|
190
194
|
)
|
|
191
195
|
if dt.date.fromisoformat(end) > cached_end:
|
|
192
196
|
a2, b2 = cached_end.isoformat(), end
|
|
193
197
|
df_new_parts.append(
|
|
194
198
|
_cloud_table_single_range(
|
|
195
|
-
lon,
|
|
199
|
+
lon=lon,
|
|
200
|
+
lat=lat,
|
|
201
|
+
edge_size=edge_size,
|
|
202
|
+
start=a2,
|
|
203
|
+
end=b2
|
|
196
204
|
)
|
|
197
205
|
)
|
|
198
206
|
df_new_parts = [df for df in df_new_parts if not df.empty]
|
|
@@ -207,21 +215,20 @@ def s2_cloud_table(
|
|
|
207
215
|
else:
|
|
208
216
|
df_full = df_cached
|
|
209
217
|
else:
|
|
210
|
-
|
|
211
|
-
if verbose:
|
|
212
|
-
msg = "Generating metadata (no cache found)…" if cache else "Generating metadata…"
|
|
213
|
-
print("⏳", msg)
|
|
218
|
+
print("⏳ Generating metadata…")
|
|
214
219
|
df_full = _cloud_table_single_range(
|
|
215
|
-
lon,
|
|
220
|
+
lon=lon,
|
|
221
|
+
lat=lat,
|
|
222
|
+
edge_size=edge_size,
|
|
223
|
+
start=start,
|
|
224
|
+
end=end
|
|
216
225
|
)
|
|
217
|
-
|
|
218
226
|
|
|
219
|
-
#
|
|
227
|
+
# Save cache
|
|
220
228
|
if cache:
|
|
221
229
|
df_full.to_parquet(cache_file, compression="zstd")
|
|
222
230
|
|
|
223
|
-
#
|
|
224
|
-
|
|
231
|
+
# Filter by cloud cover and requested date window
|
|
225
232
|
result = (
|
|
226
233
|
df_full.query("@start <= date <= @end")
|
|
227
234
|
.query("@min_cscore <= cs_cdf <= @max_cscore")
|
cubexpress/cube.py
CHANGED
|
@@ -14,9 +14,10 @@ The core download/split logic lives in *cubexpress.downloader* and
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
16
|
import pathlib
|
|
17
|
-
|
|
17
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
18
18
|
from typing import Dict, Any
|
|
19
19
|
import ee
|
|
20
|
+
from tqdm import tqdm
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
from cubexpress.downloader import download_manifest, download_manifests
|
|
@@ -29,9 +30,7 @@ from cubexpress.geotyping import RequestSet
|
|
|
29
30
|
def get_geotiff(
|
|
30
31
|
manifest: Dict[str, Any],
|
|
31
32
|
full_outname: pathlib.Path | str,
|
|
32
|
-
|
|
33
|
-
nworks: int = 4,
|
|
34
|
-
verbose: bool = True,
|
|
33
|
+
nworks: int = 4
|
|
35
34
|
) -> None:
|
|
36
35
|
"""Download *manifest* to *full_outname*, retrying with tiled requests.
|
|
37
36
|
|
|
@@ -44,34 +43,27 @@ def get_geotiff(
|
|
|
44
43
|
nworks
|
|
45
44
|
Maximum worker threads when the image must be split; default **4**.
|
|
46
45
|
"""
|
|
47
|
-
|
|
46
|
+
|
|
48
47
|
try:
|
|
49
|
-
download_manifest(
|
|
48
|
+
download_manifest(
|
|
49
|
+
ulist=manifest,
|
|
50
|
+
full_outname=full_outname
|
|
51
|
+
)
|
|
50
52
|
except ee.ee_exception.EEException as err:
|
|
51
|
-
|
|
52
|
-
size = manifest["grid"]["dimensions"]["width"] # square images assumed
|
|
53
|
+
size = manifest["grid"]["dimensions"]["width"]
|
|
53
54
|
cell_w, cell_h, power = calculate_cell_size(str(err), size)
|
|
54
55
|
tiled = quadsplit_manifest(manifest, cell_w, cell_h, power)
|
|
56
|
+
|
|
55
57
|
download_manifests(
|
|
56
|
-
manifests
|
|
57
|
-
full_outname
|
|
58
|
-
|
|
59
|
-
max_workers = nworks
|
|
58
|
+
manifests=tiled,
|
|
59
|
+
full_outname=full_outname,
|
|
60
|
+
max_workers=nworks
|
|
60
61
|
)
|
|
61
62
|
|
|
62
|
-
if verbose:
|
|
63
|
-
print(f"Downloaded {full_outname}")
|
|
64
|
-
|
|
65
|
-
|
|
66
63
|
def get_cube(
|
|
67
|
-
# table: pd.DataFrame,
|
|
68
64
|
requests: pd.DataFrame | RequestSet,
|
|
69
65
|
outfolder: pathlib.Path | str,
|
|
70
|
-
|
|
71
|
-
join: bool = True,
|
|
72
|
-
nworks: int = 4,
|
|
73
|
-
verbose: bool = True,
|
|
74
|
-
cache: bool = True
|
|
66
|
+
nworks: int = 4
|
|
75
67
|
) -> None:
|
|
76
68
|
"""Download every request in *requests* to *outfolder* using a thread pool.
|
|
77
69
|
|
|
@@ -87,46 +79,22 @@ def get_cube(
|
|
|
87
79
|
nworks
|
|
88
80
|
Pool size for concurrent downloads; default **4**.
|
|
89
81
|
"""
|
|
90
|
-
|
|
91
|
-
# requests = table_to_requestset(
|
|
92
|
-
# table=table,
|
|
93
|
-
# mosaic=mosaic
|
|
94
|
-
# )
|
|
95
82
|
|
|
96
83
|
outfolder = pathlib.Path(outfolder).expanduser().resolve()
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
join, # join = join
|
|
111
|
-
nworks, # nworks = nworks
|
|
112
|
-
verbose # verbose = verbose
|
|
113
|
-
)
|
|
114
|
-
)
|
|
115
|
-
|
|
116
|
-
for fut in concurrent.futures.as_completed(futures):
|
|
84
|
+
outfolder.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
dataframe = requests._dataframe if isinstance(requests, RequestSet) else requests
|
|
86
|
+
|
|
87
|
+
with ThreadPoolExecutor(max_workers=nworks) as executor:
|
|
88
|
+
futures = {
|
|
89
|
+
executor.submit(
|
|
90
|
+
get_geotiff,
|
|
91
|
+
manifest=row.manifest,
|
|
92
|
+
full_outname=pathlib.Path(outfolder) / f"{row.id}.tif",
|
|
93
|
+
nworks=nworks
|
|
94
|
+
): row.id for _, row in dataframe.iterrows()
|
|
95
|
+
}
|
|
96
|
+
for future in tqdm(as_completed(futures), total=len(futures)):
|
|
117
97
|
try:
|
|
118
|
-
|
|
119
|
-
except Exception as exc:
|
|
120
|
-
print(f"Download error: {exc}")
|
|
121
|
-
|
|
122
|
-
# download_df = requests._dataframe[["outname", "cs_cdf", "date"]].copy()
|
|
123
|
-
# download_df["outname"] = outfolder / requests._dataframe["outname"]
|
|
124
|
-
# download_df.rename(columns={"outname": "full_outname"}, inplace=True)
|
|
125
|
-
|
|
126
|
-
return
|
|
127
|
-
|
|
128
|
-
# manifest = row.manifest
|
|
129
|
-
# full_outname = outname
|
|
130
|
-
# join: bool = True,
|
|
131
|
-
# nworks: int = 4,
|
|
132
|
-
# verbose: bool = True,
|
|
98
|
+
future.result()
|
|
99
|
+
except Exception as exc:
|
|
100
|
+
print(f"Download error for {futures[future]}: {exc}")
|
cubexpress/downloader.py
CHANGED
|
@@ -13,16 +13,14 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
import json
|
|
15
15
|
import pathlib
|
|
16
|
-
|
|
16
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
17
17
|
from copy import deepcopy
|
|
18
|
-
from typing import Any, Dict
|
|
18
|
+
from typing import Any, Dict
|
|
19
19
|
|
|
20
20
|
import ee
|
|
21
21
|
import rasterio as rio
|
|
22
22
|
from rasterio.io import MemoryFile
|
|
23
23
|
import logging
|
|
24
|
-
from rasterio.merge import merge
|
|
25
|
-
from rasterio.enums import Resampling
|
|
26
24
|
import os
|
|
27
25
|
import shutil
|
|
28
26
|
import tempfile
|
|
@@ -31,7 +29,10 @@ from cubexpress.geospatial import merge_tifs
|
|
|
31
29
|
os.environ['CPL_LOG_ERRORS'] = 'OFF'
|
|
32
30
|
logging.getLogger('rasterio._env').setLevel(logging.ERROR)
|
|
33
31
|
|
|
34
|
-
def download_manifest(
|
|
32
|
+
def download_manifest(
|
|
33
|
+
ulist: Dict[str, Any],
|
|
34
|
+
full_outname: pathlib.Path
|
|
35
|
+
) -> None:
|
|
35
36
|
"""Download *ulist* and save it as *full_outname*.
|
|
36
37
|
|
|
37
38
|
The manifest must include either an ``assetId`` or an ``expression``
|
|
@@ -45,36 +46,38 @@ def download_manifest(ulist: Dict[str, Any], full_outname: pathlib.Path) -> None
|
|
|
45
46
|
ulist_deep = deepcopy(ulist)
|
|
46
47
|
ulist_deep["expression"] = ee_image
|
|
47
48
|
images_bytes = ee.data.computePixels(ulist_deep)
|
|
48
|
-
else:
|
|
49
|
+
else:
|
|
49
50
|
raise ValueError("Manifest does not contain 'assetId' or 'expression'")
|
|
50
|
-
|
|
51
|
-
with
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
51
|
+
|
|
52
|
+
with open(full_outname, "wb") as src:
|
|
53
|
+
src.write(images_bytes)
|
|
54
|
+
|
|
55
|
+
# with MemoryFile(images_bytes) as memfile:
|
|
56
|
+
# with memfile.open() as src:
|
|
57
|
+
# profile = src.profile
|
|
58
|
+
# profile.update(
|
|
59
|
+
# driver="GTiff",
|
|
60
|
+
# tiled=True,
|
|
61
|
+
# interleave="band",
|
|
62
|
+
# blockxsize=256,
|
|
63
|
+
# blockysize=256,
|
|
64
|
+
# compress="ZSTD",
|
|
65
|
+
# zstd_level=13,
|
|
66
|
+
# predictor=2,
|
|
67
|
+
# num_threads=20,
|
|
68
|
+
# nodata=65535,
|
|
69
|
+
# dtype="uint16",
|
|
70
|
+
# count=12,
|
|
71
|
+
# photometric="MINISBLACK"
|
|
72
|
+
# )
|
|
73
|
+
|
|
74
|
+
# with rio.open(full_outname, "w", **profile) as dst:
|
|
75
|
+
# dst.write(src.read())
|
|
72
76
|
|
|
73
77
|
def download_manifests(
|
|
74
78
|
manifests: list[Dict[str, Any]],
|
|
75
79
|
full_outname: pathlib.Path,
|
|
76
|
-
|
|
77
|
-
max_workers: int = 4,
|
|
80
|
+
max_workers: int,
|
|
78
81
|
) -> None:
|
|
79
82
|
"""Download every manifest in *manifests* concurrently.
|
|
80
83
|
|
|
@@ -82,38 +85,27 @@ def download_manifests(
|
|
|
82
85
|
``full_outname.parent/full_outname.stem`` with names ``000000.tif``,
|
|
83
86
|
``000001.tif`` … according to the list order.
|
|
84
87
|
"""
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
futures.append(
|
|
99
|
-
executor.submit(
|
|
100
|
-
download_manifest,
|
|
101
|
-
umanifest, # ulist = umanifest
|
|
102
|
-
outname # full_outname = outname
|
|
103
|
-
)
|
|
104
|
-
)
|
|
105
|
-
|
|
106
|
-
for fut in concurrent.futures.as_completed(futures):
|
|
88
|
+
tmp_dir = pathlib.Path(tempfile.mkdtemp(prefix="cubexpress_"))
|
|
89
|
+
full_outname_temp = tmp_dir / full_outname.stem
|
|
90
|
+
full_outname_temp.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
|
|
92
|
+
with ThreadPoolExecutor(max_workers=max_workers) as exe: # -
|
|
93
|
+
futures = {
|
|
94
|
+
exe.submit(
|
|
95
|
+
download_manifest,
|
|
96
|
+
ulist=umanifest,
|
|
97
|
+
full_outname=full_outname_temp / f"{index:06d}.tif"
|
|
98
|
+
): umanifest for index, umanifest in enumerate(manifests)
|
|
99
|
+
}
|
|
100
|
+
for future in as_completed(futures):
|
|
107
101
|
try:
|
|
108
|
-
|
|
109
|
-
except Exception as exc:
|
|
110
|
-
print(f"Error
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
if dir_path.exists():
|
|
115
|
-
input_files = sorted(dir_path.glob("*.tif"))
|
|
102
|
+
future.result()
|
|
103
|
+
except Exception as exc:
|
|
104
|
+
print(f"Error in one of the downloads: {exc}")
|
|
105
|
+
|
|
106
|
+
if full_outname_temp.exists():
|
|
107
|
+
input_files = sorted(full_outname_temp.glob("*.tif"))
|
|
116
108
|
merge_tifs(input_files, full_outname)
|
|
117
|
-
shutil.rmtree(
|
|
109
|
+
shutil.rmtree(full_outname_temp)
|
|
118
110
|
else:
|
|
119
111
|
raise ValueError(f"Error in {full_outname}")
|
cubexpress/geospatial.py
CHANGED
cubexpress/geotyping.py
CHANGED
|
@@ -259,13 +259,8 @@ class RequestSet(BaseModel):
|
|
|
259
259
|
def create_manifests(self) -> pd.DataFrame:
|
|
260
260
|
"""
|
|
261
261
|
Exports the raster metadata to a pandas DataFrame.
|
|
262
|
-
|
|
263
262
|
Returns:
|
|
264
263
|
pd.DataFrame: A DataFrame containing the metadata for all entries.
|
|
265
|
-
|
|
266
|
-
Example:
|
|
267
|
-
>>> df = raster_transform_set.export_df()
|
|
268
|
-
>>> print(df)
|
|
269
264
|
"""
|
|
270
265
|
# Use ProcessPoolExecutor for CPU-bound tasks to convert raster transforms to lon/lat
|
|
271
266
|
with ProcessPoolExecutor(max_workers=None) as executor:
|
cubexpress/request.py
CHANGED
|
@@ -11,9 +11,9 @@ from cubexpress.conversion import lonlat2rt
|
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def table_to_requestset(
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
table: pd.DataFrame,
|
|
15
|
+
mosaic: bool = True
|
|
16
|
+
) -> RequestSet:
|
|
17
17
|
"""Return a :class:`RequestSet` built from *df* (cloud_table result).
|
|
18
18
|
|
|
19
19
|
Parameters
|
|
@@ -35,7 +35,7 @@ def table_to_requestset(
|
|
|
35
35
|
df = table.copy()
|
|
36
36
|
|
|
37
37
|
if df.empty:
|
|
38
|
-
raise ValueError("
|
|
38
|
+
raise ValueError("There are no images in the requested period. Please check your dates or your ubication.")
|
|
39
39
|
|
|
40
40
|
rt = lonlat2rt(
|
|
41
41
|
lon=df.attrs["lon"],
|
|
@@ -43,11 +43,11 @@ def table_to_requestset(
|
|
|
43
43
|
edge_size=df.attrs["edge_size"],
|
|
44
44
|
scale=df.attrs["scale"],
|
|
45
45
|
)
|
|
46
|
+
|
|
46
47
|
centre_hash = pgh.encode(df.attrs["lat"], df.attrs["lon"], precision=5)
|
|
47
|
-
reqs
|
|
48
|
+
reqs = []
|
|
48
49
|
|
|
49
50
|
if mosaic:
|
|
50
|
-
|
|
51
51
|
grouped = (
|
|
52
52
|
df.groupby('date')
|
|
53
53
|
.agg(
|
|
@@ -66,8 +66,7 @@ def table_to_requestset(
|
|
|
66
66
|
)
|
|
67
67
|
|
|
68
68
|
for day, row in grouped.iterrows():
|
|
69
|
-
|
|
70
|
-
|
|
69
|
+
|
|
71
70
|
img_ids = row["id_list"]
|
|
72
71
|
cdf = row["cs_cdf_mean"]
|
|
73
72
|
|
|
@@ -100,16 +99,16 @@ def table_to_requestset(
|
|
|
100
99
|
else:
|
|
101
100
|
for _, row in df.iterrows():
|
|
102
101
|
img_id = row["id"]
|
|
103
|
-
|
|
102
|
+
tile = img_id.split("_")[-1][1:]
|
|
104
103
|
day = row["date"]
|
|
105
104
|
cdf = int(round(row["cs_cdf"], 2) * 100)
|
|
106
105
|
reqs.append(
|
|
107
106
|
Request(
|
|
108
|
-
id=f"{day}_{
|
|
107
|
+
id=f"{day}_{tile}_{cdf}",
|
|
109
108
|
raster_transform=rt,
|
|
110
109
|
image=f"{df.attrs['collection']}/{img_id}",
|
|
111
110
|
bands=df.attrs["bands"],
|
|
112
111
|
)
|
|
113
112
|
)
|
|
114
113
|
|
|
115
|
-
return RequestSet(requestset=reqs)
|
|
114
|
+
return RequestSet(requestset=reqs)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: cubexpress
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.12
|
|
4
4
|
Summary: Efficient processing of cubic Earth-observation (EO) data.
|
|
5
5
|
Home-page: https://github.com/andesdatacube/cubexpress
|
|
6
6
|
Keywords: earth-engine,sentinel-2,geospatial,eo,cube
|
|
@@ -31,7 +31,7 @@ Description-Content-Type: text/markdown
|
|
|
31
31
|
<h1></h1>
|
|
32
32
|
|
|
33
33
|
<p align="center">
|
|
34
|
-
<img src="
|
|
34
|
+
<img src="https://raw.githubusercontent.com/andesdatacube/cubexpress/refs/heads/main/docs/logo_cubexpress.png" width="39%">
|
|
35
35
|
</p>
|
|
36
36
|
|
|
37
37
|
<p align="center">
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
cubexpress/__init__.py,sha256=G_9FKWrdFh44oYjD78jtH9gXqrdARIFFM0MaSl6WttY,606
|
|
2
|
+
cubexpress/cache.py,sha256=EZiR2AJfplaLpqMIVFb5piCAgFqHKF1vgLIrutfz8tA,1425
|
|
3
|
+
cubexpress/cloud_utils.py,sha256=EhLqHrefia7AUp75HZXKrQSPKQb6BSYFKD3hjI_sj1M,7681
|
|
4
|
+
cubexpress/conversion.py,sha256=JSaMnswY-2n5E4H2zxb-oEOTJ8UPzXfMeSVCremtvTw,2520
|
|
5
|
+
cubexpress/cube.py,sha256=lqJJyf1EmNYszIztx62OCrdo0HGtIveOmw-pFGI2nuI,3230
|
|
6
|
+
cubexpress/downloader.py,sha256=rnk-oX51_YFWz1iZuBWEYTDSTV48F780o1aujTsKCwE,3725
|
|
7
|
+
cubexpress/geospatial.py,sha256=jldZ-aFqUEvp1SF8ZJEa-pDHSAs3akzqk43dHai0TKM,3820
|
|
8
|
+
cubexpress/geotyping.py,sha256=uTaoZnaegGUShAHy-t0og22vpBbSrnJhCxp_4UoO9TM,16597
|
|
9
|
+
cubexpress/request.py,sha256=gLY8eBkMVal0uF6auGIfqmOhUPWFHwJzBxXOXU4aRuA,3370
|
|
10
|
+
cubexpress-0.1.12.dist-info/LICENSE,sha256=XjoS-d76b7Cl-VgCWhQk83tNf2dNldKBN8SrImwGc2Q,1072
|
|
11
|
+
cubexpress-0.1.12.dist-info/METADATA,sha256=u1EpUC4n5Jdpewvjv67l2IWp9ULCqRzshPqNTY85Tcs,9724
|
|
12
|
+
cubexpress-0.1.12.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
13
|
+
cubexpress-0.1.12.dist-info/RECORD,,
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
cubexpress/__init__.py,sha256=sKXcYQQPREFhVCHP81lL_5hAurUTm8MX1xVOEOMF-nA,618
|
|
2
|
-
cubexpress/cache.py,sha256=EZiR2AJfplaLpqMIVFb5piCAgFqHKF1vgLIrutfz8tA,1425
|
|
3
|
-
cubexpress/cloud_utils.py,sha256=BxS3HADLNj6rdFGYUjpcXA1Vvsa87JoL28YEAsu51H4,7482
|
|
4
|
-
cubexpress/conversion.py,sha256=JSaMnswY-2n5E4H2zxb-oEOTJ8UPzXfMeSVCremtvTw,2520
|
|
5
|
-
cubexpress/cube.py,sha256=SMN6MvezfeHipFE4v4f23dxWGk9h2t2s2aeeppD0voY,4133
|
|
6
|
-
cubexpress/downloader.py,sha256=XsLDlq2ZHEccc1ET8ghnuOIYtGazVDwXohMSWBemVMw,4067
|
|
7
|
-
cubexpress/geospatial.py,sha256=2DGwl3pyfNEOj8nn9gjc-tiiTXhV2ez9Bghz1I0vERs,3822
|
|
8
|
-
cubexpress/geotyping.py,sha256=Fbnn7EoRvXrtjTRFTS4CPzQbxG4PA6WkfeM4YUp9iKg,16696
|
|
9
|
-
cubexpress/request.py,sha256=PiDqnt3qB9tac4KkZdPIrv5VeRHqobk1u2q1VCCH2lI,3390
|
|
10
|
-
cubexpress-0.1.11.dist-info/LICENSE,sha256=XjoS-d76b7Cl-VgCWhQk83tNf2dNldKBN8SrImwGc2Q,1072
|
|
11
|
-
cubexpress-0.1.11.dist-info/METADATA,sha256=pxtqImmO_wIyA9P_0TWaxxps0O-95O6aVLbfEQ9GvBk,9651
|
|
12
|
-
cubexpress-0.1.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
13
|
-
cubexpress-0.1.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|