cubexpress 0.1.11__tar.gz → 0.1.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cubexpress might be problematic. Click here for more details.

@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: cubexpress
3
- Version: 0.1.11
3
+ Version: 0.1.12
4
4
  Summary: Efficient processing of cubic Earth-observation (EO) data.
5
5
  Home-page: https://github.com/andesdatacube/cubexpress
6
6
  Keywords: earth-engine,sentinel-2,geospatial,eo,cube
@@ -31,7 +31,7 @@ Description-Content-Type: text/markdown
31
31
  <h1></h1>
32
32
 
33
33
  <p align="center">
34
- <img src="./docs/logo_cubexpress.png" width="39%">
34
+ <img src="https://raw.githubusercontent.com/andesdatacube/cubexpress/refs/heads/main/docs/logo_cubexpress.png" width="39%">
35
35
  </p>
36
36
 
37
37
  <p align="center">
@@ -1,7 +1,7 @@
1
1
  <h1></h1>
2
2
 
3
3
  <p align="center">
4
- <img src="./docs/logo_cubexpress.png" width="39%">
4
+ <img src="https://raw.githubusercontent.com/andesdatacube/cubexpress/refs/heads/main/docs/logo_cubexpress.png" width="39%">
5
5
  </p>
6
6
 
7
7
  <p align="center">
@@ -1,6 +1,6 @@
1
1
  from cubexpress.conversion import lonlat2rt, geo2utm
2
2
  from cubexpress.geotyping import RasterTransform, Request, RequestSet, GeotransformDict
3
- from cubexpress.cloud_utils import s2_cloud_table
3
+ from cubexpress.cloud_utils import s2_table
4
4
  from cubexpress.cube import get_cube
5
5
  from cubexpress.request import table_to_requestset
6
6
 
@@ -16,7 +16,7 @@ __all__ = [
16
16
  "RequestSet",
17
17
  "geo2utm",
18
18
  "get_cube",
19
- "s2_cloud_table",
19
+ "s2_table",
20
20
  "table_to_requestset"
21
21
  ]
22
22
 
@@ -15,9 +15,11 @@ from __future__ import annotations
15
15
  import datetime as dt
16
16
  import ee
17
17
  import pandas as pd
18
-
19
18
  from cubexpress.cache import _cache_key
19
+ import datetime as dt
20
20
  from cubexpress.geospatial import _square_roi
21
+ import warnings
22
+ warnings.filterwarnings('ignore', category=DeprecationWarning)
21
23
 
22
24
 
23
25
  def _cloud_table_single_range(
@@ -55,61 +57,64 @@ def _cloud_table_single_range(
55
57
 
56
58
  center = ee.Geometry.Point([lon, lat])
57
59
  roi = _square_roi(lon, lat, edge_size, 10)
58
-
60
+
59
61
  s2 = (
60
- ee.ImageCollection("COPERNICUS/S2_HARMONIZED")
62
+ ee.ImageCollection("COPERNICUS/S2_SR_HARMONIZED")
61
63
  .filterBounds(roi)
62
64
  .filterDate(start, end)
63
65
  )
64
-
65
- csp = ee.ImageCollection("GOOGLE/CLOUD_SCORE_PLUS/V1/S2_HARMONIZED")
66
-
67
66
  ic = (
68
67
  s2
69
- .linkCollection(csp, ["cs_cdf"])
68
+ .linkCollection(
69
+ ee.ImageCollection("GOOGLE/CLOUD_SCORE_PLUS/V1/S2_HARMONIZED"),
70
+ ["cs_cdf"]
71
+ )
70
72
  .select(["cs_cdf"])
71
73
  )
72
-
73
- # image IDs for every expected date
74
- ids = ic.aggregate_array("system:index").getInfo()
75
- df_ids = pd.DataFrame({"id": ids})
76
-
77
-
78
- region_scale = edge_size * 10 / 2
79
-
80
-
74
+ ids_inside = (
75
+ ic
76
+ .map(
77
+ lambda img: img.set(
78
+ 'roi_inside_scene',
79
+ img.geometry().contains(roi, maxError=10)
80
+ )
81
+ )
82
+ .filter(ee.Filter.eq('roi_inside_scene', True))
83
+ .aggregate_array('system:index')
84
+ .getInfo()
85
+ )
86
+
81
87
  try:
82
- raw = ic.getRegion(geometry=center, scale=region_scale).getInfo()
88
+ raw = ic.getRegion(
89
+ geometry=center,
90
+ scale=(edge_size) * 11
91
+ ).getInfo()
83
92
  except ee.ee_exception.EEException as e:
84
93
  if "No bands in collection" in str(e):
85
94
  return pd.DataFrame(
86
- columns=["id", "cs_cdf", "date", "null_flag"]
95
+ columns=["id", "longitude", "latitude", "time", "cs_cdf", "inside"]
87
96
  )
88
- raise
89
-
90
- df_raw = pd.DataFrame(raw[1:], columns=raw[0])
91
-
92
-
93
- df = (
94
- df_ids
95
- .merge(df_raw, on="id", how="left")
97
+ raise e
98
+
99
+ df_raw = (
100
+ pd.DataFrame(raw[1:], columns=raw[0])
101
+ .drop(columns=["longitude", "latitude"])
96
102
  .assign(
97
- date=lambda d: pd.to_datetime(d["id"].str[:8], format="%Y%m%d").dt.strftime("%Y-%m-%d"),
98
- null_flag=lambda d: d["cs_cdf"].isna().astype(int),
103
+ date=lambda d: pd.to_datetime(d["id"].str[:8], format="%Y%m%d").dt.strftime("%Y-%m-%d")
99
104
  )
100
- .drop(columns=["longitude", "latitude", "time"])
101
105
  )
102
-
103
- # fill missing scores with daily mean
104
- df["lon"] = lon
105
- df["lat"] = lat
106
- df["cs_cdf"] = df["cs_cdf"].fillna(df.groupby("date")["cs_cdf"].transform("mean"))
107
-
108
-
109
- return df
110
-
111
-
112
- def s2_cloud_table(
106
+ df_raw["inside"] = df_raw["id"].isin(set(ids_inside)).astype(int)
107
+ df_raw['cs_cdf'] = df_raw.groupby('date').apply(
108
+ lambda group: group['cs_cdf'].transform(
109
+ lambda _: group[group['inside'] == 1]['cs_cdf'].iloc[0]
110
+ if (group['inside'] == 1).any()
111
+ else group['cs_cdf'].mean()
112
+ )
113
+ ).reset_index(drop=True)
114
+
115
+ return df_raw
116
+
117
+ def s2_table(
113
118
  lon: float,
114
119
  lat: float,
115
120
  edge_size: int,
@@ -117,8 +122,7 @@ def s2_cloud_table(
117
122
  end: str,
118
123
  max_cscore: float = 1.0,
119
124
  min_cscore: float = 0.0,
120
- cache: bool = False,
121
- verbose: bool = True,
125
+ cache: bool = False
122
126
  ) -> pd.DataFrame:
123
127
  """Build (and cache) a per-day cloud-table for the requested ROI.
124
128
 
@@ -147,9 +151,7 @@ def s2_cloud_table(
147
151
  Downstream path hint stored in ``result.attrs``; not used internally.
148
152
  cache
149
153
  Toggle parquet caching.
150
- verbose
151
- If *True* prints cache info/progress.
152
-
154
+
153
155
  Returns
154
156
  -------
155
157
  pandas.DataFrame
@@ -161,10 +163,9 @@ def s2_cloud_table(
161
163
  scale = 10
162
164
  cache_file = _cache_key(lon, lat, edge_size, scale, collection)
163
165
 
164
- # ─── 1. Load cached data if present ────────────────────────────────────
166
+ # Load cached data if present
165
167
  if cache and cache_file.exists():
166
- if verbose:
167
- print("📂 Loading cached metadata …")
168
+ print("📂 Loading cached metadata …")
168
169
  df_cached = pd.read_parquet(cache_file)
169
170
  have_idx = pd.to_datetime(df_cached["date"], errors="coerce").dropna()
170
171
 
@@ -175,8 +176,7 @@ def s2_cloud_table(
175
176
  dt.date.fromisoformat(start) >= cached_start
176
177
  and dt.date.fromisoformat(end) <= cached_end
177
178
  ):
178
- if verbose:
179
- print("✅ Served entirely from metadata.")
179
+ print("✅ Served entirely from metadata.")
180
180
  df_full = df_cached
181
181
  else:
182
182
  # Identify missing segments and fetch only those.
@@ -185,14 +185,22 @@ def s2_cloud_table(
185
185
  a1, b1 = start, cached_start.isoformat()
186
186
  df_new_parts.append(
187
187
  _cloud_table_single_range(
188
- lon, lat, edge_size, a1, b1
188
+ lon=lon,
189
+ lat=lat,
190
+ edge_size=edge_size,
191
+ start=a1,
192
+ end=b1
189
193
  )
190
194
  )
191
195
  if dt.date.fromisoformat(end) > cached_end:
192
196
  a2, b2 = cached_end.isoformat(), end
193
197
  df_new_parts.append(
194
198
  _cloud_table_single_range(
195
- lon, lat, edge_size, a2, b2
199
+ lon=lon,
200
+ lat=lat,
201
+ edge_size=edge_size,
202
+ start=a2,
203
+ end=b2
196
204
  )
197
205
  )
198
206
  df_new_parts = [df for df in df_new_parts if not df.empty]
@@ -207,21 +215,20 @@ def s2_cloud_table(
207
215
  else:
208
216
  df_full = df_cached
209
217
  else:
210
-
211
- if verbose:
212
- msg = "Generating metadata (no cache found)…" if cache else "Generating metadata…"
213
- print("⏳", msg)
218
+ print("⏳ Generating metadata…")
214
219
  df_full = _cloud_table_single_range(
215
- lon, lat, edge_size, start, end
220
+ lon=lon,
221
+ lat=lat,
222
+ edge_size=edge_size,
223
+ start=start,
224
+ end=end
216
225
  )
217
-
218
226
 
219
- # ─── 2. Save cache ─────────────────────────────────────────────────────
227
+ # Save cache
220
228
  if cache:
221
229
  df_full.to_parquet(cache_file, compression="zstd")
222
230
 
223
- # ─── 3. Filter by cloud cover and requested date window ────────────────
224
-
231
+ # Filter by cloud cover and requested date window
225
232
  result = (
226
233
  df_full.query("@start <= date <= @end")
227
234
  .query("@min_cscore <= cs_cdf <= @max_cscore")
@@ -14,9 +14,10 @@ The core download/split logic lives in *cubexpress.downloader* and
14
14
  from __future__ import annotations
15
15
 
16
16
  import pathlib
17
- import concurrent.futures
17
+ from concurrent.futures import ThreadPoolExecutor, as_completed
18
18
  from typing import Dict, Any
19
19
  import ee
20
+ from tqdm import tqdm
20
21
 
21
22
 
22
23
  from cubexpress.downloader import download_manifest, download_manifests
@@ -29,9 +30,7 @@ from cubexpress.geotyping import RequestSet
29
30
  def get_geotiff(
30
31
  manifest: Dict[str, Any],
31
32
  full_outname: pathlib.Path | str,
32
- join: bool = True,
33
- nworks: int = 4,
34
- verbose: bool = True,
33
+ nworks: int = 4
35
34
  ) -> None:
36
35
  """Download *manifest* to *full_outname*, retrying with tiled requests.
37
36
 
@@ -44,34 +43,27 @@ def get_geotiff(
44
43
  nworks
45
44
  Maximum worker threads when the image must be split; default **4**.
46
45
  """
47
- full_outname = pathlib.Path(full_outname)
46
+
48
47
  try:
49
- download_manifest(manifest, full_outname)
48
+ download_manifest(
49
+ ulist=manifest,
50
+ full_outname=full_outname
51
+ )
50
52
  except ee.ee_exception.EEException as err:
51
-
52
- size = manifest["grid"]["dimensions"]["width"] # square images assumed
53
+ size = manifest["grid"]["dimensions"]["width"]
53
54
  cell_w, cell_h, power = calculate_cell_size(str(err), size)
54
55
  tiled = quadsplit_manifest(manifest, cell_w, cell_h, power)
56
+
55
57
  download_manifests(
56
- manifests = tiled,
57
- full_outname = full_outname,
58
- join = join,
59
- max_workers = nworks
58
+ manifests=tiled,
59
+ full_outname=full_outname,
60
+ max_workers=nworks
60
61
  )
61
62
 
62
- if verbose:
63
- print(f"Downloaded {full_outname}")
64
-
65
-
66
63
  def get_cube(
67
- # table: pd.DataFrame,
68
64
  requests: pd.DataFrame | RequestSet,
69
65
  outfolder: pathlib.Path | str,
70
- mosaic: bool = True,
71
- join: bool = True,
72
- nworks: int = 4,
73
- verbose: bool = True,
74
- cache: bool = True
66
+ nworks: int = 4
75
67
  ) -> None:
76
68
  """Download every request in *requests* to *outfolder* using a thread pool.
77
69
 
@@ -87,46 +79,22 @@ def get_cube(
87
79
  nworks
88
80
  Pool size for concurrent downloads; default **4**.
89
81
  """
90
-
91
- # requests = table_to_requestset(
92
- # table=table,
93
- # mosaic=mosaic
94
- # )
95
82
 
96
83
  outfolder = pathlib.Path(outfolder).expanduser().resolve()
97
-
98
- with concurrent.futures.ThreadPoolExecutor(max_workers=nworks) as pool:
99
- futures = []
100
- for _, row in requests._dataframe.iterrows():
101
- outname = pathlib.Path(outfolder) / f"{row.id}.tif"
102
- if outname.exists() and cache:
103
- continue
104
- outname.parent.mkdir(parents=True, exist_ok=True)
105
- futures.append(
106
- pool.submit(
107
- get_geotiff,
108
- row.manifest, # manifest = row.manifest
109
- outname, # full_outname = outname
110
- join, # join = join
111
- nworks, # nworks = nworks
112
- verbose # verbose = verbose
113
- )
114
- )
115
-
116
- for fut in concurrent.futures.as_completed(futures):
84
+ outfolder.mkdir(parents=True, exist_ok=True)
85
+ dataframe = requests._dataframe if isinstance(requests, RequestSet) else requests
86
+
87
+ with ThreadPoolExecutor(max_workers=nworks) as executor:
88
+ futures = {
89
+ executor.submit(
90
+ get_geotiff,
91
+ manifest=row.manifest,
92
+ full_outname=pathlib.Path(outfolder) / f"{row.id}.tif",
93
+ nworks=nworks
94
+ ): row.id for _, row in dataframe.iterrows()
95
+ }
96
+ for future in tqdm(as_completed(futures), total=len(futures)):
117
97
  try:
118
- fut.result()
119
- except Exception as exc: # noqa: BLE001 – log and keep going
120
- print(f"Download error: {exc}")
121
-
122
- # download_df = requests._dataframe[["outname", "cs_cdf", "date"]].copy()
123
- # download_df["outname"] = outfolder / requests._dataframe["outname"]
124
- # download_df.rename(columns={"outname": "full_outname"}, inplace=True)
125
-
126
- return
127
-
128
- # manifest = row.manifest
129
- # full_outname = outname
130
- # join: bool = True,
131
- # nworks: int = 4,
132
- # verbose: bool = True,
98
+ future.result()
99
+ except Exception as exc:
100
+ print(f"Download error for {futures[future]}: {exc}")
@@ -0,0 +1,111 @@
1
+ """Low-level download helpers for Earth Engine manifests.
2
+
3
+ Only two public callables are exposed:
4
+
5
+ * :func:`download_manifest` – fetch a single manifest and write one GeoTIFF.
6
+ * :func:`download_manifests` – convenience wrapper to parallel-download a list
7
+ of manifests with a thread pool.
8
+
9
+ Both functions are fully I/O bound; no return value is expected.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import pathlib
16
+ from concurrent.futures import ThreadPoolExecutor, as_completed
17
+ from copy import deepcopy
18
+ from typing import Any, Dict
19
+
20
+ import ee
21
+ import rasterio as rio
22
+ from rasterio.io import MemoryFile
23
+ import logging
24
+ import os
25
+ import shutil
26
+ import tempfile
27
+ from cubexpress.geospatial import merge_tifs
28
+
29
+ os.environ['CPL_LOG_ERRORS'] = 'OFF'
30
+ logging.getLogger('rasterio._env').setLevel(logging.ERROR)
31
+
32
+ def download_manifest(
33
+ ulist: Dict[str, Any],
34
+ full_outname: pathlib.Path
35
+ ) -> None:
36
+ """Download *ulist* and save it as *full_outname*.
37
+
38
+ The manifest must include either an ``assetId`` or an ``expression``
39
+ (serialized EE image). RasterIO is used to write a tiled, compressed
40
+ GeoTIFF; the function is silent apart from the final ``print``.
41
+ """
42
+ if "assetId" in ulist:
43
+ images_bytes = ee.data.getPixels(ulist)
44
+ elif "expression" in ulist:
45
+ ee_image = ee.deserializer.decode(json.loads(ulist["expression"]))
46
+ ulist_deep = deepcopy(ulist)
47
+ ulist_deep["expression"] = ee_image
48
+ images_bytes = ee.data.computePixels(ulist_deep)
49
+ else:
50
+ raise ValueError("Manifest does not contain 'assetId' or 'expression'")
51
+
52
+ with open(full_outname, "wb") as src:
53
+ src.write(images_bytes)
54
+
55
+ # with MemoryFile(images_bytes) as memfile:
56
+ # with memfile.open() as src:
57
+ # profile = src.profile
58
+ # profile.update(
59
+ # driver="GTiff",
60
+ # tiled=True,
61
+ # interleave="band",
62
+ # blockxsize=256,
63
+ # blockysize=256,
64
+ # compress="ZSTD",
65
+ # zstd_level=13,
66
+ # predictor=2,
67
+ # num_threads=20,
68
+ # nodata=65535,
69
+ # dtype="uint16",
70
+ # count=12,
71
+ # photometric="MINISBLACK"
72
+ # )
73
+
74
+ # with rio.open(full_outname, "w", **profile) as dst:
75
+ # dst.write(src.read())
76
+
77
+ def download_manifests(
78
+ manifests: list[Dict[str, Any]],
79
+ full_outname: pathlib.Path,
80
+ max_workers: int,
81
+ ) -> None:
82
+ """Download every manifest in *manifests* concurrently.
83
+
84
+ Each output file is saved in the folder
85
+ ``full_outname.parent/full_outname.stem`` with names ``000000.tif``,
86
+ ``000001.tif`` … according to the list order.
87
+ """
88
+ tmp_dir = pathlib.Path(tempfile.mkdtemp(prefix="cubexpress_"))
89
+ full_outname_temp = tmp_dir / full_outname.stem
90
+ full_outname_temp.mkdir(parents=True, exist_ok=True)
91
+
92
+ with ThreadPoolExecutor(max_workers=max_workers) as exe: # -
93
+ futures = {
94
+ exe.submit(
95
+ download_manifest,
96
+ ulist=umanifest,
97
+ full_outname=full_outname_temp / f"{index:06d}.tif"
98
+ ): umanifest for index, umanifest in enumerate(manifests)
99
+ }
100
+ for future in as_completed(futures):
101
+ try:
102
+ future.result()
103
+ except Exception as exc:
104
+ print(f"Error in one of the downloads: {exc}")
105
+
106
+ if full_outname_temp.exists():
107
+ input_files = sorted(full_outname_temp.glob("*.tif"))
108
+ merge_tifs(input_files, full_outname)
109
+ shutil.rmtree(full_outname_temp)
110
+ else:
111
+ raise ValueError(f"Error in {full_outname}")
@@ -57,8 +57,6 @@ def _square_roi(lon: float, lat: float, edge_size: int, scale: int) -> ee.Geomet
57
57
  point = ee.Geometry.Point([lon, lat])
58
58
  return point.buffer(half).bounds()
59
59
 
60
-
61
-
62
60
  def merge_tifs(
63
61
  input_files: list[pathlib.Path],
64
62
  output_path: pathlib.Path,
@@ -259,13 +259,8 @@ class RequestSet(BaseModel):
259
259
  def create_manifests(self) -> pd.DataFrame:
260
260
  """
261
261
  Exports the raster metadata to a pandas DataFrame.
262
-
263
262
  Returns:
264
263
  pd.DataFrame: A DataFrame containing the metadata for all entries.
265
-
266
- Example:
267
- >>> df = raster_transform_set.export_df()
268
- >>> print(df)
269
264
  """
270
265
  # Use ProcessPoolExecutor for CPU-bound tasks to convert raster transforms to lon/lat
271
266
  with ProcessPoolExecutor(max_workers=None) as executor:
@@ -11,9 +11,9 @@ from cubexpress.conversion import lonlat2rt
11
11
 
12
12
 
13
13
  def table_to_requestset(
14
- table: pd.DataFrame,
15
- mosaic: bool = True
16
- ) -> RequestSet:
14
+ table: pd.DataFrame,
15
+ mosaic: bool = True
16
+ ) -> RequestSet:
17
17
  """Return a :class:`RequestSet` built from *df* (cloud_table result).
18
18
 
19
19
  Parameters
@@ -35,7 +35,7 @@ def table_to_requestset(
35
35
  df = table.copy()
36
36
 
37
37
  if df.empty:
38
- raise ValueError("cloud_table returned no rows; nothing to request.")
38
+ raise ValueError("There are no images in the requested period. Please check your dates or your ubication.")
39
39
 
40
40
  rt = lonlat2rt(
41
41
  lon=df.attrs["lon"],
@@ -43,11 +43,11 @@ def table_to_requestset(
43
43
  edge_size=df.attrs["edge_size"],
44
44
  scale=df.attrs["scale"],
45
45
  )
46
+
46
47
  centre_hash = pgh.encode(df.attrs["lat"], df.attrs["lon"], precision=5)
47
- reqs: list[Request] = []
48
+ reqs = []
48
49
 
49
50
  if mosaic:
50
-
51
51
  grouped = (
52
52
  df.groupby('date')
53
53
  .agg(
@@ -66,8 +66,7 @@ def table_to_requestset(
66
66
  )
67
67
 
68
68
  for day, row in grouped.iterrows():
69
-
70
-
69
+
71
70
  img_ids = row["id_list"]
72
71
  cdf = row["cs_cdf_mean"]
73
72
 
@@ -100,16 +99,16 @@ def table_to_requestset(
100
99
  else:
101
100
  for _, row in df.iterrows():
102
101
  img_id = row["id"]
103
- # tile = img_id.split("_")[-1][1:]
102
+ tile = img_id.split("_")[-1][1:]
104
103
  day = row["date"]
105
104
  cdf = int(round(row["cs_cdf"], 2) * 100)
106
105
  reqs.append(
107
106
  Request(
108
- id=f"{day}_{centre_hash}_{cdf}",
107
+ id=f"{day}_{tile}_{cdf}",
109
108
  raster_transform=rt,
110
109
  image=f"{df.attrs['collection']}/{img_id}",
111
110
  bands=df.attrs["bands"],
112
111
  )
113
112
  )
114
113
 
115
- return RequestSet(requestset=reqs)
114
+ return RequestSet(requestset=reqs)
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "cubexpress"
3
- version = "0.1.11"
3
+ version = "0.1.12"
4
4
  description = "Efficient processing of cubic Earth-observation (EO) data."
5
5
  authors = [
6
6
  "Julio Contreras <contrerasnetk@gmail.com>",
@@ -1,119 +0,0 @@
1
- """Low-level download helpers for Earth Engine manifests.
2
-
3
- Only two public callables are exposed:
4
-
5
- * :func:`download_manifest` – fetch a single manifest and write one GeoTIFF.
6
- * :func:`download_manifests` – convenience wrapper to parallel-download a list
7
- of manifests with a thread pool.
8
-
9
- Both functions are fully I/O bound; no return value is expected.
10
- """
11
-
12
- from __future__ import annotations
13
-
14
- import json
15
- import pathlib
16
- import concurrent.futures
17
- from copy import deepcopy
18
- from typing import Any, Dict, List
19
-
20
- import ee
21
- import rasterio as rio
22
- from rasterio.io import MemoryFile
23
- import logging
24
- from rasterio.merge import merge
25
- from rasterio.enums import Resampling
26
- import os
27
- import shutil
28
- import tempfile
29
- from cubexpress.geospatial import merge_tifs
30
-
31
- os.environ['CPL_LOG_ERRORS'] = 'OFF'
32
- logging.getLogger('rasterio._env').setLevel(logging.ERROR)
33
-
34
- def download_manifest(ulist: Dict[str, Any], full_outname: pathlib.Path) -> None:
35
- """Download *ulist* and save it as *full_outname*.
36
-
37
- The manifest must include either an ``assetId`` or an ``expression``
38
- (serialized EE image). RasterIO is used to write a tiled, compressed
39
- GeoTIFF; the function is silent apart from the final ``print``.
40
- """
41
- if "assetId" in ulist:
42
- images_bytes = ee.data.getPixels(ulist)
43
- elif "expression" in ulist:
44
- ee_image = ee.deserializer.decode(json.loads(ulist["expression"]))
45
- ulist_deep = deepcopy(ulist)
46
- ulist_deep["expression"] = ee_image
47
- images_bytes = ee.data.computePixels(ulist_deep)
48
- else: # pragma: no cover
49
- raise ValueError("Manifest does not contain 'assetId' or 'expression'")
50
-
51
- with MemoryFile(images_bytes) as memfile:
52
- with memfile.open() as src:
53
- profile = src.profile
54
- profile.update(
55
- driver="GTiff",
56
- tiled=True,
57
- interleave="band",
58
- blockxsize=256,
59
- blockysize=256,
60
- compress="ZSTD",
61
- zstd_level=13,
62
- predictor=2,
63
- num_threads=20,
64
- nodata=65535,
65
- dtype="uint16",
66
- count=12,
67
- photometric="MINISBLACK"
68
- )
69
-
70
- with rio.open(full_outname, "w", **profile) as dst:
71
- dst.write(src.read())
72
-
73
- def download_manifests(
74
- manifests: list[Dict[str, Any]],
75
- full_outname: pathlib.Path,
76
- join: bool = True,
77
- max_workers: int = 4,
78
- ) -> None:
79
- """Download every manifest in *manifests* concurrently.
80
-
81
- Each output file is saved in the folder
82
- ``full_outname.parent/full_outname.stem`` with names ``000000.tif``,
83
- ``000001.tif`` … according to the list order.
84
- """
85
- # full_outname = pathlib.Path("/home/contreras/Documents/GitHub/cubexpress/cubexpress_test/2017-08-19_6mfrw_18LVN.tif")
86
-
87
- if join:
88
- tmp_dir = pathlib.Path(tempfile.mkdtemp(prefix="s2tmp_"))
89
- full_outname_temp = tmp_dir / full_outname.name
90
-
91
- with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
92
- futures = []
93
-
94
- for index, umanifest in enumerate(manifests):
95
- folder = full_outname_temp.parent / full_outname_temp.stem
96
- folder.mkdir(parents=True, exist_ok=True)
97
- outname = folder / f"{index:06d}.tif"
98
- futures.append(
99
- executor.submit(
100
- download_manifest,
101
- umanifest, # ulist = umanifest
102
- outname # full_outname = outname
103
- )
104
- )
105
-
106
- for fut in concurrent.futures.as_completed(futures):
107
- try:
108
- fut.result()
109
- except Exception as exc: # noqa: BLE001
110
- print(f"Error en una de las descargas: {exc}") # noqa: T201
111
-
112
-
113
- dir_path = full_outname_temp.parent / full_outname_temp.stem
114
- if dir_path.exists():
115
- input_files = sorted(dir_path.glob("*.tif"))
116
- merge_tifs(input_files, full_outname)
117
- shutil.rmtree(dir_path)
118
- else:
119
- raise ValueError(f"Error in {full_outname}")
File without changes