PyGeoFetch 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pygeofetch/__init__.py +1 -1
- pygeofetch/cli/search_commands.py +30 -1
- pygeofetch/core/authenticator.py +1 -1
- pygeofetch/core/downloader.py +398 -54
- pygeofetch/core/logging.py +1 -1
- pygeofetch/core/orbits.py +1 -1
- pygeofetch/insar/__init__.py +1 -1
- pygeofetch/insar/atmosphere.py +1 -1
- pygeofetch/insar/interferogram.py +1 -1
- pygeofetch/insar/timeseries.py +1 -1
- pygeofetch/meta.yaml +1 -1
- pygeofetch/models/satellite_data.py +2 -2
- pygeofetch/processing/preprocessor.py +35 -1
- pygeofetch/processor/__init__.py +4 -1
- pygeofetch/processor/landsat.py +71 -12
- pygeofetch/processor/timeseries.py +400 -0
- pygeofetch/providers/airbus_oneatlas.py +20 -4
- pygeofetch/providers/aws_earth.py +36 -8
- pygeofetch/providers/copernicus.py +1 -1
- pygeofetch/providers/esa_scihub.py +20 -4
- pygeofetch/providers/google_earth_engine.py +20 -4
- pygeofetch/providers/usgs.py +162 -16
- pygeofetch/viz/plot.py +446 -28
- {pygeofetch-1.2.0.dist-info → pygeofetch-1.3.0.dist-info}/METADATA +20 -1
- {pygeofetch-1.2.0.dist-info → pygeofetch-1.3.0.dist-info}/RECORD +29 -28
- {pygeofetch-1.2.0.dist-info → pygeofetch-1.3.0.dist-info}/WHEEL +0 -0
- {pygeofetch-1.2.0.dist-info → pygeofetch-1.3.0.dist-info}/entry_points.txt +0 -0
- {pygeofetch-1.2.0.dist-info → pygeofetch-1.3.0.dist-info}/licenses/LICENSE +0 -0
- {pygeofetch-1.2.0.dist-info → pygeofetch-1.3.0.dist-info}/top_level.txt +0 -0
pygeofetch/__init__.py
CHANGED
|
@@ -53,6 +53,22 @@ def search() -> None:
|
|
|
53
53
|
default=None,
|
|
54
54
|
help="Comma-separated satellite names e.g. Sentinel-2,Landsat-8.",
|
|
55
55
|
)
|
|
56
|
+
@click.option(
|
|
57
|
+
"--product-type",
|
|
58
|
+
default=None,
|
|
59
|
+
help=(
|
|
60
|
+
"SAR product type e.g. GRD, SLC, GRD-COG. Currently only "
|
|
61
|
+
"meaningfully filters Copernicus (OData-based, not STAC) — "
|
|
62
|
+
"STAC providers like planetary_computer/element84 offer only "
|
|
63
|
+
"pre-processed Sentinel-1 RTC via their collection mapping, not "
|
|
64
|
+
"raw GRD/SLC, so this has no effect there regardless of value."
|
|
65
|
+
),
|
|
66
|
+
)
|
|
67
|
+
@click.option(
|
|
68
|
+
"--polarisation",
|
|
69
|
+
default=None,
|
|
70
|
+
help='SAR polarisation e.g. VV, VH, HH, HV. Same Copernicus-only caveat as --product-type.',
|
|
71
|
+
)
|
|
56
72
|
@click.option("--max-results", "-n", default=100, show_default=True)
|
|
57
73
|
@click.option(
|
|
58
74
|
"--sort-by",
|
|
@@ -98,6 +114,8 @@ def search_run(
|
|
|
98
114
|
processing_level,
|
|
99
115
|
providers,
|
|
100
116
|
satellites,
|
|
117
|
+
product_type,
|
|
118
|
+
polarisation,
|
|
101
119
|
max_results,
|
|
102
120
|
sort_by,
|
|
103
121
|
sort_order,
|
|
@@ -196,7 +214,16 @@ def search_run(
|
|
|
196
214
|
|
|
197
215
|
query = SearchQuery(
|
|
198
216
|
bbox=bbox_obj,
|
|
199
|
-
geometry_geojson=geometry_geojson,
|
|
217
|
+
# Was "geometry_geojson=geometry_geojson" — a SEPARATE, unused
|
|
218
|
+
# field that nothing in the codebase actually reads. Every
|
|
219
|
+
# provider (and the model's own to_stac_params()/
|
|
220
|
+
# has_spatial_filter() methods) reads query.geometry, not
|
|
221
|
+
# query.geometry_geojson. This meant --geometry-file silently
|
|
222
|
+
# never applied real polygon-based spatial filtering at all —
|
|
223
|
+
# it fell back to the bbox approximation computed just above
|
|
224
|
+
# (still functional, but loses the actual polygon precision
|
|
225
|
+
# that geometry-based filtering exists to provide).
|
|
226
|
+
geometry=geometry_geojson,
|
|
200
227
|
start_date=start_date,
|
|
201
228
|
end_date=end_date,
|
|
202
229
|
cloud_cover_min=cloud_min,
|
|
@@ -205,6 +232,8 @@ def search_run(
|
|
|
205
232
|
resolution_max_m=res_max,
|
|
206
233
|
processing_levels=pl_list,
|
|
207
234
|
satellites=sat_list,
|
|
235
|
+
product_type=product_type,
|
|
236
|
+
polarisation=polarisation,
|
|
208
237
|
max_results=max_results,
|
|
209
238
|
providers=provider_list or [],
|
|
210
239
|
sort_by=sort_by,
|
pygeofetch/core/authenticator.py
CHANGED
pygeofetch/core/downloader.py
CHANGED
|
@@ -25,7 +25,7 @@ import hashlib
|
|
|
25
25
|
import time
|
|
26
26
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
27
27
|
from pathlib import Path
|
|
28
|
-
from typing import TYPE_CHECKING, Any, Callable
|
|
28
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict
|
|
29
29
|
|
|
30
30
|
from pygeofetch.core.logging import DownloadProgress, get_logger
|
|
31
31
|
from pygeofetch.models.download_task import (
|
|
@@ -41,6 +41,43 @@ if TYPE_CHECKING:
|
|
|
41
41
|
logger = get_logger(__name__)
|
|
42
42
|
|
|
43
43
|
|
|
44
|
+
# Required common-band-name inputs for each single-date spectral index
|
|
45
|
+
# supported by pygeofetch.processor.SpectralIndex's built-in (non-spyndex)
|
|
46
|
+
# formulae. Matches _BUILTIN in pygeofetch/processor/indices.py exactly —
|
|
47
|
+
# keep these in sync if that dict changes. dNBR is deliberately excluded:
|
|
48
|
+
# it needs paired pre/post-event bands, which a single downloaded scene
|
|
49
|
+
# can't supply (handled as an explicit error instead, see _apply_action).
|
|
50
|
+
_SINGLE_DATE_INDEX_BANDS: dict = {
|
|
51
|
+
"EVI": ("NIR", "RED", "BLUE"),
|
|
52
|
+
"SAVI": ("NIR", "RED"),
|
|
53
|
+
"NDWI": ("GREEN", "NIR"),
|
|
54
|
+
"MNDWI": ("GREEN", "SWIR1"),
|
|
55
|
+
"NDBI": ("SWIR1", "NIR"),
|
|
56
|
+
"NDSI": ("GREEN", "SWIR1"),
|
|
57
|
+
"NDMI": ("NIR", "SWIR1"),
|
|
58
|
+
"NBR": ("NIR", "SWIR2"),
|
|
59
|
+
"BSI": ("SWIR1", "RED", "NIR", "BLUE"),
|
|
60
|
+
"ARVI": ("NIR", "RED", "BLUE"),
|
|
61
|
+
"GNDVI": ("NIR", "GREEN"),
|
|
62
|
+
"RVI": ("NIR", "RED"),
|
|
63
|
+
"VCI": ("NIR", "RED", "BLUE"),
|
|
64
|
+
"CRI1": ("BLUE", "GREEN"),
|
|
65
|
+
"PSRI": ("RED", "BLUE", "NIR"),
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
# Common band name -> substrings to look for in a downloaded filename.
|
|
69
|
+
# Covers Sentinel-2 band codes and generic common names, matching the
|
|
70
|
+
# pattern already used by the hardcoded ndvi handler ("B04"/"red").
|
|
71
|
+
_BAND_FILENAME_PATTERNS: dict = {
|
|
72
|
+
"BLUE": ("B02", "blue"),
|
|
73
|
+
"GREEN": ("B03", "green"),
|
|
74
|
+
"RED": ("B04", "red"),
|
|
75
|
+
"NIR": ("B08", "nir"),
|
|
76
|
+
"SWIR1": ("B11", "swir1"),
|
|
77
|
+
"SWIR2": ("B12", "swir2"),
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
44
81
|
class AdaptiveDownloader:
|
|
45
82
|
"""
|
|
46
83
|
Resilient parallel download manager.
|
|
@@ -154,26 +191,59 @@ class AdaptiveDownloader:
|
|
|
154
191
|
f"File validation failed for {data.id!r}: {err_msg}"
|
|
155
192
|
)
|
|
156
193
|
break
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
194
|
+
|
|
195
|
+
# Check result.success ONCE here, AFTER validation may have
|
|
196
|
+
# changed it — not nested inside the now-stale `if
|
|
197
|
+
# result.success:` above. Previously the success-logging
|
|
198
|
+
# and `return result` sat unconditionally inside that outer
|
|
199
|
+
# block, so a validation failure that flipped success to
|
|
200
|
+
# False partway through still fell through to logging "✓
|
|
201
|
+
# success" and returning immediately — never retrying, and
|
|
202
|
+
# misreporting a detected truncated download as if it had
|
|
203
|
+
# worked.
|
|
204
|
+
if result.success:
|
|
205
|
+
if options.verify_checksum:
|
|
206
|
+
result = self._verify_checksums(result, data, options)
|
|
207
|
+
if options.post_process:
|
|
208
|
+
result = self._run_post_process(result, options)
|
|
162
209
|
size_mb = (
|
|
163
210
|
result.bytes_downloaded / (1024 * 1024)
|
|
164
211
|
if result.bytes_downloaded
|
|
165
212
|
else 0
|
|
166
213
|
)
|
|
167
|
-
(
|
|
168
|
-
size_mb / result.duration_seconds
|
|
169
|
-
) if result.duration_seconds > 0 else 0
|
|
170
214
|
logger.info(
|
|
171
215
|
" ✓ %-45s %6.0f MB %5.1fs",
|
|
172
216
|
str(data.id)[:45],
|
|
173
217
|
size_mb,
|
|
174
218
|
result.duration_seconds,
|
|
175
219
|
)
|
|
176
|
-
|
|
220
|
+
return result
|
|
221
|
+
|
|
222
|
+
# result.success is False here — either the provider itself
|
|
223
|
+
# reported failure, or file validation caught a corrupt/
|
|
224
|
+
# truncated download above. Previously this fell through to
|
|
225
|
+
# an unconditional `return result` regardless of success,
|
|
226
|
+
# meaning a detected truncated download gave up immediately
|
|
227
|
+
# on attempt 1 and never used retry_attempts at all — the
|
|
228
|
+
# exact case a retry is most likely to actually fix, since
|
|
229
|
+
# network truncation is typically transient. Route through
|
|
230
|
+
# the same backoff-and-retry path a raised exception would
|
|
231
|
+
# take, instead of returning early.
|
|
232
|
+
if attempt < options.retry_attempts:
|
|
233
|
+
delay = self._backoff(attempt, options)
|
|
234
|
+
logger.warning(
|
|
235
|
+
f"Download attempt {attempt + 1}/{options.retry_attempts + 1} "
|
|
236
|
+
f"for {data.id!r} failed validation: {result.error}. "
|
|
237
|
+
f"Retrying in {delay:.1f}s..."
|
|
238
|
+
)
|
|
239
|
+
time.sleep(delay)
|
|
240
|
+
continue
|
|
241
|
+
else:
|
|
242
|
+
logger.error(
|
|
243
|
+
f"Download failed validation after "
|
|
244
|
+
f"{options.retry_attempts + 1} attempts: {result.error}"
|
|
245
|
+
)
|
|
246
|
+
return result
|
|
177
247
|
except Exception as exc:
|
|
178
248
|
if attempt < options.retry_attempts:
|
|
179
249
|
delay = self._backoff(attempt, options)
|
|
@@ -304,7 +374,7 @@ class AdaptiveDownloader:
|
|
|
304
374
|
if r and r.success and r.bytes_downloaded
|
|
305
375
|
)
|
|
306
376
|
# summary printed by DownloadProgress footer
|
|
307
|
-
return results
|
|
377
|
+
return results
|
|
308
378
|
|
|
309
379
|
def _get_provider(self, provider_id: str) -> Any:
|
|
310
380
|
"""Get a provider instance with a fresh authenticated session."""
|
|
@@ -362,7 +432,23 @@ class AdaptiveDownloader:
|
|
|
362
432
|
try:
|
|
363
433
|
result = self._apply_action(result, action)
|
|
364
434
|
except Exception as exc:
|
|
365
|
-
|
|
435
|
+
# Rasterio/GDAL errors frequently say "see previous
|
|
436
|
+
# exception for details" — that previous exception is
|
|
437
|
+
# exc.__cause__/__context__, which str(exc) alone does NOT
|
|
438
|
+
# include. Logging only str(exc) (the previous behavior)
|
|
439
|
+
# made that message actively misleading: it points at
|
|
440
|
+
# information the log line doesn't actually contain.
|
|
441
|
+
detail = ""
|
|
442
|
+
cause = exc.__cause__ or exc.__context__
|
|
443
|
+
if cause is not None and str(cause) != str(exc):
|
|
444
|
+
detail = f" | caused by: {type(cause).__name__}: {cause}"
|
|
445
|
+
logger.warning(
|
|
446
|
+
f"Post-process action {action.action!r} failed: {exc}{detail}"
|
|
447
|
+
)
|
|
448
|
+
logger.debug(
|
|
449
|
+
f"Post-process action {action.action!r} full traceback:",
|
|
450
|
+
exc_info=exc,
|
|
451
|
+
)
|
|
366
452
|
result.post_process_completed.append("completed")
|
|
367
453
|
return result
|
|
368
454
|
|
|
@@ -382,6 +468,57 @@ class AdaptiveDownloader:
|
|
|
382
468
|
if new_paths:
|
|
383
469
|
result.output_path = new_paths[0]
|
|
384
470
|
|
|
471
|
+
elif action.action == "cloud_mask":
|
|
472
|
+
try:
|
|
473
|
+
from pygeofetch.processing.preprocessor import Preprocessor
|
|
474
|
+
|
|
475
|
+
method = action.params.get("method", "scl")
|
|
476
|
+
cloud_classes = action.params.get("cloud_classes")
|
|
477
|
+
|
|
478
|
+
scl_path = None
|
|
479
|
+
if method == "scl":
|
|
480
|
+
scl_path = next(
|
|
481
|
+
(p for p in result.output_paths if "scl" in p.name.lower()),
|
|
482
|
+
None,
|
|
483
|
+
)
|
|
484
|
+
if scl_path is None:
|
|
485
|
+
raise ValueError(
|
|
486
|
+
"cloud_mask (method=scl) requires an SCL band to be "
|
|
487
|
+
'downloaded alongside your other bands — add "SCL" '
|
|
488
|
+
'to --bands, e.g. --bands "B04,B08,SCL"'
|
|
489
|
+
)
|
|
490
|
+
|
|
491
|
+
pp = Preprocessor()
|
|
492
|
+
new_paths = []
|
|
493
|
+
for path in result.output_paths:
|
|
494
|
+
if path == scl_path:
|
|
495
|
+
# Keep the SCL band itself untouched — it's the mask
|
|
496
|
+
# SOURCE, not something to mask against itself.
|
|
497
|
+
new_paths.append(path)
|
|
498
|
+
continue
|
|
499
|
+
if path.suffix.lower() in (".tif", ".tiff"):
|
|
500
|
+
masked = pp.cloud_mask(
|
|
501
|
+
path,
|
|
502
|
+
method=method,
|
|
503
|
+
scl_band=scl_path,
|
|
504
|
+
cloud_classes=cloud_classes,
|
|
505
|
+
)
|
|
506
|
+
if masked.success and masked.output_path:
|
|
507
|
+
new_paths.append(masked.output_path)
|
|
508
|
+
else:
|
|
509
|
+
logger.warning(
|
|
510
|
+
f"cloud_mask failed for {path.name}, keeping "
|
|
511
|
+
f"original: {masked.error}"
|
|
512
|
+
)
|
|
513
|
+
new_paths.append(path)
|
|
514
|
+
else:
|
|
515
|
+
new_paths.append(path)
|
|
516
|
+
result.output_paths = new_paths
|
|
517
|
+
except ImportError:
|
|
518
|
+
logger.warning(
|
|
519
|
+
"processing extras not installed; skipping cloud_mask action"
|
|
520
|
+
)
|
|
521
|
+
|
|
385
522
|
elif action.action == "reproject":
|
|
386
523
|
# Requires rasterio - skip if not available
|
|
387
524
|
target_crs = action.params.get("value", "EPSG:4326")
|
|
@@ -390,16 +527,37 @@ class AdaptiveDownloader:
|
|
|
390
527
|
from rasterio.warp import Resampling, reproject # noqa: F401
|
|
391
528
|
|
|
392
529
|
new_paths = []
|
|
530
|
+
any_failed = False
|
|
393
531
|
for path in result.output_paths:
|
|
394
532
|
if path.suffix.lower() in (".tif", ".tiff"):
|
|
395
533
|
out_path = path.with_stem(
|
|
396
534
|
f"{path.stem}_{target_crs.replace(':', '_')}"
|
|
397
535
|
)
|
|
398
|
-
|
|
399
|
-
|
|
536
|
+
try:
|
|
537
|
+
self._reproject_with_validation(path, out_path, target_crs)
|
|
538
|
+
new_paths.append(out_path)
|
|
539
|
+
except Exception as exc:
|
|
540
|
+
# A single file failing must not discard the
|
|
541
|
+
# other files' successful reprojections — keep
|
|
542
|
+
# the ORIGINAL for this one file, not the
|
|
543
|
+
# (already-deleted) failed output, and continue.
|
|
544
|
+
logger.warning(
|
|
545
|
+
f"Reproject failed for {path.name}, keeping "
|
|
546
|
+
f"original: {exc}"
|
|
547
|
+
)
|
|
548
|
+
new_paths.append(path)
|
|
549
|
+
any_failed = True
|
|
400
550
|
else:
|
|
401
551
|
new_paths.append(path)
|
|
402
552
|
result.output_paths = new_paths
|
|
553
|
+
if any_failed:
|
|
554
|
+
# Ensure downstream log context makes clear this scene's
|
|
555
|
+
# reproject step was only partially successful, rather
|
|
556
|
+
# than looking identical to a fully clean run.
|
|
557
|
+
logger.warning(
|
|
558
|
+
"reproject: one or more files kept their original "
|
|
559
|
+
"CRS after a per-file failure — see warnings above."
|
|
560
|
+
)
|
|
403
561
|
except ImportError:
|
|
404
562
|
logger.warning("rasterio not installed; skipping reproject action")
|
|
405
563
|
|
|
@@ -599,6 +757,7 @@ class AdaptiveDownloader:
|
|
|
599
757
|
if red_path and nir_path:
|
|
600
758
|
with rasterio.open(red_path) as rs:
|
|
601
759
|
red = rs.read(1).astype(np.float32)
|
|
760
|
+
red_nodata = rs.nodata
|
|
602
761
|
profile = rs.profile.copy()
|
|
603
762
|
with rasterio.open(nir_path) as ns:
|
|
604
763
|
nir = ns.read(
|
|
@@ -606,14 +765,30 @@ class AdaptiveDownloader:
|
|
|
606
765
|
out_shape=red.shape,
|
|
607
766
|
resampling=rasterio.enums.Resampling.bilinear,
|
|
608
767
|
).astype(np.float32)
|
|
768
|
+
nir_nodata = ns.nodata
|
|
769
|
+
|
|
770
|
+
# Respect each source band's own nodata flag (e.g. a
|
|
771
|
+
# cloud_mask step earlier in the chain, or a provider's
|
|
772
|
+
# native fill value) — without this, masked/fill pixels
|
|
773
|
+
# get computed as if they were real reflectance values
|
|
774
|
+
# instead of correctly propagating as "no data" through
|
|
775
|
+
# to the NDVI output. Previously neither red nor nir
|
|
776
|
+
# nodata was checked at all here.
|
|
777
|
+
if red_nodata is not None:
|
|
778
|
+
red = np.where(red == red_nodata, np.nan, red)
|
|
779
|
+
if nir_nodata is not None:
|
|
780
|
+
nir = np.where(nir == nir_nodata, np.nan, nir)
|
|
781
|
+
|
|
609
782
|
with np.errstate(divide="ignore", invalid="ignore"):
|
|
610
783
|
ndvi = np.where(
|
|
611
|
-
nir + red > 0, (nir - red) / (nir + red),
|
|
784
|
+
nir + red > 0, (nir - red) / (nir + red), np.nan
|
|
612
785
|
)
|
|
786
|
+
ndvi = np.where(np.isnan(red) | np.isnan(nir), np.nan, ndvi)
|
|
613
787
|
ndvi_path = red_path.parent / "ndvi.tif"
|
|
614
788
|
profile.update(count=1, dtype="float32", nodata=-9999.0)
|
|
789
|
+
ndvi_write = np.where(np.isnan(ndvi), -9999.0, ndvi)
|
|
615
790
|
with rasterio.open(ndvi_path, "w", **profile) as dst:
|
|
616
|
-
dst.write(
|
|
791
|
+
dst.write(ndvi_write[np.newaxis, :, :])
|
|
617
792
|
(result.output_paths or []).append(ndvi_path)
|
|
618
793
|
result.output_path = ndvi_path
|
|
619
794
|
logger.info(f"NDVI computed → {ndvi_path}")
|
|
@@ -658,8 +833,124 @@ class AdaptiveDownloader:
|
|
|
658
833
|
except ImportError:
|
|
659
834
|
logger.warning("rasterio not installed; skipping merge action")
|
|
660
835
|
|
|
836
|
+
elif action.action.upper() in _SINGLE_DATE_INDEX_BANDS:
|
|
837
|
+
# Generic handler for any single-date spectral index beyond the
|
|
838
|
+
# hardcoded "ndvi" branch above (EVI, SAVI, NDWI, MNDWI, NDBI,
|
|
839
|
+
# NDSI, NDMI, NBR, BSI, ARVI, GNDVI, RVI, VCI, CRI1, PSRI).
|
|
840
|
+
#
|
|
841
|
+
# Previously, requesting any of these via --post-process fell
|
|
842
|
+
# through every branch with no match and hit `return result`
|
|
843
|
+
# unconditionally — a completely silent no-op. The user's
|
|
844
|
+
# download "succeeded" with the requested index never computed
|
|
845
|
+
# and zero indication anything was wrong. This handler and the
|
|
846
|
+
# explicit fallback error below close that gap: every index
|
|
847
|
+
# documented as supported now either actually runs, or fails
|
|
848
|
+
# with a clear, actionable message — never silently.
|
|
849
|
+
result = self._compute_single_date_index(result, action.action.upper())
|
|
850
|
+
|
|
851
|
+
elif action.action.upper() == "DNBR":
|
|
852
|
+
# dNBR needs two dates (pre/post) — the single-scene download
|
|
853
|
+
# post-process chain has no second scene to pair with, so this
|
|
854
|
+
# cannot be silently auto-wired the way the single-date indices
|
|
855
|
+
# above are. Fail clearly rather than either silently no-op or
|
|
856
|
+
# guess at pairing unrelated scenes.
|
|
857
|
+
raise ValueError(
|
|
858
|
+
"dNBR requires paired pre-event/post-event imagery and can't "
|
|
859
|
+
"run as a single-scene --post-process step. Use "
|
|
860
|
+
"pygeofetch.processor.SpectralIndex.compute('dNBR', "
|
|
861
|
+
"NIR_PRE=..., SWIR2_PRE=..., NIR_POST=..., SWIR2_POST=...) "
|
|
862
|
+
"directly with both dates' bands instead."
|
|
863
|
+
)
|
|
864
|
+
|
|
865
|
+
else:
|
|
866
|
+
known_actions = (
|
|
867
|
+
"unzip, reproject, compress, cog, clip, resample, merge, "
|
|
868
|
+
"ndvi, " + ", ".join(sorted(_SINGLE_DATE_INDEX_BANDS)).lower()
|
|
869
|
+
)
|
|
870
|
+
raise ValueError(
|
|
871
|
+
f"Unknown --post-process action {action.action!r}. "
|
|
872
|
+
f"Supported actions: {known_actions}."
|
|
873
|
+
)
|
|
874
|
+
|
|
875
|
+
return result
|
|
876
|
+
|
|
877
|
+
def _compute_single_date_index(
|
|
878
|
+
self, result: DownloadResult, index_name: str
|
|
879
|
+
) -> DownloadResult:
|
|
880
|
+
"""
|
|
881
|
+
Compute any single-date spectral index (beyond the hardcoded NDVI
|
|
882
|
+
branch) using pygeofetch.processor.SpectralIndex, matching required
|
|
883
|
+
bands to downloaded files the same way the ndvi branch does.
|
|
884
|
+
"""
|
|
885
|
+
import numpy as np
|
|
886
|
+
import rasterio
|
|
887
|
+
|
|
888
|
+
try:
|
|
889
|
+
from pygeofetch.processor import SpectralIndex
|
|
890
|
+
except ImportError as exc:
|
|
891
|
+
raise ImportError(
|
|
892
|
+
f'{index_name} requires the processor extra: '
|
|
893
|
+
f'pip install "pygeofetch[processor]"'
|
|
894
|
+
) from exc
|
|
895
|
+
|
|
896
|
+
required_bands = _SINGLE_DATE_INDEX_BANDS[index_name]
|
|
897
|
+
band_arrays: Dict[str, Any] = {}
|
|
898
|
+
profile = None
|
|
899
|
+
ref_shape = None
|
|
900
|
+
|
|
901
|
+
for band_name in required_bands:
|
|
902
|
+
patterns = _BAND_FILENAME_PATTERNS[band_name]
|
|
903
|
+
path = next(
|
|
904
|
+
(
|
|
905
|
+
p
|
|
906
|
+
for p in result.output_paths
|
|
907
|
+
if any(pat in p.name or pat.lower() in p.name.lower() for pat in patterns)
|
|
908
|
+
),
|
|
909
|
+
None,
|
|
910
|
+
)
|
|
911
|
+
if path is None:
|
|
912
|
+
raise ValueError(
|
|
913
|
+
f"{index_name}: could not find a downloaded band matching "
|
|
914
|
+
f"{band_name} (looked for {patterns} in filenames)"
|
|
915
|
+
)
|
|
916
|
+
with rasterio.open(path) as src:
|
|
917
|
+
arr = src.read(1, out_shape=ref_shape, resampling=rasterio.enums.Resampling.bilinear) \
|
|
918
|
+
if ref_shape else src.read(1)
|
|
919
|
+
if ref_shape is None:
|
|
920
|
+
ref_shape = arr.shape
|
|
921
|
+
profile = src.profile.copy()
|
|
922
|
+
arr = arr.astype(np.float32)
|
|
923
|
+
# Respect this band's own nodata flag — see the matching
|
|
924
|
+
# fix in the ndvi handler above for why this matters (a
|
|
925
|
+
# cloud_mask step earlier in the chain, or a provider's
|
|
926
|
+
# native fill value, must correctly propagate as "no data"
|
|
927
|
+
# rather than being computed as if it were a real value).
|
|
928
|
+
if src.nodata is not None:
|
|
929
|
+
arr = np.where(arr == src.nodata, np.nan, arr)
|
|
930
|
+
band_arrays[band_name] = arr
|
|
931
|
+
|
|
932
|
+
si = SpectralIndex(prefer_spyndex=False)
|
|
933
|
+
index_arr = np.asarray(si.compute(index_name, **band_arrays), dtype=np.float32)
|
|
934
|
+
# Any band being NaN at a pixel must make the index NaN there too,
|
|
935
|
+
# even if the formula's arithmetic happens to produce a finite
|
|
936
|
+
# result from a NaN input for some index shapes.
|
|
937
|
+
any_nan = np.zeros(ref_shape, dtype=bool)
|
|
938
|
+
for arr in band_arrays.values():
|
|
939
|
+
any_nan |= np.isnan(arr)
|
|
940
|
+
index_arr = np.where(any_nan, np.nan, index_arr)
|
|
941
|
+
|
|
942
|
+
out_path = result.output_paths[0].parent / f"{index_name.lower()}.tif"
|
|
943
|
+
profile.update(count=1, dtype="float32", nodata=-9999.0)
|
|
944
|
+
index_write = np.where(np.isnan(index_arr), -9999.0, index_arr)
|
|
945
|
+
with rasterio.open(out_path, "w", **profile) as dst:
|
|
946
|
+
dst.write(index_write[np.newaxis, :, :])
|
|
947
|
+
|
|
948
|
+
result.output_paths = list(result.output_paths) + [out_path]
|
|
949
|
+
result.output_path = out_path
|
|
950
|
+
logger.info(f"{index_name} computed → {out_path}")
|
|
661
951
|
return result
|
|
662
952
|
|
|
953
|
+
|
|
663
954
|
def _find_existing_download(
|
|
664
955
|
self, data: "SatelliteData", destination: Path
|
|
665
956
|
) -> Path | None:
|
|
@@ -816,13 +1107,42 @@ class AdaptiveDownloader:
|
|
|
816
1107
|
False,
|
|
817
1108
|
f"Raster has zero dimensions: {src.width}x{src.height}",
|
|
818
1109
|
)
|
|
819
|
-
# Read
|
|
1110
|
+
# Read multiple tiles spread across the file, not just
|
|
1111
|
+
# the first one. A truncated/incomplete download (a
|
|
1112
|
+
# dropped connection mid-stream, the single most common
|
|
1113
|
+
# real-world download corruption) cuts a file off from
|
|
1114
|
+
# the END — the header, TIFF directory, and early tiles
|
|
1115
|
+
# are almost always intact, while later tiles are
|
|
1116
|
+
# missing. Sampling only the first tile (the previous
|
|
1117
|
+
# behavior) is therefore nearly blind to exactly the
|
|
1118
|
+
# corruption pattern most likely to occur in practice:
|
|
1119
|
+
# confirmed by direct reproduction — an 80%-truncated
|
|
1120
|
+
# file opens fine, its dimensions read correctly, and
|
|
1121
|
+
# its first tile reads fine, but its last tile fails
|
|
1122
|
+
# with the same RasterioIOError downstream processing
|
|
1123
|
+
# (reproject/ndvi/cog) would hit minutes later.
|
|
820
1124
|
windows = list(src.block_windows(1))
|
|
821
1125
|
if windows:
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
1126
|
+
sample_indices = sorted(
|
|
1127
|
+
{0, len(windows) // 2, len(windows) - 1}
|
|
1128
|
+
)
|
|
1129
|
+
for idx in sample_indices:
|
|
1130
|
+
_, window = windows[idx]
|
|
1131
|
+
try:
|
|
1132
|
+
sample = src.read(1, window=window)
|
|
1133
|
+
except Exception as exc:
|
|
1134
|
+
return (
|
|
1135
|
+
False,
|
|
1136
|
+
f"Tile {idx + 1}/{len(windows)} read failed "
|
|
1137
|
+
f"(likely a truncated/incomplete download): "
|
|
1138
|
+
f"{exc}",
|
|
1139
|
+
)
|
|
1140
|
+
if sample is None or sample.size == 0:
|
|
1141
|
+
return (
|
|
1142
|
+
False,
|
|
1143
|
+
f"Tile {idx + 1}/{len(windows)} read returned "
|
|
1144
|
+
f"an empty array",
|
|
1145
|
+
)
|
|
826
1146
|
return True, ""
|
|
827
1147
|
except ImportError:
|
|
828
1148
|
# rasterio not installed — size check already passed
|
|
@@ -861,49 +1181,72 @@ class AdaptiveDownloader:
|
|
|
861
1181
|
"""
|
|
862
1182
|
Reproject source to target CRS and validate the output transform.
|
|
863
1183
|
Raises RuntimeError if reprojection produces an identity transform.
|
|
1184
|
+
|
|
1185
|
+
On ANY failure, the partially-written target file is deleted rather
|
|
1186
|
+
than left on disk — a previous version left corrupt/incomplete
|
|
1187
|
+
output files behind on failure, which downstream post-process
|
|
1188
|
+
actions (e.g. cog) would then silently pick up and fail on with a
|
|
1189
|
+
confusing, disconnected error, since nothing recorded that this
|
|
1190
|
+
file was the product of a failed operation.
|
|
864
1191
|
"""
|
|
865
1192
|
import rasterio
|
|
866
1193
|
from rasterio.enums import Resampling as RS
|
|
867
1194
|
from rasterio.warp import calculate_default_transform, reproject
|
|
868
1195
|
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
1196
|
+
try:
|
|
1197
|
+
with rasterio.open(str(source_path)) as src:
|
|
1198
|
+
src_crs = src.crs
|
|
1199
|
+
src_transform = src.transform
|
|
1200
|
+
src_width = src.width
|
|
1201
|
+
src_height = src.height
|
|
1202
|
+
|
|
1203
|
+
dst_transform, dst_width, dst_height = calculate_default_transform(
|
|
1204
|
+
src_crs, target_crs, src_width, src_height, *src.bounds
|
|
1205
|
+
)
|
|
1206
|
+
kwargs = src.meta.copy()
|
|
1207
|
+
kwargs.update(
|
|
1208
|
+
{
|
|
1209
|
+
"crs": target_crs,
|
|
1210
|
+
"transform": dst_transform,
|
|
1211
|
+
"width": dst_width,
|
|
1212
|
+
"height": dst_height,
|
|
1213
|
+
}
|
|
1214
|
+
)
|
|
1215
|
+
with rasterio.open(str(target_path), "w", **kwargs) as dst:
|
|
1216
|
+
for band_idx in range(1, src.count + 1):
|
|
1217
|
+
reproject(
|
|
1218
|
+
source=rasterio.band(src, band_idx),
|
|
1219
|
+
destination=rasterio.band(dst, band_idx),
|
|
1220
|
+
src_transform=src_transform,
|
|
1221
|
+
src_crs=src_crs,
|
|
1222
|
+
dst_transform=dst_transform,
|
|
1223
|
+
dst_crs=target_crs,
|
|
1224
|
+
resampling=RS.bilinear,
|
|
1225
|
+
)
|
|
1226
|
+
except Exception as exc:
|
|
1227
|
+
# Surface the REAL underlying error (GDAL's own message is
|
|
1228
|
+
# normally far more specific than a generic wrapper like
|
|
1229
|
+
# "Chunk and warp failed" suggests) rather than losing it —
|
|
1230
|
+
# previously this propagated up to be logged only as
|
|
1231
|
+
# "Post-process action 'reproject' failed: <str(exc)>", which
|
|
1232
|
+
# is exactly this message, but the caller's log line gave no
|
|
1233
|
+
# indication of *which* file/CRS/band was involved.
|
|
1234
|
+
target_path.unlink(missing_ok=True)
|
|
1235
|
+
raise RuntimeError(
|
|
1236
|
+
f"Reprojection failed for {source_path.name} -> {target_crs}: "
|
|
1237
|
+
f"{type(exc).__name__}: {exc}"
|
|
1238
|
+
) from exc
|
|
898
1239
|
|
|
899
1240
|
if self._has_identity_transform(target_path):
|
|
1241
|
+
target_path.unlink(missing_ok=True)
|
|
900
1242
|
msg = (
|
|
901
|
-
f"Reprojection produced identity transform in {target_path}
|
|
902
|
-
f"Source: {source_path}, Target CRS: {target_crs}. "
|
|
1243
|
+
f"Reprojection produced identity transform in {target_path.name} "
|
|
1244
|
+
f"(deleted). Source: {source_path.name}, Target CRS: {target_crs}. "
|
|
903
1245
|
"This is a known rasterio/GDAL edge case with certain input CRS."
|
|
904
1246
|
)
|
|
905
1247
|
raise RuntimeError(msg)
|
|
906
1248
|
|
|
1249
|
+
|
|
907
1250
|
def _backoff(self, attempt: int, options: DownloadOptions) -> float:
|
|
908
1251
|
"""Calculate retry delay based on strategy."""
|
|
909
1252
|
import random
|
|
@@ -925,7 +1268,8 @@ class AdaptiveDownloader:
|
|
|
925
1268
|
return
|
|
926
1269
|
# Emit lightweight progress update (counts only)
|
|
927
1270
|
n_done = sum(1 for r in results if r is not None)
|
|
928
|
-
|
|
1271
|
+
# n_done computed for the progress signal below, not used directly here
|
|
1272
|
+
_ = n_done
|
|
929
1273
|
progress = DownloadProgress(total=len(results), destination="")
|
|
930
1274
|
try:
|
|
931
1275
|
self.progress_callback(progress)
|
|
@@ -935,4 +1279,4 @@ class AdaptiveDownloader:
|
|
|
935
1279
|
@property
|
|
936
1280
|
def _result_total_size_mb(self) -> float:
|
|
937
1281
|
"""Property used by result objects."""
|
|
938
|
-
return 0.0
|
|
1282
|
+
return 0.0
|
pygeofetch/core/logging.py
CHANGED