PyGeoFetch 2.6.2.3__py3-none-any.whl → 2.6.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pygeofetch/__init__.py +1 -1
- pygeofetch/cli/main.py +3 -1
- pygeofetch/cli/optical_commands.py +105 -0
- pygeofetch/cli/search_commands.py +121 -8
- pygeofetch/core/data_organizer.py +364 -0
- pygeofetch/core/engine.py +7 -2
- pygeofetch/insar/preflight.py +136 -1
- pygeofetch/meta.yaml +1 -1
- pygeofetch/models/satellite_data.py +49 -0
- pygeofetch/optical/__init__.py +27 -0
- pygeofetch/optical/offset_tracking.py +486 -0
- pygeofetch/processor/indices.py +143 -11
- pygeofetch/providers/airbus_oneatlas.py +1 -1
- pygeofetch/providers/alaska_satellite_facility.py +202 -97
- pygeofetch/providers/digitalglobe.py +281 -112
- pygeofetch/providers/earth_explorer_additional.py +95 -245
- pygeofetch/providers/geoserver_generic.py +212 -91
- pygeofetch/providers/inpe_cbers.py +114 -96
- pygeofetch/providers/isro_bhuvan.py +201 -98
- pygeofetch/providers/jaxa_earth.py +170 -150
- pygeofetch/providers/maxar_gbdx.py +122 -90
- pygeofetch/validation/optical_validator.py +69 -33
- pygeofetch/viz/plot.py +24 -5
- {pygeofetch-2.6.2.3.dist-info → pygeofetch-2.6.2.4.dist-info}/METADATA +5 -5
- {pygeofetch-2.6.2.3.dist-info → pygeofetch-2.6.2.4.dist-info}/RECORD +29 -25
- {pygeofetch-2.6.2.3.dist-info → pygeofetch-2.6.2.4.dist-info}/WHEEL +0 -0
- {pygeofetch-2.6.2.3.dist-info → pygeofetch-2.6.2.4.dist-info}/entry_points.txt +0 -0
- {pygeofetch-2.6.2.3.dist-info → pygeofetch-2.6.2.4.dist-info}/licenses/LICENSE +0 -0
- {pygeofetch-2.6.2.3.dist-info → pygeofetch-2.6.2.4.dist-info}/top_level.txt +0 -0
pygeofetch/__init__.py
CHANGED
pygeofetch/cli/main.py
CHANGED
|
@@ -26,6 +26,7 @@ from pygeofetch.cli.auth_commands import auth
|
|
|
26
26
|
from pygeofetch.cli.config_commands import config
|
|
27
27
|
from pygeofetch.cli.download_commands import download
|
|
28
28
|
from pygeofetch.cli.index_commands import index
|
|
29
|
+
from pygeofetch.cli.optical_commands import optical
|
|
29
30
|
from pygeofetch.cli.pipeline_process_commands import proc_pipeline
|
|
30
31
|
from pygeofetch.cli.postprocess_commands import post
|
|
31
32
|
from pygeofetch.cli.preprocess_commands import preprocess
|
|
@@ -103,6 +104,7 @@ cli.add_command(index)
|
|
|
103
104
|
cli.add_command(post)
|
|
104
105
|
cli.add_command(sar)
|
|
105
106
|
cli.add_command(proc_pipeline)
|
|
107
|
+
cli.add_command(optical)
|
|
106
108
|
|
|
107
109
|
|
|
108
110
|
# ---------------------------------------------------------------------------
|
|
@@ -919,4 +921,4 @@ def main() -> None:
|
|
|
919
921
|
|
|
920
922
|
|
|
921
923
|
if __name__ == "__main__":
|
|
922
|
-
main()
|
|
924
|
+
main()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""CLI commands for optical pixel offset tracking (POT)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
|
|
10
|
+
console = Console()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@click.group()
|
|
14
|
+
def optical() -> None:
|
|
15
|
+
"""Optical pixel offset tracking — measures large 2D ground
|
|
16
|
+
displacement via image correlation, complementing InSAR for
|
|
17
|
+
motion that exceeds phase InSAR's coherence limit."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@optical.command("offset-track")
|
|
21
|
+
@click.option("--ref", "reference_path", required=True, type=click.Path(exists=True), help="Reference-date raster.")
|
|
22
|
+
@click.option("--sec", "secondary_path", required=True, type=click.Path(exists=True), help="Secondary-date raster.")
|
|
23
|
+
@click.option("--output", "-o", required=True, type=click.Path(), help="Output GeoTIFF (dx/dy/snr, 3 bands).")
|
|
24
|
+
@click.option("--band", default=1, show_default=True, type=int, help="1-based band index to correlate.")
|
|
25
|
+
@click.option("--window-size", default=64, show_default=True, type=int, help="Search window size, pixels.")
|
|
26
|
+
@click.option("--step-size", default=16, show_default=True, type=int, help="Spacing between window centers, pixels.")
|
|
27
|
+
@click.option("--chip-size", default=None, type=int, help="Reference template size, pixels. Default: window-size // 2.")
|
|
28
|
+
@click.option("--snr-threshold", default=3.0, show_default=True, type=float, help="Minimum SNR to mark a window reliable.")
|
|
29
|
+
@click.option("--cloud-mask", default=None, type=click.Path(exists=True), help="Real Sentinel-2 SCL raster, already aligned to --ref.")
|
|
30
|
+
@click.option("--water-mask-threshold", default=None, type=float, help="Mask pixels where NDWI exceeds this value (needs --green-band/--nir-band).")
|
|
31
|
+
@click.option("--green-band", default=None, type=int, help="1-based green band index, for --water-mask-threshold.")
|
|
32
|
+
@click.option("--nir-band", default=None, type=int, help="1-based NIR band index, for --water-mask-threshold.")
|
|
33
|
+
def offset_track_cmd(
|
|
34
|
+
reference_path,
|
|
35
|
+
secondary_path,
|
|
36
|
+
output,
|
|
37
|
+
band,
|
|
38
|
+
window_size,
|
|
39
|
+
step_size,
|
|
40
|
+
chip_size,
|
|
41
|
+
snr_threshold,
|
|
42
|
+
cloud_mask,
|
|
43
|
+
water_mask_threshold,
|
|
44
|
+
green_band,
|
|
45
|
+
nir_band,
|
|
46
|
+
):
|
|
47
|
+
"""
|
|
48
|
+
Measure 2D ground displacement between two optical scenes via
|
|
49
|
+
image correlation.
|
|
50
|
+
|
|
51
|
+
\b
|
|
52
|
+
Examples:
|
|
53
|
+
# Basic: two co-registered scenes, real cloud masking via SCL
|
|
54
|
+
pygeofetch optical offset-track \\
|
|
55
|
+
--ref before.tif --sec after.tif \\
|
|
56
|
+
--cloud-mask before_SCL.tif \\
|
|
57
|
+
--output displacement.tif
|
|
58
|
+
|
|
59
|
+
# Mining-scale displacement, larger search window
|
|
60
|
+
pygeofetch optical offset-track \\
|
|
61
|
+
--ref jan.tif --sec jun.tif \\
|
|
62
|
+
--window-size 96 --step-size 24 \\
|
|
63
|
+
--output mine_displacement.tif
|
|
64
|
+
"""
|
|
65
|
+
from pygeofetch.optical.offset_tracking import compute_pixel_offsets, prepare_optical_pair
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
ref_arr, sec_arr, ref_profile = prepare_optical_pair(
|
|
69
|
+
reference_path,
|
|
70
|
+
secondary_path,
|
|
71
|
+
band_index=band,
|
|
72
|
+
cloud_mask_path=cloud_mask,
|
|
73
|
+
water_mask_threshold=water_mask_threshold,
|
|
74
|
+
green_band_index=green_band,
|
|
75
|
+
nir_band_index=nir_band,
|
|
76
|
+
)
|
|
77
|
+
except ValueError as exc:
|
|
78
|
+
console.print(f"[red]✗[/] {exc}")
|
|
79
|
+
sys.exit(1)
|
|
80
|
+
|
|
81
|
+
pixel_size_m = abs(ref_profile["transform"].a)
|
|
82
|
+
result = compute_pixel_offsets(
|
|
83
|
+
ref_arr,
|
|
84
|
+
sec_arr,
|
|
85
|
+
pixel_size_m=pixel_size_m,
|
|
86
|
+
window_size=window_size,
|
|
87
|
+
step_size=step_size,
|
|
88
|
+
chip_size=chip_size,
|
|
89
|
+
snr_threshold=snr_threshold,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
out_transform = ref_profile["transform"] * ref_profile["transform"].scale(step_size, step_size)
|
|
93
|
+
out_profile = dict(ref_profile)
|
|
94
|
+
out_profile.update(
|
|
95
|
+
height=result.dx.shape[0],
|
|
96
|
+
width=result.dx.shape[1],
|
|
97
|
+
transform=out_transform,
|
|
98
|
+
)
|
|
99
|
+
written = result.export_geotiff(output, out_profile)
|
|
100
|
+
|
|
101
|
+
n_reliable = int(result.reliable.sum())
|
|
102
|
+
n_total = int(result.reliable.size)
|
|
103
|
+
console.print(f"[green]✓[/] offset-track → {written}")
|
|
104
|
+
console.print(f" {n_reliable}/{n_total} windows reliable ({100 * n_reliable / n_total:.1f}%)")
|
|
105
|
+
console.print(f" window={window_size}px step={step_size}px pixel_size={pixel_size_m:.2f}m")
|
|
@@ -282,6 +282,7 @@ def search_run(
|
|
|
282
282
|
|
|
283
283
|
sb = PyGeoFetch()
|
|
284
284
|
|
|
285
|
+
optical_config = None
|
|
285
286
|
if validate_optical and (
|
|
286
287
|
optical_max_cloud_cover,
|
|
287
288
|
optical_min_coverage,
|
|
@@ -298,22 +299,48 @@ def search_run(
|
|
|
298
299
|
overrides["required_bands"] = [
|
|
299
300
|
b.strip() for b in optical_required_bands.split(",")
|
|
300
301
|
]
|
|
301
|
-
|
|
302
|
+
optical_config = OpticalValidationConfig(**overrides)
|
|
303
|
+
sb.optical_validation_config = optical_config
|
|
302
304
|
|
|
303
305
|
with console.status(
|
|
304
306
|
f"[cyan]Searching {len(provider_list or [])} provider(s)...[/]"
|
|
305
307
|
):
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
308
|
+
if validate_optical:
|
|
309
|
+
# Real, deliberate choice: search WITHOUT filtering here, then
|
|
310
|
+
# validate directly in this CLI layer, rather than calling
|
|
311
|
+
# search(validate_optical=True) and losing visibility into
|
|
312
|
+
# what got rejected and why. sb.search()'s own contract
|
|
313
|
+
# (returns only the passed scenes) is left untouched for
|
|
314
|
+
# every other caller -- this is a CLI-only reporting path,
|
|
315
|
+
# not a change to the library's real filtering behavior.
|
|
316
|
+
raw_results = sb.search(
|
|
317
|
+
query,
|
|
318
|
+
providers=provider_list,
|
|
319
|
+
use_cache=not no_cache,
|
|
320
|
+
validate_optical=False,
|
|
321
|
+
)
|
|
322
|
+
results, validation_summary = _validate_and_summarize(
|
|
323
|
+
sb, raw_results, query, optical_config
|
|
324
|
+
)
|
|
325
|
+
else:
|
|
326
|
+
results = sb.search(
|
|
327
|
+
query,
|
|
328
|
+
providers=provider_list,
|
|
329
|
+
use_cache=not no_cache,
|
|
330
|
+
validate_optical=False,
|
|
331
|
+
)
|
|
332
|
+
validation_summary = None
|
|
312
333
|
|
|
313
334
|
if not results:
|
|
314
335
|
console.print("[yellow]No results found.[/]")
|
|
315
336
|
return
|
|
316
337
|
|
|
338
|
+
# Real, clean validation info + full results table, organized so the
|
|
339
|
+
# summary always appears before the detailed table -- not interleaved
|
|
340
|
+
# with save/format output below.
|
|
341
|
+
if validation_summary is not None:
|
|
342
|
+
_display_validation_summary(validation_summary)
|
|
343
|
+
|
|
317
344
|
# Save if requested
|
|
318
345
|
if output:
|
|
319
346
|
out_path = Path(output)
|
|
@@ -365,6 +392,92 @@ def search_run(
|
|
|
365
392
|
click.echo(r.id)
|
|
366
393
|
|
|
367
394
|
|
|
395
|
+
def _validate_and_summarize(sb, raw_results, query, optical_config):
|
|
396
|
+
"""
|
|
397
|
+
Real validation pass over every raw candidate, returning both the
|
|
398
|
+
scenes that actually pass and a real, structured summary of what got
|
|
399
|
+
rejected and why -- the information sb.search(validate_optical=True)
|
|
400
|
+
alone discards once it returns only the filtered list.
|
|
401
|
+
"""
|
|
402
|
+
from collections import Counter
|
|
403
|
+
|
|
404
|
+
from pygeofetch.validation import OpticalPreflightValidator, OpticalValidationConfig
|
|
405
|
+
|
|
406
|
+
validator = OpticalPreflightValidator(optical_config or OpticalValidationConfig())
|
|
407
|
+
aoi = sb._derive_aoi_polygon(query)
|
|
408
|
+
|
|
409
|
+
passed_results = []
|
|
410
|
+
rejection_reasons: Counter = Counter()
|
|
411
|
+
warning_reasons: Counter = Counter()
|
|
412
|
+
n_warned_but_passed = 0
|
|
413
|
+
|
|
414
|
+
for scene in raw_results:
|
|
415
|
+
report = validator.validate_scene(scene, aoi, query.start_date, query.end_date)
|
|
416
|
+
if report.passed:
|
|
417
|
+
passed_results.append(scene)
|
|
418
|
+
if report.issues:
|
|
419
|
+
n_warned_but_passed += 1
|
|
420
|
+
for issue in report.issues:
|
|
421
|
+
warning_reasons[issue.code] += 1
|
|
422
|
+
else:
|
|
423
|
+
for issue in report.issues:
|
|
424
|
+
if issue.severity == "ERROR":
|
|
425
|
+
rejection_reasons[issue.code] += 1
|
|
426
|
+
|
|
427
|
+
summary = {
|
|
428
|
+
"n_total": len(raw_results),
|
|
429
|
+
"n_passed": len(passed_results),
|
|
430
|
+
"n_rejected": len(raw_results) - len(passed_results),
|
|
431
|
+
"n_warned_but_passed": n_warned_but_passed,
|
|
432
|
+
"rejection_reasons": rejection_reasons,
|
|
433
|
+
"warning_reasons": warning_reasons,
|
|
434
|
+
}
|
|
435
|
+
return passed_results, summary
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _display_validation_summary(summary: dict) -> None:
|
|
439
|
+
"""Clean, real info block summarizing optical validation -- shown
|
|
440
|
+
before the detailed results table, not interleaved with it."""
|
|
441
|
+
n_total = summary["n_total"]
|
|
442
|
+
n_passed = summary["n_passed"]
|
|
443
|
+
n_rejected = summary["n_rejected"]
|
|
444
|
+
pct = (100 * n_passed / n_total) if n_total else 0.0
|
|
445
|
+
|
|
446
|
+
console.print(
|
|
447
|
+
f"\n[bold]Optical validation:[/] {n_passed}/{n_total} scenes passed "
|
|
448
|
+
f"({pct:.0f}%)"
|
|
449
|
+
+ (f", [yellow]{n_rejected} rejected[/]" if n_rejected else "")
|
|
450
|
+
+ (
|
|
451
|
+
f", [cyan]{summary['n_warned_but_passed']} passed with warnings[/]"
|
|
452
|
+
if summary["n_warned_but_passed"]
|
|
453
|
+
else ""
|
|
454
|
+
)
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
if summary["rejection_reasons"]:
|
|
458
|
+
reasons_table = Table(
|
|
459
|
+
title="Rejection reasons", header_style="bold red", show_lines=False
|
|
460
|
+
)
|
|
461
|
+
reasons_table.add_column("Code", style="red")
|
|
462
|
+
reasons_table.add_column("Count", justify="right")
|
|
463
|
+
for code, count in summary["rejection_reasons"].most_common():
|
|
464
|
+
reasons_table.add_row(code, str(count))
|
|
465
|
+
console.print(reasons_table)
|
|
466
|
+
|
|
467
|
+
if summary["warning_reasons"]:
|
|
468
|
+
warn_table = Table(
|
|
469
|
+
title="Warnings on passed scenes",
|
|
470
|
+
header_style="bold yellow",
|
|
471
|
+
show_lines=False,
|
|
472
|
+
)
|
|
473
|
+
warn_table.add_column("Code", style="yellow")
|
|
474
|
+
warn_table.add_column("Count", justify="right")
|
|
475
|
+
for code, count in summary["warning_reasons"].most_common():
|
|
476
|
+
warn_table.add_row(code, str(count))
|
|
477
|
+
console.print(warn_table)
|
|
478
|
+
console.print() # real spacing before the full results table below
|
|
479
|
+
|
|
480
|
+
|
|
368
481
|
def _display_table(results) -> None:
|
|
369
482
|
table = Table(title=f"{len(results)} Results", header_style="bold blue")
|
|
370
483
|
table.add_column("ID", style="cyan", max_width=30)
|
|
@@ -441,4 +554,4 @@ def _save_csv(results, path: Path) -> None:
|
|
|
441
554
|
)
|
|
442
555
|
writer.writeheader()
|
|
443
556
|
for r in results:
|
|
444
|
-
writer.writerow(_item_to_dict(r))
|
|
557
|
+
writer.writerow(_item_to_dict(r))
|
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pygeofetch.core.data_organizer — post-download organization.
|
|
3
|
+
|
|
4
|
+
A flat download directory of dozens of scenes across dates, orbits, and
|
|
5
|
+
sensors is genuinely unusable for downstream InSAR or multi-date optical
|
|
6
|
+
work — this module turns that into a real, structured hierarchy plus a
|
|
7
|
+
real manifest mapping the structure back to the underlying scene metadata.
|
|
8
|
+
|
|
9
|
+
Real design choices, stated rather than left implicit:
|
|
10
|
+
|
|
11
|
+
- Grouping never *loses* the original files by default: `file_operation`
|
|
12
|
+
defaults to "copy", not "move". A user who wants to reclaim disk space
|
|
13
|
+
can opt into "move" or "symlink" explicitly, but the safe default
|
|
14
|
+
doesn't require trusting the grouping logic before deleting the
|
|
15
|
+
original flat download.
|
|
16
|
+
- `prepare_insar_stack()` reuses the existing, already-verified
|
|
17
|
+
`pygeofetch.insar.stack_selection.select_burst_synchronized_dates`
|
|
18
|
+
engine for burst-timing-family classification rather than
|
|
19
|
+
reimplementing burst-sync checking a second time — the same real
|
|
20
|
+
engine the InSAR tutorials use directly. It only runs when real orbit
|
|
21
|
+
files are supplied; without them, this method still does real
|
|
22
|
+
track/date grouping, but honestly reports that burst-sync compatibility
|
|
23
|
+
could not be checked, rather than silently skipping the check.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import logging
|
|
30
|
+
import shutil
|
|
31
|
+
from collections import defaultdict
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
from datetime import date, datetime, timezone
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
from typing import Any, Literal
|
|
36
|
+
|
|
37
|
+
from pydantic import BaseModel, Field
|
|
38
|
+
|
|
39
|
+
from pygeofetch.models.download_task import DownloadResult
|
|
40
|
+
from pygeofetch.models.satellite_data import SatelliteData
|
|
41
|
+
|
|
42
|
+
logger = logging.getLogger("pygeofetch.data_organizer")
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"OrganizerConfig",
|
|
46
|
+
"OrganizeReport",
|
|
47
|
+
"InSARStackReport",
|
|
48
|
+
"DataOrganizer",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
GroupCriterion = Literal["date", "orbit", "satellite", "processing_level"]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class OrganizerConfig(BaseModel):
|
|
55
|
+
"""
|
|
56
|
+
Configuration for `DataOrganizer`.
|
|
57
|
+
|
|
58
|
+
Attributes
|
|
59
|
+
----------
|
|
60
|
+
group_by : list[GroupCriterion]
|
|
61
|
+
Real grouping hierarchy, applied in order — e.g.
|
|
62
|
+
``["satellite", "date"]`` produces ``S2A/2024-06-15/scene.tif``,
|
|
63
|
+
while ``["date", "orbit"]`` produces ``2024-06-15/R054/scene.tif``.
|
|
64
|
+
Defaults to ``["date"]`` — the one grouping every real workflow
|
|
65
|
+
in this project (InSAR, optical time series) needs regardless of
|
|
66
|
+
anything else.
|
|
67
|
+
file_operation : "copy" | "symlink" | "move"
|
|
68
|
+
How files are placed into the grouped hierarchy. "copy" (the
|
|
69
|
+
default) never touches the original flat download; "symlink"
|
|
70
|
+
avoids duplicating large scenes on disk while keeping the
|
|
71
|
+
original layout intact too; "move" reclaims disk space but is
|
|
72
|
+
destructive if the grouping logic ever gets something wrong —
|
|
73
|
+
opt in deliberately, it is never the default.
|
|
74
|
+
manifest_filename : str
|
|
75
|
+
Real manifest file written to `output_dir` after organizing.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
group_by: list[GroupCriterion] = Field(default_factory=lambda: ["date"])
|
|
79
|
+
file_operation: Literal["copy", "symlink", "move"] = "copy"
|
|
80
|
+
manifest_filename: str = "manifest.json"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class OrganizeReport:
|
|
85
|
+
"""Real result of an `organize()` call."""
|
|
86
|
+
|
|
87
|
+
grouped: int = 0
|
|
88
|
+
skipped_no_output: int = 0
|
|
89
|
+
skipped_failed_download: int = 0
|
|
90
|
+
manifest_path: Path | None = None
|
|
91
|
+
groups: dict[str, list[str]] = field(
|
|
92
|
+
default_factory=dict
|
|
93
|
+
) # group path -> scene ids
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class InSARStackReport:
|
|
98
|
+
"""Real result of a `prepare_insar_stack()` call."""
|
|
99
|
+
|
|
100
|
+
tracks: dict[int, list[str]] = field(
|
|
101
|
+
default_factory=dict
|
|
102
|
+
) # relative_orbit -> dates
|
|
103
|
+
orphaned: list[str] = field(default_factory=list) # scenes with no relative_orbit
|
|
104
|
+
burst_sync_checked: bool = False
|
|
105
|
+
burst_family_report: dict[str, Any] | None = (
|
|
106
|
+
None # from select_burst_synchronized_dates
|
|
107
|
+
)
|
|
108
|
+
ready_paths: dict[str, Path] = field(default_factory=dict) # date -> real SLC path
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _group_key_for(scene: SatelliteData, criterion: GroupCriterion) -> str:
|
|
112
|
+
"""Real, single-criterion grouping key for one scene. Every branch
|
|
113
|
+
has an explicit, honest fallback for missing metadata rather than
|
|
114
|
+
raising or silently dropping the scene from grouping."""
|
|
115
|
+
if criterion == "date":
|
|
116
|
+
if scene.datetime is None:
|
|
117
|
+
return "unknown-date"
|
|
118
|
+
dt = scene.datetime
|
|
119
|
+
d = dt.date() if isinstance(dt, datetime) else dt
|
|
120
|
+
return d.isoformat() if isinstance(d, date) else str(dt)[:10]
|
|
121
|
+
if criterion == "orbit":
|
|
122
|
+
return (
|
|
123
|
+
f"R{scene.relative_orbit:03d}"
|
|
124
|
+
if scene.relative_orbit is not None
|
|
125
|
+
else "unknown-orbit"
|
|
126
|
+
)
|
|
127
|
+
if criterion == "satellite":
|
|
128
|
+
return (scene.satellite or "unknown-satellite").replace(" ", "_")
|
|
129
|
+
if criterion == "processing_level":
|
|
130
|
+
return str(
|
|
131
|
+
scene.processing_level.value
|
|
132
|
+
if hasattr(scene.processing_level, "value")
|
|
133
|
+
else scene.processing_level
|
|
134
|
+
)
|
|
135
|
+
raise ValueError(f"Unknown group_by criterion: {criterion!r}") # pragma: no cover
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class DataOrganizer:
|
|
139
|
+
"""
|
|
140
|
+
Groups downloaded scenes into a real, structured hierarchy and
|
|
141
|
+
generates a manifest mapping that structure back to scene metadata.
|
|
142
|
+
|
|
143
|
+
Examples
|
|
144
|
+
--------
|
|
145
|
+
>>> organizer = DataOrganizer(OrganizerConfig(group_by=["satellite", "date"]))
|
|
146
|
+
>>> report = organizer.organize(list(zip(scenes, download_results)), output_dir="./organized")
|
|
147
|
+
>>> print(f"{report.grouped} scenes organized -> {report.manifest_path}")
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
def __init__(self, config: OrganizerConfig | None = None) -> None:
|
|
151
|
+
self.config = config or OrganizerConfig()
|
|
152
|
+
|
|
153
|
+
def _group_dir_for(self, scene: SatelliteData) -> Path:
|
|
154
|
+
parts = [_group_key_for(scene, c) for c in self.config.group_by]
|
|
155
|
+
return Path(*parts) if parts else Path(".")
|
|
156
|
+
|
|
157
|
+
def _place_file(self, src: Path, dst: Path) -> None:
|
|
158
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
159
|
+
if dst.exists():
|
|
160
|
+
return # real idempotency: re-running organize() doesn't re-copy/error
|
|
161
|
+
op = self.config.file_operation
|
|
162
|
+
if op == "copy":
|
|
163
|
+
shutil.copy2(src, dst)
|
|
164
|
+
elif op == "symlink":
|
|
165
|
+
dst.symlink_to(src.resolve())
|
|
166
|
+
elif op == "move":
|
|
167
|
+
shutil.move(str(src), str(dst))
|
|
168
|
+
|
|
169
|
+
def organize(
|
|
170
|
+
self,
|
|
171
|
+
results: list[tuple[SatelliteData, DownloadResult]],
|
|
172
|
+
output_dir: str | Path,
|
|
173
|
+
) -> OrganizeReport:
|
|
174
|
+
"""
|
|
175
|
+
Group real downloaded files into `output_dir` per `config.group_by`,
|
|
176
|
+
and write a real manifest.
|
|
177
|
+
|
|
178
|
+
Parameters
|
|
179
|
+
----------
|
|
180
|
+
results : list of (SatelliteData, DownloadResult)
|
|
181
|
+
Real, matched pairs — typically built by zipping the list
|
|
182
|
+
returned from `client.search()` with the list returned from
|
|
183
|
+
`client.download()`, in the same order both calls preserve.
|
|
184
|
+
output_dir : str or Path
|
|
185
|
+
Root of the real, organized hierarchy.
|
|
186
|
+
|
|
187
|
+
Returns
|
|
188
|
+
-------
|
|
189
|
+
OrganizeReport
|
|
190
|
+
Real counts and the manifest path — inspect `.skipped_*` to
|
|
191
|
+
see how many inputs were genuinely un-organizable (failed
|
|
192
|
+
downloads, or successful downloads with no real output path)
|
|
193
|
+
rather than silently losing that information.
|
|
194
|
+
"""
|
|
195
|
+
output_dir = Path(output_dir)
|
|
196
|
+
manifest_entries: list[dict[str, Any]] = []
|
|
197
|
+
groups: dict[str, list[str]] = defaultdict(list)
|
|
198
|
+
report = OrganizeReport()
|
|
199
|
+
|
|
200
|
+
for scene, dl in results:
|
|
201
|
+
if not dl.success:
|
|
202
|
+
report.skipped_failed_download += 1
|
|
203
|
+
continue
|
|
204
|
+
real_paths = (
|
|
205
|
+
list(dl.output_paths)
|
|
206
|
+
if dl.output_paths
|
|
207
|
+
else ([dl.output_path] if dl.output_path else [])
|
|
208
|
+
)
|
|
209
|
+
if not real_paths:
|
|
210
|
+
report.skipped_no_output += 1
|
|
211
|
+
continue
|
|
212
|
+
|
|
213
|
+
group_dir = self._group_dir_for(scene)
|
|
214
|
+
dest_dir = output_dir / group_dir
|
|
215
|
+
new_paths: list[str] = []
|
|
216
|
+
for src in real_paths:
|
|
217
|
+
src = Path(src)
|
|
218
|
+
if not src.exists():
|
|
219
|
+
logger.warning(
|
|
220
|
+
"organize(): %s -- recorded output path does not exist on "
|
|
221
|
+
"disk (%s), skipping this file but still manifesting the "
|
|
222
|
+
"scene's other real files",
|
|
223
|
+
scene.id,
|
|
224
|
+
src,
|
|
225
|
+
)
|
|
226
|
+
continue
|
|
227
|
+
dst = dest_dir / src.name
|
|
228
|
+
self._place_file(src, dst)
|
|
229
|
+
new_paths.append(str(dst))
|
|
230
|
+
|
|
231
|
+
report.grouped += 1
|
|
232
|
+
groups[str(group_dir)].append(scene.id)
|
|
233
|
+
manifest_entries.append(
|
|
234
|
+
{
|
|
235
|
+
"id": scene.id,
|
|
236
|
+
"provider": scene.provider,
|
|
237
|
+
"satellite": scene.satellite,
|
|
238
|
+
"sensor": scene.sensor,
|
|
239
|
+
"datetime": str(scene.datetime) if scene.datetime else None,
|
|
240
|
+
"bbox": list(scene.bbox) if scene.bbox else None,
|
|
241
|
+
"cloud_cover": scene.cloud_cover,
|
|
242
|
+
"processing_level": str(
|
|
243
|
+
scene.processing_level.value
|
|
244
|
+
if hasattr(scene.processing_level, "value")
|
|
245
|
+
else scene.processing_level
|
|
246
|
+
),
|
|
247
|
+
"relative_orbit": scene.relative_orbit,
|
|
248
|
+
"orbit_number": scene.orbit_number,
|
|
249
|
+
"pass_direction": scene.pass_direction,
|
|
250
|
+
"product_type": scene.product_type,
|
|
251
|
+
"polarisation": scene.polarisation,
|
|
252
|
+
"group": str(group_dir),
|
|
253
|
+
"paths": new_paths,
|
|
254
|
+
}
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
report.groups = dict(groups)
|
|
258
|
+
report.manifest_path = self._write_manifest(manifest_entries, output_dir)
|
|
259
|
+
return report
|
|
260
|
+
|
|
261
|
+
def _write_manifest(self, entries: list[dict[str, Any]], output_dir: Path) -> Path:
|
|
262
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
263
|
+
manifest = {
|
|
264
|
+
"pygeofetch_manifest_version": 1,
|
|
265
|
+
"generated": datetime.now(timezone.utc).isoformat(),
|
|
266
|
+
"group_by": self.config.group_by,
|
|
267
|
+
"scene_count": len(entries),
|
|
268
|
+
"scenes": entries,
|
|
269
|
+
}
|
|
270
|
+
path = output_dir / self.config.manifest_filename
|
|
271
|
+
path.write_text(json.dumps(manifest, indent=2, default=str))
|
|
272
|
+
return path
|
|
273
|
+
|
|
274
|
+
def prepare_insar_stack(
|
|
275
|
+
self,
|
|
276
|
+
results: list[tuple[SatelliteData, DownloadResult]],
|
|
277
|
+
orbit_files: dict[str, Path] | None = None,
|
|
278
|
+
ground_point: tuple[float, float, float] | None = None,
|
|
279
|
+
**burst_sync_kwargs: Any,
|
|
280
|
+
) -> InSARStackReport:
|
|
281
|
+
"""
|
|
282
|
+
Group downloaded SLCs by real track/relative-orbit and, when
|
|
283
|
+
real orbit files are supplied, check burst-timing-family
|
|
284
|
+
compatibility using the already-verified
|
|
285
|
+
`select_burst_synchronized_dates` engine — not a second,
|
|
286
|
+
independent reimplementation of that check.
|
|
287
|
+
|
|
288
|
+
Parameters
|
|
289
|
+
----------
|
|
290
|
+
results : list of (SatelliteData, DownloadResult)
|
|
291
|
+
Real, matched SLC scenes and their download results.
|
|
292
|
+
orbit_files : dict[str, Path], optional
|
|
293
|
+
Real `{date: orbit_file_path}` mapping, e.g. from
|
|
294
|
+
`fetch_orbit_file()` for each date. Burst-sync checking is
|
|
295
|
+
skipped (honestly, not silently) when this is omitted --
|
|
296
|
+
it's a real, separate download step, not something this
|
|
297
|
+
method can fabricate.
|
|
298
|
+
ground_point : tuple[float, float, float], optional
|
|
299
|
+
Real ECEF ground point for the AOI center, required only
|
|
300
|
+
when `orbit_files` is supplied (needed by the underlying
|
|
301
|
+
burst-sync engine's zero-Doppler geometry).
|
|
302
|
+
**burst_sync_kwargs
|
|
303
|
+
Forwarded to `select_burst_synchronized_dates()` --
|
|
304
|
+
`swath_hints`, `min_majority_dates`, `redundancy`, etc.
|
|
305
|
+
|
|
306
|
+
Returns
|
|
307
|
+
-------
|
|
308
|
+
InSARStackReport
|
|
309
|
+
Real per-track grouping, orphaned scenes (no relative_orbit
|
|
310
|
+
metadata -- can't be placed in any track), and, when
|
|
311
|
+
burst-sync was checked, the real classification report from
|
|
312
|
+
`select_burst_synchronized_dates`.
|
|
313
|
+
"""
|
|
314
|
+
report = InSARStackReport()
|
|
315
|
+
by_date: dict[str, Path] = {}
|
|
316
|
+
by_track: dict[int, list[str]] = defaultdict(list)
|
|
317
|
+
|
|
318
|
+
for scene, dl in results:
|
|
319
|
+
if not dl.success or scene.datetime is None:
|
|
320
|
+
continue
|
|
321
|
+
date_str = _group_key_for(scene, "date")
|
|
322
|
+
real_path = dl.output_path or (
|
|
323
|
+
dl.output_paths[0] if dl.output_paths else None
|
|
324
|
+
)
|
|
325
|
+
if real_path is None:
|
|
326
|
+
continue
|
|
327
|
+
by_date[date_str] = Path(real_path)
|
|
328
|
+
if scene.relative_orbit is not None:
|
|
329
|
+
by_track[scene.relative_orbit].append(date_str)
|
|
330
|
+
else:
|
|
331
|
+
report.orphaned.append(scene.id)
|
|
332
|
+
|
|
333
|
+
report.tracks = dict(by_track)
|
|
334
|
+
report.ready_paths = by_date
|
|
335
|
+
|
|
336
|
+
if orbit_files and ground_point is not None:
|
|
337
|
+
from pygeofetch.insar.stack_selection import select_burst_synchronized_dates
|
|
338
|
+
|
|
339
|
+
dates = sorted(by_date.keys())
|
|
340
|
+
safe_zips = {d: by_date[d] for d in dates}
|
|
341
|
+
usable_orbits = {d: orbit_files[d] for d in dates if d in orbit_files}
|
|
342
|
+
if len(usable_orbits) < len(dates):
|
|
343
|
+
logger.warning(
|
|
344
|
+
"prepare_insar_stack(): %d/%d dates have a real orbit file -- "
|
|
345
|
+
"burst-sync classification will only consider those; widen "
|
|
346
|
+
"orbit_files to cover the rest for a complete check.",
|
|
347
|
+
len(usable_orbits),
|
|
348
|
+
len(dates),
|
|
349
|
+
)
|
|
350
|
+
good_dates, family_report = select_burst_synchronized_dates(
|
|
351
|
+
dates, safe_zips, usable_orbits, ground_point, **burst_sync_kwargs
|
|
352
|
+
)
|
|
353
|
+
report.burst_sync_checked = True
|
|
354
|
+
report.burst_family_report = family_report
|
|
355
|
+
report.ready_paths = {d: by_date[d] for d in good_dates}
|
|
356
|
+
else:
|
|
357
|
+
logger.info(
|
|
358
|
+
"prepare_insar_stack(): no orbit_files/ground_point supplied -- "
|
|
359
|
+
"returning real track/date grouping only, burst-sync "
|
|
360
|
+
"compatibility was not checked. Pass real orbit files "
|
|
361
|
+
"(e.g. from fetch_orbit_file()) to enable that check."
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
return report
|
pygeofetch/core/engine.py
CHANGED
|
@@ -58,7 +58,10 @@ if TYPE_CHECKING:
|
|
|
58
58
|
|
|
59
59
|
from pygeofetch.models.download_task import DownloadOptions, DownloadResult
|
|
60
60
|
from pygeofetch.models.search_query import SearchQuery
|
|
61
|
-
from pygeofetch.validation.optical_validator import
|
|
61
|
+
from pygeofetch.validation.optical_validator import (
|
|
62
|
+
OpticalPreflightValidator,
|
|
63
|
+
OpticalValidationConfig,
|
|
64
|
+
)
|
|
62
65
|
|
|
63
66
|
logger = get_logger(__name__)
|
|
64
67
|
|
|
@@ -161,7 +164,9 @@ class PyGeoFetch:
|
|
|
161
164
|
# used — see _get_optical_validator().
|
|
162
165
|
self.validate_optical = validate_optical
|
|
163
166
|
self.optical_validation_config = optical_validation_config
|
|
164
|
-
self._optical_validator
|
|
167
|
+
self._optical_validator: "OpticalPreflightValidator | None" = (
|
|
168
|
+
None # lazily built by _get_optical_validator()
|
|
169
|
+
)
|
|
165
170
|
|
|
166
171
|
logger.info("PyGeoFetch ready")
|
|
167
172
|
|