PyGeoFetch 2.6.2__py3-none-any.whl → 2.6.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pygeofetch/__init__.py +1 -1
- pygeofetch/cli/auth_commands.py +6 -3
- pygeofetch/cli/download_commands.py +62 -1
- pygeofetch/cli/main.py +10 -6
- pygeofetch/cli/monitor_commands.py +49 -14
- pygeofetch/cli/search_commands.py +62 -2
- pygeofetch/core/authenticator.py +122 -19
- pygeofetch/core/downloader.py +31 -13
- pygeofetch/core/engine.py +236 -1
- pygeofetch/core/monitor.py +16 -5
- pygeofetch/core/scheduler.py +239 -8
- pygeofetch/core/searcher.py +26 -2
- pygeofetch/core/state.py +20 -6
- pygeofetch/insar/__init__.py +96 -71
- pygeofetch/insar/analysis.py +62 -40
- pygeofetch/insar/annotation.py +116 -70
- pygeofetch/insar/atmosphere.py +168 -70
- pygeofetch/insar/coregister.py +126 -58
- pygeofetch/insar/deburst.py +39 -12
- pygeofetch/insar/esd.py +65 -28
- pygeofetch/insar/extraction.py +126 -67
- pygeofetch/insar/flatearth.py +45 -18
- pygeofetch/insar/geolocation.py +79 -29
- pygeofetch/insar/gpu.py +3 -1
- pygeofetch/insar/interferogram.py +502 -193
- pygeofetch/insar/ionosphere.py +118 -45
- pygeofetch/insar/offset_geometry.py +47 -43
- pygeofetch/insar/offset_tracking.py +45 -20
- pygeofetch/insar/preflight.py +295 -170
- pygeofetch/insar/provenance.py +2 -1
- pygeofetch/insar/ps_selection.py +21 -9
- pygeofetch/insar/splitspectrum_diagnostic.py +12 -4
- pygeofetch/insar/stack_selection.py +179 -77
- pygeofetch/insar/synthetic.py +53 -17
- pygeofetch/insar/timeseries.py +268 -152
- pygeofetch/insar/unwrap.py +408 -156
- pygeofetch/insar/validate.py +36 -13
- pygeofetch/insar/visualize.py +15 -5
- pygeofetch/insar/workflow.py +102 -29
- pygeofetch/meta.yaml +1 -1
- pygeofetch/models/satellite_data.py +5 -3
- pygeofetch/processing/indices.py +5 -0
- pygeofetch/processing/preprocessor.py +55 -25
- pygeofetch/processing/sar.py +13 -4
- pygeofetch/processor/indices.py +4 -0
- pygeofetch/processor/stacker.py +4 -0
- pygeofetch/processor/timeseries.py +41 -13
- pygeofetch/providers/airbus_oneatlas.py +297 -96
- pygeofetch/providers/alaska_satellite_facility.py +5 -2
- pygeofetch/providers/aws_earth.py +1 -1
- pygeofetch/providers/base.py +2 -0
- pygeofetch/providers/copernicus.py +19 -8
- pygeofetch/providers/copernicus_nodes.py +73 -23
- pygeofetch/providers/digitalglobe.py +5 -2
- pygeofetch/providers/earth_explorer_additional.py +5 -2
- pygeofetch/providers/element84.py +1 -1
- pygeofetch/providers/eodag_provider.py +9 -5
- pygeofetch/providers/esa_scihub.py +32 -133
- pygeofetch/providers/geoserver_generic.py +5 -2
- pygeofetch/providers/google_earth_engine.py +37 -148
- pygeofetch/providers/inpe_cbers.py +5 -2
- pygeofetch/providers/isro_bhuvan.py +5 -2
- pygeofetch/providers/jaxa_earth.py +5 -2
- pygeofetch/providers/maxar_gbdx.py +5 -2
- pygeofetch/providers/nasa_earthdata.py +6 -4
- pygeofetch/providers/nasa_earthdata_cloud.py +2 -2
- pygeofetch/providers/noaa_big_data.py +206 -124
- pygeofetch/providers/opentopography.py +6 -2
- pygeofetch/providers/planet.py +4 -3
- pygeofetch/providers/planetary_computer.py +1 -1
- pygeofetch/providers/sentinel_hub.py +11 -7
- pygeofetch/providers/terrabotics.py +4 -3
- pygeofetch/providers/usgs.py +20 -5
- pygeofetch/sar/__init__.py +1 -1
- pygeofetch/sar/_native.py +1 -1
- pygeofetch/sar/extraction.py +8 -4
- pygeofetch/utils/gcp_recovery.py +1 -3
- pygeofetch/validation/__init__.py +24 -0
- pygeofetch/validation/optical_validator.py +918 -0
- pygeofetch/viz/map.py +82 -33
- pygeofetch/viz/plot.py +106 -45
- {pygeofetch-2.6.2.dist-info → pygeofetch-2.6.2.2.dist-info}/METADATA +47 -44
- pygeofetch-2.6.2.2.dist-info/RECORD +124 -0
- pygeofetch-2.6.2.dist-info/RECORD +0 -122
- {pygeofetch-2.6.2.dist-info → pygeofetch-2.6.2.2.dist-info}/WHEEL +0 -0
- {pygeofetch-2.6.2.dist-info → pygeofetch-2.6.2.2.dist-info}/entry_points.txt +0 -0
- {pygeofetch-2.6.2.dist-info → pygeofetch-2.6.2.2.dist-info}/licenses/LICENSE +0 -0
- {pygeofetch-2.6.2.dist-info → pygeofetch-2.6.2.2.dist-info}/top_level.txt +0 -0
pygeofetch/__init__.py
CHANGED
pygeofetch/cli/auth_commands.py
CHANGED
|
@@ -221,8 +221,11 @@ def auth_remove(provider: str, yes: bool) -> None:
|
|
|
221
221
|
click.confirm(f"Remove credentials for {provider!r}?", abort=True)
|
|
222
222
|
|
|
223
223
|
mgr = AuthManager()
|
|
224
|
-
mgr.
|
|
225
|
-
|
|
224
|
+
removed = mgr.remove(provider)
|
|
225
|
+
if removed:
|
|
226
|
+
console.print(f"[green]Credentials for [bold]{provider}[/] removed.[/]")
|
|
227
|
+
else:
|
|
228
|
+
console.print(f"[yellow]No stored credentials found for {provider!r}.[/]")
|
|
226
229
|
|
|
227
230
|
|
|
228
231
|
@auth.command(name="export")
|
|
@@ -250,7 +253,7 @@ def auth_export(provider: str | None, output: str) -> None:
|
|
|
250
253
|
from pygeofetch.core.authenticator import AuthManager
|
|
251
254
|
|
|
252
255
|
mgr = AuthManager()
|
|
253
|
-
data = mgr.export_credentials(
|
|
256
|
+
data = mgr.export_credentials(providers=[provider] if provider else None)
|
|
254
257
|
|
|
255
258
|
with open(output, "w") as f:
|
|
256
259
|
json.dump(data, f, indent=2, default=str)
|
|
@@ -114,6 +114,36 @@ def download() -> None:
|
|
|
114
114
|
help='Comma-separated band names to download e.g. "B02,B03,B04" (default: all data assets).',
|
|
115
115
|
)
|
|
116
116
|
@click.option("--json", "as_json", is_flag=True, help="Output results summary as JSON.")
|
|
117
|
+
@click.option(
|
|
118
|
+
"--validate-optical",
|
|
119
|
+
is_flag=True,
|
|
120
|
+
default=False,
|
|
121
|
+
help=(
|
|
122
|
+
"Run optical preflight validation as a final gate before "
|
|
123
|
+
"downloading (AOI coverage, cloud cover, required bands, "
|
|
124
|
+
"processing level, temporal bounds). Items that fail are "
|
|
125
|
+
"never attempted -- a failed result explaining why is "
|
|
126
|
+
"returned in their place, same position, so output length "
|
|
127
|
+
"always matches input."
|
|
128
|
+
),
|
|
129
|
+
)
|
|
130
|
+
@click.option(
|
|
131
|
+
"--optical-max-cloud-cover",
|
|
132
|
+
default=None,
|
|
133
|
+
type=float,
|
|
134
|
+
help="Override the max cloud cover threshold used by --validate-optical (default 20.0).",
|
|
135
|
+
)
|
|
136
|
+
@click.option(
|
|
137
|
+
"--optical-min-coverage",
|
|
138
|
+
default=None,
|
|
139
|
+
type=float,
|
|
140
|
+
help="Override the min AOI coverage ratio used by --validate-optical, 0-1 (default 0.8).",
|
|
141
|
+
)
|
|
142
|
+
@click.option(
|
|
143
|
+
"--optical-required-bands",
|
|
144
|
+
default=None,
|
|
145
|
+
help="Comma-separated required bands for --validate-optical (default B02,B03,B04,B08,SCL).",
|
|
146
|
+
)
|
|
117
147
|
def download_run(
|
|
118
148
|
from_search,
|
|
119
149
|
scene_ids,
|
|
@@ -132,6 +162,10 @@ def download_run(
|
|
|
132
162
|
overwrite,
|
|
133
163
|
bands,
|
|
134
164
|
as_json,
|
|
165
|
+
validate_optical,
|
|
166
|
+
optical_max_cloud_cover,
|
|
167
|
+
optical_min_coverage,
|
|
168
|
+
optical_required_bands,
|
|
135
169
|
) -> None:
|
|
136
170
|
"""
|
|
137
171
|
Download satellite data products from a search results file or scene IDs.
|
|
@@ -151,6 +185,11 @@ def download_run(
|
|
|
151
185
|
--notify webhook:https://hooks.slack.com/T01/B01/XYZ \\
|
|
152
186
|
--notify email:user@example.com \\
|
|
153
187
|
--on-failure skip
|
|
188
|
+
|
|
189
|
+
# With optical preflight validation as a final gate before downloading
|
|
190
|
+
pygeofetch download run \\
|
|
191
|
+
--from-search results.geojson --output ./data/ \\
|
|
192
|
+
--validate-optical --optical-max-cloud-cover 10
|
|
154
193
|
"""
|
|
155
194
|
from pygeofetch.core.engine import PyGeoFetch
|
|
156
195
|
from pygeofetch.core.searcher import FederatedSearcher
|
|
@@ -218,6 +257,24 @@ def download_run(
|
|
|
218
257
|
|
|
219
258
|
sb = PyGeoFetch()
|
|
220
259
|
|
|
260
|
+
if validate_optical and (
|
|
261
|
+
optical_max_cloud_cover,
|
|
262
|
+
optical_min_coverage,
|
|
263
|
+
optical_required_bands,
|
|
264
|
+
) != (None, None, None):
|
|
265
|
+
from pygeofetch.validation import OpticalValidationConfig
|
|
266
|
+
|
|
267
|
+
overrides = {}
|
|
268
|
+
if optical_max_cloud_cover is not None:
|
|
269
|
+
overrides["max_cloud_cover_pct"] = optical_max_cloud_cover
|
|
270
|
+
if optical_min_coverage is not None:
|
|
271
|
+
overrides["min_coverage_ratio"] = optical_min_coverage
|
|
272
|
+
if optical_required_bands is not None:
|
|
273
|
+
overrides["required_bands"] = [
|
|
274
|
+
b.strip() for b in optical_required_bands.split(",")
|
|
275
|
+
]
|
|
276
|
+
sb.optical_validation_config = OpticalValidationConfig(**overrides)
|
|
277
|
+
|
|
221
278
|
# Load data
|
|
222
279
|
if from_search:
|
|
223
280
|
data_list = FederatedSearcher.load_results(Path(from_search))
|
|
@@ -273,7 +330,11 @@ def download_run(
|
|
|
273
330
|
|
|
274
331
|
# Use sb.download() with the already-sliced data_list (respects --max-items)
|
|
275
332
|
results = sb.download(
|
|
276
|
-
data_list,
|
|
333
|
+
data_list,
|
|
334
|
+
Path(output),
|
|
335
|
+
options,
|
|
336
|
+
item_done_callback=on_item_done,
|
|
337
|
+
validate_optical=validate_optical,
|
|
277
338
|
)
|
|
278
339
|
|
|
279
340
|
succeeded = [r for r in results if r.success]
|
pygeofetch/cli/main.py
CHANGED
|
@@ -484,9 +484,11 @@ def providers_info(provider_id: str, as_json: bool) -> None:
|
|
|
484
484
|
table.add_row("Auth type", caps.auth_type)
|
|
485
485
|
table.add_row(
|
|
486
486
|
"Auth status",
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
487
|
+
(
|
|
488
|
+
f"[green]{auth_status}[/]"
|
|
489
|
+
if "✓" in auth_status or "🌐" in auth_status
|
|
490
|
+
else f"[red]{auth_status}[/]"
|
|
491
|
+
),
|
|
490
492
|
)
|
|
491
493
|
table.add_row("Satellites", ", ".join(caps.satellites) if caps.satellites else "—")
|
|
492
494
|
table.add_row("SAR support", "[green]Yes[/]" if caps.supports_sar else "No")
|
|
@@ -495,9 +497,11 @@ def providers_info(provider_id: str, as_json: bool) -> None:
|
|
|
495
497
|
table.add_row("CQL2 filter", "[green]Yes[/]" if caps.supports_cql2 else "No")
|
|
496
498
|
table.add_row(
|
|
497
499
|
"Resolution",
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
500
|
+
(
|
|
501
|
+
f"{caps.resolution_min_m}m – {caps.resolution_max_m}m"
|
|
502
|
+
if caps.resolution_min_m
|
|
503
|
+
else "Varies"
|
|
504
|
+
),
|
|
501
505
|
)
|
|
502
506
|
table.add_row("Regions", ", ".join(caps.regions) if caps.regions else "global")
|
|
503
507
|
table.add_row("Endpoint", caps.endpoint_url or "—")
|
|
@@ -33,22 +33,51 @@ console = Console()
|
|
|
33
33
|
logger = logging.getLogger("pygeofetch.cli.monitor")
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
@click.group(
|
|
36
|
+
@click.group(
|
|
37
|
+
name="monitor", help="Automated, incremental InSAR monitoring (Feature 4)."
|
|
38
|
+
)
|
|
37
39
|
def monitor():
|
|
38
40
|
pass
|
|
39
41
|
|
|
40
42
|
|
|
41
|
-
@monitor.command(
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
@click.option(
|
|
46
|
-
|
|
43
|
+
@monitor.command(
|
|
44
|
+
name="run",
|
|
45
|
+
help="Run one real monitoring cycle: search for new scenes, plan incremental processing.",
|
|
46
|
+
)
|
|
47
|
+
@click.option(
|
|
48
|
+
"--state-db",
|
|
49
|
+
required=True,
|
|
50
|
+
type=click.Path(),
|
|
51
|
+
help="Path to this project's real state database.",
|
|
52
|
+
)
|
|
53
|
+
@click.option(
|
|
54
|
+
"--bbox",
|
|
55
|
+
required=True,
|
|
56
|
+
type=str,
|
|
57
|
+
help="Real AOI bounding box, 'minlon,minlat,maxlon,maxlat'.",
|
|
58
|
+
)
|
|
59
|
+
@click.option(
|
|
60
|
+
"--satellites",
|
|
61
|
+
default="Sentinel-1A,Sentinel-1B",
|
|
62
|
+
help="Comma-separated real satellite names.",
|
|
63
|
+
)
|
|
64
|
+
@click.option(
|
|
65
|
+
"--n-neighbors",
|
|
66
|
+
default=3,
|
|
67
|
+
type=int,
|
|
68
|
+
help="Real neighbors each new date connects to (spec default: 3).",
|
|
69
|
+
)
|
|
70
|
+
@click.option(
|
|
71
|
+
"--cron-daily",
|
|
72
|
+
is_flag=True,
|
|
73
|
+
help="Marker flag for scheduled invocation (real cron job); does not change behavior, documents intent in logs.",
|
|
74
|
+
)
|
|
47
75
|
@click.pass_context
|
|
48
76
|
def monitor_run(ctx, state_db, bbox, satellites, n_neighbors, cron_daily):
|
|
49
77
|
client = ctx.obj.get("client") if ctx.obj else None
|
|
50
78
|
if client is None:
|
|
51
79
|
from pygeofetch import PyGeoFetch
|
|
80
|
+
|
|
52
81
|
client = PyGeoFetch()
|
|
53
82
|
|
|
54
83
|
state = ProjectState(state_db)
|
|
@@ -66,7 +95,9 @@ def monitor_run(ctx, state_db, bbox, satellites, n_neighbors, cron_daily):
|
|
|
66
95
|
if state.last_download_timestamp:
|
|
67
96
|
query.start_date = state.last_download_timestamp
|
|
68
97
|
|
|
69
|
-
console.print(
|
|
98
|
+
console.print(
|
|
99
|
+
f"[bold]pygeofetch monitor[/bold] — {'scheduled' if cron_daily else 'manual'} run"
|
|
100
|
+
)
|
|
70
101
|
console.print(f"State: {state_db}")
|
|
71
102
|
|
|
72
103
|
try:
|
|
@@ -76,7 +107,9 @@ def monitor_run(ctx, state_db, bbox, satellites, n_neighbors, cron_daily):
|
|
|
76
107
|
console.print(f"[red]Search failed:[/red] {exc}")
|
|
77
108
|
raise
|
|
78
109
|
|
|
79
|
-
available_dates = sorted(
|
|
110
|
+
available_dates = sorted(
|
|
111
|
+
{str(r.datetime)[:10] for r in results if getattr(r, "datetime", None)}
|
|
112
|
+
)
|
|
80
113
|
plan = plan_monitoring_run(state, available_dates, n_neighbors=n_neighbors)
|
|
81
114
|
|
|
82
115
|
console.print(plan.message)
|
|
@@ -84,11 +117,13 @@ def monitor_run(ctx, state_db, bbox, satellites, n_neighbors, cron_daily):
|
|
|
84
117
|
console.print(f" New dates: {plan.new_dates}")
|
|
85
118
|
console.print(f" New pairs needed: {len(plan.new_pairs)}")
|
|
86
119
|
|
|
87
|
-
state.record_run(
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
120
|
+
state.record_run(
|
|
121
|
+
RunSummary(
|
|
122
|
+
status="success" if plan.network_changed else "no_new_scenes",
|
|
123
|
+
n_new_scenes=len(plan.new_dates),
|
|
124
|
+
detail=plan.message,
|
|
125
|
+
)
|
|
126
|
+
)
|
|
92
127
|
|
|
93
128
|
return plan
|
|
94
129
|
|
|
@@ -67,7 +67,7 @@ def search() -> None:
|
|
|
67
67
|
@click.option(
|
|
68
68
|
"--polarisation",
|
|
69
69
|
default=None,
|
|
70
|
-
help=
|
|
70
|
+
help="SAR polarisation e.g. VV, VH, HH, HV. Same Copernicus-only caveat as --product-type.",
|
|
71
71
|
)
|
|
72
72
|
@click.option("--max-results", "-n", default=100, show_default=True)
|
|
73
73
|
@click.option(
|
|
@@ -104,6 +104,34 @@ def search() -> None:
|
|
|
104
104
|
"--timeout", default=60, show_default=True, help="Per-provider timeout in seconds."
|
|
105
105
|
)
|
|
106
106
|
@click.option("--no-cache", is_flag=True, default=False, help="Bypass result cache.")
|
|
107
|
+
@click.option(
|
|
108
|
+
"--validate-optical",
|
|
109
|
+
is_flag=True,
|
|
110
|
+
default=False,
|
|
111
|
+
help=(
|
|
112
|
+
"Run optical preflight validation (AOI coverage, cloud cover, "
|
|
113
|
+
"required bands, processing level, temporal bounds) on results "
|
|
114
|
+
"before returning them. Not appropriate for SAR/InSAR searches "
|
|
115
|
+
"-- the default required bands are optical band names."
|
|
116
|
+
),
|
|
117
|
+
)
|
|
118
|
+
@click.option(
|
|
119
|
+
"--optical-max-cloud-cover",
|
|
120
|
+
default=None,
|
|
121
|
+
type=float,
|
|
122
|
+
help="Override the max cloud cover threshold used by --validate-optical (default 20.0).",
|
|
123
|
+
)
|
|
124
|
+
@click.option(
|
|
125
|
+
"--optical-min-coverage",
|
|
126
|
+
default=None,
|
|
127
|
+
type=float,
|
|
128
|
+
help="Override the min AOI coverage ratio used by --validate-optical, 0-1 (default 0.8).",
|
|
129
|
+
)
|
|
130
|
+
@click.option(
|
|
131
|
+
"--optical-required-bands",
|
|
132
|
+
default=None,
|
|
133
|
+
help="Comma-separated required bands for --validate-optical (default B02,B03,B04,B08,SCL).",
|
|
134
|
+
)
|
|
107
135
|
def search_run(
|
|
108
136
|
bbox,
|
|
109
137
|
geometry_file,
|
|
@@ -125,6 +153,10 @@ def search_run(
|
|
|
125
153
|
on_provider_failure,
|
|
126
154
|
timeout,
|
|
127
155
|
no_cache,
|
|
156
|
+
validate_optical,
|
|
157
|
+
optical_max_cloud_cover,
|
|
158
|
+
optical_min_coverage,
|
|
159
|
+
optical_required_bands,
|
|
128
160
|
) -> None:
|
|
129
161
|
"""
|
|
130
162
|
Search for satellite imagery across one or more providers.
|
|
@@ -146,6 +178,11 @@ def search_run(
|
|
|
146
178
|
--geometry-file area.geojson \\
|
|
147
179
|
--cql2 "eo:cloud_cover < 5 AND platform = 'sentinel-2b'" \\
|
|
148
180
|
--format geoparquet --output results.parquet
|
|
181
|
+
|
|
182
|
+
# With optical preflight validation (skip cloudy/off-target/wrong-level scenes)
|
|
183
|
+
pygeofetch search run \\
|
|
184
|
+
--bbox "-74,40,-73,41" --cloud-cover 0-100 \\
|
|
185
|
+
--validate-optical --optical-max-cloud-cover 10 --optical-min-coverage 0.9
|
|
149
186
|
"""
|
|
150
187
|
from pygeofetch.core.engine import PyGeoFetch
|
|
151
188
|
from pygeofetch.models.search_query import BoundingBox, SearchQuery
|
|
@@ -245,10 +282,33 @@ def search_run(
|
|
|
245
282
|
|
|
246
283
|
sb = PyGeoFetch()
|
|
247
284
|
|
|
285
|
+
if validate_optical and (
|
|
286
|
+
optical_max_cloud_cover,
|
|
287
|
+
optical_min_coverage,
|
|
288
|
+
optical_required_bands,
|
|
289
|
+
) != (None, None, None):
|
|
290
|
+
from pygeofetch.validation import OpticalValidationConfig
|
|
291
|
+
|
|
292
|
+
overrides = {}
|
|
293
|
+
if optical_max_cloud_cover is not None:
|
|
294
|
+
overrides["max_cloud_cover_pct"] = optical_max_cloud_cover
|
|
295
|
+
if optical_min_coverage is not None:
|
|
296
|
+
overrides["min_coverage_ratio"] = optical_min_coverage
|
|
297
|
+
if optical_required_bands is not None:
|
|
298
|
+
overrides["required_bands"] = [
|
|
299
|
+
b.strip() for b in optical_required_bands.split(",")
|
|
300
|
+
]
|
|
301
|
+
sb.optical_validation_config = OpticalValidationConfig(**overrides)
|
|
302
|
+
|
|
248
303
|
with console.status(
|
|
249
304
|
f"[cyan]Searching {len(provider_list or [])} provider(s)...[/]"
|
|
250
305
|
):
|
|
251
|
-
results = sb.search(
|
|
306
|
+
results = sb.search(
|
|
307
|
+
query,
|
|
308
|
+
providers=provider_list,
|
|
309
|
+
use_cache=not no_cache,
|
|
310
|
+
validate_optical=validate_optical,
|
|
311
|
+
)
|
|
252
312
|
|
|
253
313
|
if not results:
|
|
254
314
|
console.print("[yellow]No results found.[/]")
|
pygeofetch/core/authenticator.py
CHANGED
|
@@ -30,6 +30,8 @@ from pygeofetch.models.user_auth import AuthSession, AuthType, Credentials
|
|
|
30
30
|
if TYPE_CHECKING:
|
|
31
31
|
import builtins
|
|
32
32
|
|
|
33
|
+
from cryptography.fernet import Fernet
|
|
34
|
+
|
|
33
35
|
logger = get_logger(__name__)
|
|
34
36
|
|
|
35
37
|
|
|
@@ -45,8 +47,12 @@ class CredentialStore:
|
|
|
45
47
|
def __init__(self, storage_backend: str = "keyring") -> None:
|
|
46
48
|
self.storage_backend = storage_backend
|
|
47
49
|
self._config_dir = Path.home() / ".pygeofetch"
|
|
48
|
-
self._cred_file = self._config_dir / "credentials.
|
|
50
|
+
self._cred_file = self._config_dir / "credentials.enc"
|
|
51
|
+
self._legacy_cred_file = self._config_dir / "credentials.json"
|
|
52
|
+
self._key_file = self._config_dir / "credentials.key"
|
|
49
53
|
self._in_memory: dict[str, dict[str, Any]] = {}
|
|
54
|
+
self._fernet: Fernet | None = None # lazily initialized — see _get_fernet()
|
|
55
|
+
self._migrating = False # re-entrancy guard for _migrate_legacy_file_if_needed
|
|
50
56
|
|
|
51
57
|
def save(self, provider: str, credentials: dict[str, Any]) -> None:
|
|
52
58
|
"""Persist credentials for a provider."""
|
|
@@ -126,7 +132,88 @@ class CredentialStore:
|
|
|
126
132
|
except Exception:
|
|
127
133
|
return self._delete_file(provider)
|
|
128
134
|
|
|
135
|
+
_SENSITIVE_FIELDS = ("password", "api_key", "token", "secret_key", "client_secret")
|
|
136
|
+
|
|
137
|
+
def _get_fernet(self) -> Fernet:
|
|
138
|
+
"""
|
|
139
|
+
Return a cached Fernet instance, generating and persisting a new
|
|
140
|
+
local encryption key on first use.
|
|
141
|
+
|
|
142
|
+
REAL BUG FIXED: this file backend previously used
|
|
143
|
+
``base64.b64encode`` on sensitive fields with a comment stating
|
|
144
|
+
outright "Basic obfuscation (not encryption)" -- base64 is
|
|
145
|
+
trivially reversible by anyone who can read the file, with no
|
|
146
|
+
key required, and the docs previously (and inaccurately) claimed
|
|
147
|
+
this path used Fernet AES-128-CBC. It now genuinely does.
|
|
148
|
+
"""
|
|
149
|
+
if self._fernet is not None:
|
|
150
|
+
return self._fernet
|
|
151
|
+
from cryptography.fernet import Fernet
|
|
152
|
+
|
|
153
|
+
self._config_dir.mkdir(parents=True, exist_ok=True)
|
|
154
|
+
if self._key_file.exists():
|
|
155
|
+
key = self._key_file.read_bytes()
|
|
156
|
+
else:
|
|
157
|
+
key = Fernet.generate_key()
|
|
158
|
+
self._key_file.write_bytes(key)
|
|
159
|
+
self._key_file.chmod(0o600)
|
|
160
|
+
self._fernet = Fernet(key)
|
|
161
|
+
return self._fernet
|
|
162
|
+
|
|
163
|
+
def _migrate_legacy_file_if_needed(self) -> None:
|
|
164
|
+
"""
|
|
165
|
+
One-time transparent migration from the old base64-"obfuscated"
|
|
166
|
+
``credentials.json`` to the new Fernet-encrypted
|
|
167
|
+
``credentials.enc``, so existing users don't lose stored
|
|
168
|
+
credentials when upgrading. Runs at most once: after migration,
|
|
169
|
+
the legacy file is left in place but unused (never read again
|
|
170
|
+
once ``credentials.enc`` exists) rather than deleted, in case a
|
|
171
|
+
user needs to roll back.
|
|
172
|
+
"""
|
|
173
|
+
if (
|
|
174
|
+
self._cred_file.exists()
|
|
175
|
+
or not self._legacy_cred_file.exists()
|
|
176
|
+
or self._migrating
|
|
177
|
+
):
|
|
178
|
+
return
|
|
179
|
+
self._migrating = True
|
|
180
|
+
try:
|
|
181
|
+
try:
|
|
182
|
+
legacy_data = json.loads(self._legacy_cred_file.read_text())
|
|
183
|
+
except Exception as exc:
|
|
184
|
+
logger.warning(
|
|
185
|
+
f"Could not read legacy credentials file for migration: {exc}"
|
|
186
|
+
)
|
|
187
|
+
return
|
|
188
|
+
|
|
189
|
+
migrated = 0
|
|
190
|
+
for provider, creds in legacy_data.items():
|
|
191
|
+
if not isinstance(creds, dict):
|
|
192
|
+
continue
|
|
193
|
+
plain = dict(creds)
|
|
194
|
+
if plain.pop("__obfuscated", False):
|
|
195
|
+
for k in self._SENSITIVE_FIELDS:
|
|
196
|
+
if k in plain:
|
|
197
|
+
try:
|
|
198
|
+
plain[k] = base64.b64decode(
|
|
199
|
+
str(plain[k]).encode()
|
|
200
|
+
).decode()
|
|
201
|
+
except Exception:
|
|
202
|
+
pass
|
|
203
|
+
self._save_file(provider, plain)
|
|
204
|
+
migrated += 1
|
|
205
|
+
if migrated:
|
|
206
|
+
logger.info(
|
|
207
|
+
f"Migrated {migrated} provider credential(s) from the old "
|
|
208
|
+
f"{self._legacy_cred_file.name} (base64) to the new "
|
|
209
|
+
f"{self._cred_file.name} (Fernet-encrypted). The old file "
|
|
210
|
+
"was left in place, unused, and can be deleted manually."
|
|
211
|
+
)
|
|
212
|
+
finally:
|
|
213
|
+
self._migrating = False
|
|
214
|
+
|
|
129
215
|
def _save_file(self, provider: str, credentials: dict[str, Any]) -> None:
|
|
216
|
+
self._migrate_legacy_file_if_needed()
|
|
130
217
|
self._config_dir.mkdir(parents=True, exist_ok=True)
|
|
131
218
|
existing: dict[str, Any] = {}
|
|
132
219
|
if self._cred_file.exists():
|
|
@@ -134,19 +221,23 @@ class CredentialStore:
|
|
|
134
221
|
existing = json.loads(self._cred_file.read_text())
|
|
135
222
|
except Exception:
|
|
136
223
|
pass
|
|
137
|
-
|
|
138
|
-
|
|
224
|
+
|
|
225
|
+
fernet = self._get_fernet()
|
|
226
|
+
encrypted = {}
|
|
139
227
|
for k, v in credentials.items():
|
|
140
|
-
if k in
|
|
141
|
-
|
|
228
|
+
if k in self._SENSITIVE_FIELDS:
|
|
229
|
+
encrypted[k] = fernet.encrypt(str(v).encode()).decode()
|
|
142
230
|
else:
|
|
143
|
-
|
|
144
|
-
existing[provider] = {"
|
|
231
|
+
encrypted[k] = v
|
|
232
|
+
existing[provider] = {"__encrypted": True, **encrypted}
|
|
145
233
|
self._cred_file.write_text(json.dumps(existing, indent=2))
|
|
146
234
|
self._cred_file.chmod(0o600)
|
|
147
|
-
logger.debug(
|
|
235
|
+
logger.debug(
|
|
236
|
+
f"Credentials for {provider!r} saved to {self._cred_file} (encrypted)"
|
|
237
|
+
)
|
|
148
238
|
|
|
149
239
|
def _load_file(self, provider: str) -> dict[str, Any] | None:
|
|
240
|
+
self._migrate_legacy_file_if_needed()
|
|
150
241
|
if not self._cred_file.exists():
|
|
151
242
|
return None
|
|
152
243
|
try:
|
|
@@ -154,19 +245,29 @@ class CredentialStore:
|
|
|
154
245
|
creds = data.get(provider)
|
|
155
246
|
if not creds:
|
|
156
247
|
return None
|
|
248
|
+
if creds.get("__encrypted"):
|
|
249
|
+
creds = {k: v for k, v in creds.items() if k != "__encrypted"}
|
|
250
|
+
fernet = self._get_fernet()
|
|
251
|
+
decoded = {}
|
|
252
|
+
for k, v in creds.items():
|
|
253
|
+
if k in self._SENSITIVE_FIELDS:
|
|
254
|
+
try:
|
|
255
|
+
decoded[k] = fernet.decrypt(str(v).encode()).decode()
|
|
256
|
+
except Exception:
|
|
257
|
+
decoded[k] = v
|
|
258
|
+
else:
|
|
259
|
+
decoded[k] = v
|
|
260
|
+
return decoded
|
|
261
|
+
# Legacy plain/base64 entries encountered directly in
|
|
262
|
+
# credentials.enc (shouldn't normally happen, but handled
|
|
263
|
+
# rather than silently failing).
|
|
157
264
|
if creds.get("__obfuscated"):
|
|
158
265
|
creds = {k: v for k, v in creds.items() if k != "__obfuscated"}
|
|
159
266
|
decoded = {}
|
|
160
267
|
for k, v in creds.items():
|
|
161
|
-
if k in
|
|
162
|
-
"password",
|
|
163
|
-
"api_key",
|
|
164
|
-
"token",
|
|
165
|
-
"secret_key",
|
|
166
|
-
"client_secret",
|
|
167
|
-
):
|
|
268
|
+
if k in self._SENSITIVE_FIELDS:
|
|
168
269
|
try:
|
|
169
|
-
decoded[k] = base64.b64decode(v.encode()).decode()
|
|
270
|
+
decoded[k] = base64.b64decode(str(v).encode()).decode()
|
|
170
271
|
except Exception:
|
|
171
272
|
decoded[k] = v
|
|
172
273
|
else:
|
|
@@ -447,9 +548,11 @@ class AuthManager:
|
|
|
447
548
|
{
|
|
448
549
|
"provider": provider,
|
|
449
550
|
"has_session": session is not None and session.is_valid,
|
|
450
|
-
"session_expires":
|
|
451
|
-
|
|
452
|
-
|
|
551
|
+
"session_expires": (
|
|
552
|
+
session.expires_at.isoformat()
|
|
553
|
+
if session and session.expires_at
|
|
554
|
+
else None
|
|
555
|
+
),
|
|
453
556
|
}
|
|
454
557
|
)
|
|
455
558
|
return result
|
pygeofetch/core/downloader.py
CHANGED
|
@@ -25,7 +25,7 @@ import hashlib
|
|
|
25
25
|
import time
|
|
26
26
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
27
27
|
from pathlib import Path
|
|
28
|
-
from typing import TYPE_CHECKING, Any, Callable, Dict
|
|
28
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, cast
|
|
29
29
|
|
|
30
30
|
from pygeofetch.core.logging import DownloadProgress, get_logger
|
|
31
31
|
from pygeofetch.models.download_task import (
|
|
@@ -365,16 +365,19 @@ class AdaptiveDownloader:
|
|
|
365
365
|
sum(1 for r in results if r and r.success)
|
|
366
366
|
sum(1 for r in results if r and not r.success)
|
|
367
367
|
dp.__exit__(None, None, None)
|
|
368
|
-
assert len(results) == len(
|
|
369
|
-
|
|
370
|
-
)
|
|
368
|
+
assert len(results) == len(
|
|
369
|
+
data_list
|
|
370
|
+
), f"BUG: result count {len(results)} != input count {len(data_list)}"
|
|
371
371
|
sum(
|
|
372
372
|
r.bytes_downloaded / (1024 * 1024)
|
|
373
373
|
for r in results
|
|
374
374
|
if r and r.success and r.bytes_downloaded
|
|
375
375
|
)
|
|
376
376
|
# summary printed by DownloadProgress footer
|
|
377
|
-
|
|
377
|
+
# The loop above has already replaced every None slot with a
|
|
378
|
+
# real DownloadResult (failure placeholder), so this is
|
|
379
|
+
# genuinely list[DownloadResult] at this point.
|
|
380
|
+
return cast(List[DownloadResult], results)
|
|
378
381
|
|
|
379
382
|
def _get_provider(self, provider_id: str) -> Any:
|
|
380
383
|
"""Get a provider instance with a fresh authenticated session."""
|
|
@@ -889,7 +892,7 @@ class AdaptiveDownloader:
|
|
|
889
892
|
from pygeofetch.processor import SpectralIndex
|
|
890
893
|
except ImportError as exc:
|
|
891
894
|
raise ImportError(
|
|
892
|
-
f
|
|
895
|
+
f"{index_name} requires the processor extra: "
|
|
893
896
|
f'pip install "pygeofetch[processor]"'
|
|
894
897
|
) from exc
|
|
895
898
|
|
|
@@ -904,7 +907,10 @@ class AdaptiveDownloader:
|
|
|
904
907
|
(
|
|
905
908
|
p
|
|
906
909
|
for p in result.output_paths
|
|
907
|
-
if any(
|
|
910
|
+
if any(
|
|
911
|
+
pat in p.name or pat.lower() in p.name.lower()
|
|
912
|
+
for pat in patterns
|
|
913
|
+
)
|
|
908
914
|
),
|
|
909
915
|
None,
|
|
910
916
|
)
|
|
@@ -914,8 +920,15 @@ class AdaptiveDownloader:
|
|
|
914
920
|
f"{band_name} (looked for {patterns} in filenames)"
|
|
915
921
|
)
|
|
916
922
|
with rasterio.open(path) as src:
|
|
917
|
-
arr =
|
|
918
|
-
|
|
923
|
+
arr = (
|
|
924
|
+
src.read(
|
|
925
|
+
1,
|
|
926
|
+
out_shape=ref_shape,
|
|
927
|
+
resampling=rasterio.enums.Resampling.bilinear,
|
|
928
|
+
)
|
|
929
|
+
if ref_shape
|
|
930
|
+
else src.read(1)
|
|
931
|
+
)
|
|
919
932
|
if ref_shape is None:
|
|
920
933
|
ref_shape = arr.shape
|
|
921
934
|
profile = src.profile.copy()
|
|
@@ -929,6 +942,12 @@ class AdaptiveDownloader:
|
|
|
929
942
|
arr = np.where(arr == src.nodata, np.nan, arr)
|
|
930
943
|
band_arrays[band_name] = arr
|
|
931
944
|
|
|
945
|
+
assert ref_shape is not None and profile is not None, (
|
|
946
|
+
f"{index_name}: required_bands was empty or the loop never "
|
|
947
|
+
f"ran -- ref_shape/profile must be set by the first band "
|
|
948
|
+
f"processed above."
|
|
949
|
+
)
|
|
950
|
+
|
|
932
951
|
si = SpectralIndex(prefer_spyndex=False)
|
|
933
952
|
index_arr = np.asarray(si.compute(index_name, **band_arrays), dtype=np.float32)
|
|
934
953
|
# Any band being NaN at a pixel must make the index NaN there too,
|
|
@@ -950,7 +969,6 @@ class AdaptiveDownloader:
|
|
|
950
969
|
logger.info(f"{index_name} computed → {out_path}")
|
|
951
970
|
return result
|
|
952
971
|
|
|
953
|
-
|
|
954
972
|
def _find_existing_download(
|
|
955
973
|
self, data: "SatelliteData", destination: Path
|
|
956
974
|
) -> Path | None:
|
|
@@ -1230,7 +1248,8 @@ class AdaptiveDownloader:
|
|
|
1230
1248
|
"%s has no direct CRS/transform, using its %d "
|
|
1231
1249
|
"ground control points for reprojection instead "
|
|
1232
1250
|
"(common for raw SAR/GRD delivery).",
|
|
1233
|
-
source_path.name,
|
|
1251
|
+
source_path.name,
|
|
1252
|
+
len(src_gcps),
|
|
1234
1253
|
)
|
|
1235
1254
|
elif src_crs is None:
|
|
1236
1255
|
raise RuntimeError(
|
|
@@ -1301,7 +1320,6 @@ class AdaptiveDownloader:
|
|
|
1301
1320
|
)
|
|
1302
1321
|
raise RuntimeError(msg)
|
|
1303
1322
|
|
|
1304
|
-
|
|
1305
1323
|
def _backoff(self, attempt: int, options: DownloadOptions) -> float:
|
|
1306
1324
|
"""Calculate retry delay based on strategy."""
|
|
1307
1325
|
import random
|
|
@@ -1334,4 +1352,4 @@ class AdaptiveDownloader:
|
|
|
1334
1352
|
@property
|
|
1335
1353
|
def _result_total_size_mb(self) -> float:
|
|
1336
1354
|
"""Property used by result objects."""
|
|
1337
|
-
return 0.0
|
|
1355
|
+
return 0.0
|