fimeval 0.1.57__tar.gz → 0.1.58__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fimeval-0.1.57 → fimeval-0.1.58}/PKG-INFO +14 -8
- {fimeval-0.1.57 → fimeval-0.1.58}/README.md +13 -7
- {fimeval-0.1.57 → fimeval-0.1.58}/pyproject.toml +1 -1
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/BenchFIMQuery/__init__.py +1 -1
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/BenchFIMQuery/access_benchfim.py +150 -87
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/BenchFIMQuery/utilis.py +72 -7
- fimeval-0.1.58/src/fimeval/BuildingFootprint/__init__.py +4 -0
- fimeval-0.1.58/src/fimeval/BuildingFootprint/arcgis_API.py +195 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/BuildingFootprint/evaluationwithBF.py +21 -63
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/ContingencyMap/__init__.py +2 -2
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/ContingencyMap/evaluationFIM.py +45 -24
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/ContingencyMap/printcontingency.py +3 -1
- fimeval-0.1.58/src/fimeval/ContingencyMap/water_bodies.py +175 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/__init__.py +7 -2
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/setup_benchFIM.py +2 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/utilis.py +6 -8
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval.egg-info/PKG-INFO +14 -8
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval.egg-info/SOURCES.txt +2 -2
- {fimeval-0.1.57 → fimeval-0.1.58}/tests/test_accessbenchmarkFIM.py +12 -10
- {fimeval-0.1.57 → fimeval-0.1.58}/tests/test_evaluationfim.py +17 -20
- fimeval-0.1.57/src/fimeval/BuildingFootprint/__init__.py +0 -3
- fimeval-0.1.57/src/fimeval/BuildingFootprint/microsoftBF.py +0 -134
- fimeval-0.1.57/src/fimeval/ContingencyMap/PWBs3.py +0 -42
- {fimeval-0.1.57 → fimeval-0.1.58}/LICENSE.txt +0 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/setup.cfg +0 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/ContingencyMap/methods.py +0 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/ContingencyMap/metrics.py +0 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval/ContingencyMap/plotevaluationmetrics.py +0 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval.egg-info/dependency_links.txt +0 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval.egg-info/requires.txt +0 -0
- {fimeval-0.1.57 → fimeval-0.1.58}/src/fimeval.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fimeval
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.58
|
|
4
4
|
Summary: A Framework for Automatic Evaluation of Flood Inundation Mapping Predictions Evaluation
|
|
5
5
|
Author: Surface Dynamics Modeling Lab
|
|
6
6
|
Author-email: Supath Dhital <sdhital@crimson.ua.edu>, Dipshika Devi <ddevi@ua.edu>
|
|
@@ -720,11 +720,15 @@ fimeval/
|
|
|
720
720
|
├── docs/ # Documentation (contains 'FIMserv' Tool usage sample codes)
|
|
721
721
|
│ └── sampledata/ # Contains the sample data to demonstrate how this frameworks works
|
|
722
722
|
│ └── fimeval_usage.ipynb #Sample code usage of the Evaluation framework
|
|
723
|
+
│ └── fimbench_usage.ipynb #Sample code usage of the FIMbench Query and getting benchmark dataset
|
|
723
724
|
├── Images/ # have sample images for documentation
|
|
724
725
|
├── src/
|
|
725
|
-
│ └── fimeval/
|
|
726
|
+
│ └── fimeval/
|
|
727
|
+
│ ├──BenchFIMQuery/ #Module to interact with the extensive FIMdatabase, hosted in AWS S3
|
|
728
|
+
│ │ └── access_benchfim.py #Different classes to query right benchmark FIM for any given location and set of filter
|
|
729
|
+
│ │ └── utilis.py #Support utility
|
|
726
730
|
│ ├──BuildingFootprint/ # Contains the evaluation of model predicted FIM with microsoft building footprint
|
|
727
|
-
│ │ └──
|
|
731
|
+
│ │ └── arcgis_API.py #seamless integration of building footprint in evaluation through ArcGIS REST API
|
|
728
732
|
│ │ └── evaluationwithBF.py
|
|
729
733
|
│ └── ContingencyMap/ # Contains all the metrics calculation and contingency map generation
|
|
730
734
|
│ │ ├── evaluationFIM.py # main evaluation moodule
|
|
@@ -732,14 +736,14 @@ fimeval/
|
|
|
732
736
|
│ │ └── metrics.py # metrics calculation module
|
|
733
737
|
│ │ └── plotevaluationmetrics.py # use to vizualize the different performance metrics
|
|
734
738
|
│ │ └── printcontingency.py # prints the contingency map to quickly generate the Map layout
|
|
735
|
-
│ │ └──
|
|
739
|
+
│ │ └── water_bodies.py # module which to get permanent water bodies from s3 bucket and ArcGIS REST API
|
|
736
740
|
│ └── utilis.py #Includes the resampling and reprojection of FIMs
|
|
737
741
|
└── tests/ # Includes test cases for different functionality
|
|
738
742
|
```
|
|
739
743
|
The graphical representation of fimeval pipeline can be summarized as follows in **```Figure 1```**. Here, it will show all the steps incorporated within the ```fimeval``` during packaging and all functionality are interconnected to each other, resulting the automation of the framework.
|
|
740
744
|
|
|
741
745
|
<div align="center">
|
|
742
|
-
<img width="
|
|
746
|
+
<img width="800" alt="image" src="./Images/flowchart.jpg">
|
|
743
747
|
</div>
|
|
744
748
|
Figure 1: Flowchart showing the entire framework pipeline.
|
|
745
749
|
|
|
@@ -781,7 +785,7 @@ main_dir = Path('./path/to/main/dir')
|
|
|
781
785
|
```
|
|
782
786
|
|
|
783
787
|
#### **Permanent Water Bodies (PWB)**
|
|
784
|
-
This framework uses PWB to first to delineate the PWB in the FIM and assign into different class so that the evaluation will be more fair. For the Contiguous United States (CONUS), the PWB is already integrated within the framework however, if user have more accurate PWB or using fimeval for outside US they can initialize and use PWB within fimeval framework. Currently it is using PWB publicly hosted by ESRI: https://hub.arcgis.com/datasets/esri::usa-detailed-water-bodies/about
|
|
788
|
+
This framework uses PWB to first to delineate the PWB in the FIM and assign into different class so that the evaluation will be more fair. For the Contiguous United States (CONUS), the PWB is already integrated within the framework however, if user have more accurate PWB or using fimeval for outside US they can initialize and use PWB within fimeval framework. Currently it is using PWB publicly hosted by ESRI through REST API: https://hub.arcgis.com/datasets/esri::usa-detailed-water-bodies/about
|
|
785
789
|
|
|
786
790
|
If user have more precise PWB, they can input their own PWB boundary as .shp and .gpkg format and need to assign the shapefile of the PWB and define directory as,
|
|
787
791
|
```bash
|
|
@@ -826,7 +830,7 @@ Table 1: Modules in `fimeval` are in order of execution.
|
|
|
826
830
|
| `EvaluateFIM` | It runs all the evaluation of FIM between B-FIM and M-FIMs. | `main_dir`: Main directory containing the case study folders, <br> `method_name`: How users wants to evaluate their FIM, <br> `outpur_dir`: Output directory where all the results and the intermidiate files will be saved for further calculation, <br> *`PWB_dir`*: The permanenet water bodies vectory file directory if user wants to user their own boundary, <br> *`target_crs`*: this fimeval framework needs the floodmaps to be in projected CRS so define the projected CRS in epsg code format, <br> *`target_resolution`*: sometime if the benchmark is very high resolution than candidate FIMs, it needs heavy computational time, so user can define the resolution if there FIMs are in different spatial resolution, else it will use the coarser resolution among all FIMS within that case. |The outputs includes generated files in TIFF, SHP, CSV, and PNG formats, all stored within the output folder. Users can visualize the TIFF files using any geospatial platform. The TIFF files consist of the binary Benchmark-FIM (Benchmark.tif), Model-FIM (Candidate.tif), and Agreement-FIM (Contingency.tif). The shp files contain the boundary of the generated flood extent.|
|
|
827
831
|
| `PlotContingencyMap` | For better understanding, It will print the agreement maps derived in first step. | `main_dir`, `method_name`, `output_dir` : Based on the those arguments, once all the evaluation is done, it will dynamically get the corresponding contingency raster for printing.| This prints the contingency map showing different class of evaluation (TP, FP, no data, PWB etc). The outputs look like- Figure 4 first row.|
|
|
828
832
|
| `PlotEvaluationMetrics` | For quick understanding of the evaluation metrics, to plot bar of evaluation scores. | `main_dir`, `method_name`, `output_dir` : Based on the those arguments, once all the evaluation is done, it will dynamically get the corresponding file for printing based on all those info.| This prints the bar plots which includes different performance metrics calculated by EvaluateFIM module. The outputs look like- Figure 4 second row.|
|
|
829
|
-
| `EvaluationWithBuildingFootprint` | For Building Footprint Analysis, user can specify shapefile of building footprints as .shp or .gpkg format. By default it consider global Microsoft building footprint dataset
|
|
833
|
+
| `EvaluationWithBuildingFootprint` | For Building Footprint Analysis, user can specify shapefile of building footprints as .shp or .gpkg format. By default it consider global Microsoft building footprint dataset hosted in ArcGIS Online. It is seamlessly integrated within framework through ArcGIS REST API. | `main_dir`, `method_name`, `output_dir`: Those arguments are as it is, same as all other modules. <br> *`building_footprint`*: If user wants to use their own building footprint file then pass the directory here, *`shapefile_dir`*: this is the directory of user defined AOI if user is working with their own boundary and automatic Building footprint download and evaluation,| It will calculate the different metrics (e.g. TP, FP, CSI, F1, Accuracy etc) based on hit and miss of building on different M-FIM and B-FIM. Those all metrics will be saved as CSV format in `output_dir` and finally using that info it prints the counts of building foorpint in each FIMs as well as scenario on the evaluation end via bar plot.|
|
|
830
834
|
|
|
831
835
|
<p align="center">
|
|
832
836
|
<img src="./Images/methodsresults_combined.jpg" width="750" />
|
|
@@ -909,7 +913,9 @@ pip install fimeval
|
|
|
909
913
|
|  | Funding for this project was provided by the National Oceanic & Atmospheric Administration (NOAA), awarded to the Cooperative Institute for Research to Operations in Hydrology (CIROH) through the NOAA Cooperative Agreement with The University of Alabama.
|
|
910
914
|
|
|
911
915
|
### For More Information
|
|
916
|
+
|
|
912
917
|
Contact <a href="https://geography.ua.edu/people/sagy-cohen/" target="_blank">Sagy Cohen</a>
|
|
913
918
|
(sagy.cohen@ua.edu)
|
|
919
|
+
Supath Dhital, (sdhital@crimson.ua.edu)
|
|
914
920
|
Dipsikha Devi, (ddevi@ua.edu)
|
|
915
|
-
|
|
921
|
+
|
|
@@ -30,11 +30,15 @@ fimeval/
|
|
|
30
30
|
├── docs/ # Documentation (contains 'FIMserv' Tool usage sample codes)
|
|
31
31
|
│ └── sampledata/ # Contains the sample data to demonstrate how this frameworks works
|
|
32
32
|
│ └── fimeval_usage.ipynb #Sample code usage of the Evaluation framework
|
|
33
|
+
│ └── fimbench_usage.ipynb #Sample code usage of the FIMbench Query and getting benchmark dataset
|
|
33
34
|
├── Images/ # have sample images for documentation
|
|
34
35
|
├── src/
|
|
35
|
-
│ └── fimeval/
|
|
36
|
+
│ └── fimeval/
|
|
37
|
+
│ ├──BenchFIMQuery/ #Module to interact with the extensive FIMdatabase, hosted in AWS S3
|
|
38
|
+
│ │ └── access_benchfim.py #Different classes to query right benchmark FIM for any given location and set of filter
|
|
39
|
+
│ │ └── utilis.py #Support utility
|
|
36
40
|
│ ├──BuildingFootprint/ # Contains the evaluation of model predicted FIM with microsoft building footprint
|
|
37
|
-
│ │ └──
|
|
41
|
+
│ │ └── arcgis_API.py #seamless integration of building footprint in evaluation through ArcGIS REST API
|
|
38
42
|
│ │ └── evaluationwithBF.py
|
|
39
43
|
│ └── ContingencyMap/ # Contains all the metrics calculation and contingency map generation
|
|
40
44
|
│ │ ├── evaluationFIM.py # main evaluation moodule
|
|
@@ -42,14 +46,14 @@ fimeval/
|
|
|
42
46
|
│ │ └── metrics.py # metrics calculation module
|
|
43
47
|
│ │ └── plotevaluationmetrics.py # use to vizualize the different performance metrics
|
|
44
48
|
│ │ └── printcontingency.py # prints the contingency map to quickly generate the Map layout
|
|
45
|
-
│ │ └──
|
|
49
|
+
│ │ └── water_bodies.py # module which to get permanent water bodies from s3 bucket and ArcGIS REST API
|
|
46
50
|
│ └── utilis.py #Includes the resampling and reprojection of FIMs
|
|
47
51
|
└── tests/ # Includes test cases for different functionality
|
|
48
52
|
```
|
|
49
53
|
The graphical representation of fimeval pipeline can be summarized as follows in **```Figure 1```**. Here, it will show all the steps incorporated within the ```fimeval``` during packaging and all functionality are interconnected to each other, resulting the automation of the framework.
|
|
50
54
|
|
|
51
55
|
<div align="center">
|
|
52
|
-
<img width="
|
|
56
|
+
<img width="800" alt="image" src="./Images/flowchart.jpg">
|
|
53
57
|
</div>
|
|
54
58
|
Figure 1: Flowchart showing the entire framework pipeline.
|
|
55
59
|
|
|
@@ -91,7 +95,7 @@ main_dir = Path('./path/to/main/dir')
|
|
|
91
95
|
```
|
|
92
96
|
|
|
93
97
|
#### **Permanent Water Bodies (PWB)**
|
|
94
|
-
This framework uses PWB to first to delineate the PWB in the FIM and assign into different class so that the evaluation will be more fair. For the Contiguous United States (CONUS), the PWB is already integrated within the framework however, if user have more accurate PWB or using fimeval for outside US they can initialize and use PWB within fimeval framework. Currently it is using PWB publicly hosted by ESRI: https://hub.arcgis.com/datasets/esri::usa-detailed-water-bodies/about
|
|
98
|
+
This framework uses PWB to first to delineate the PWB in the FIM and assign into different class so that the evaluation will be more fair. For the Contiguous United States (CONUS), the PWB is already integrated within the framework however, if user have more accurate PWB or using fimeval for outside US they can initialize and use PWB within fimeval framework. Currently it is using PWB publicly hosted by ESRI through REST API: https://hub.arcgis.com/datasets/esri::usa-detailed-water-bodies/about
|
|
95
99
|
|
|
96
100
|
If user have more precise PWB, they can input their own PWB boundary as .shp and .gpkg format and need to assign the shapefile of the PWB and define directory as,
|
|
97
101
|
```bash
|
|
@@ -136,7 +140,7 @@ Table 1: Modules in `fimeval` are in order of execution.
|
|
|
136
140
|
| `EvaluateFIM` | It runs all the evaluation of FIM between B-FIM and M-FIMs. | `main_dir`: Main directory containing the case study folders, <br> `method_name`: How users wants to evaluate their FIM, <br> `outpur_dir`: Output directory where all the results and the intermidiate files will be saved for further calculation, <br> *`PWB_dir`*: The permanenet water bodies vectory file directory if user wants to user their own boundary, <br> *`target_crs`*: this fimeval framework needs the floodmaps to be in projected CRS so define the projected CRS in epsg code format, <br> *`target_resolution`*: sometime if the benchmark is very high resolution than candidate FIMs, it needs heavy computational time, so user can define the resolution if there FIMs are in different spatial resolution, else it will use the coarser resolution among all FIMS within that case. |The outputs includes generated files in TIFF, SHP, CSV, and PNG formats, all stored within the output folder. Users can visualize the TIFF files using any geospatial platform. The TIFF files consist of the binary Benchmark-FIM (Benchmark.tif), Model-FIM (Candidate.tif), and Agreement-FIM (Contingency.tif). The shp files contain the boundary of the generated flood extent.|
|
|
137
141
|
| `PlotContingencyMap` | For better understanding, It will print the agreement maps derived in first step. | `main_dir`, `method_name`, `output_dir` : Based on the those arguments, once all the evaluation is done, it will dynamically get the corresponding contingency raster for printing.| This prints the contingency map showing different class of evaluation (TP, FP, no data, PWB etc). The outputs look like- Figure 4 first row.|
|
|
138
142
|
| `PlotEvaluationMetrics` | For quick understanding of the evaluation metrics, to plot bar of evaluation scores. | `main_dir`, `method_name`, `output_dir` : Based on the those arguments, once all the evaluation is done, it will dynamically get the corresponding file for printing based on all those info.| This prints the bar plots which includes different performance metrics calculated by EvaluateFIM module. The outputs look like- Figure 4 second row.|
|
|
139
|
-
| `EvaluationWithBuildingFootprint` | For Building Footprint Analysis, user can specify shapefile of building footprints as .shp or .gpkg format. By default it consider global Microsoft building footprint dataset
|
|
143
|
+
| `EvaluationWithBuildingFootprint` | For Building Footprint Analysis, user can specify shapefile of building footprints as .shp or .gpkg format. By default it consider global Microsoft building footprint dataset hosted in ArcGIS Online. It is seamlessly integrated within framework through ArcGIS REST API. | `main_dir`, `method_name`, `output_dir`: Those arguments are as it is, same as all other modules. <br> *`building_footprint`*: If user wants to use their own building footprint file then pass the directory here, *`shapefile_dir`*: this is the directory of user defined AOI if user is working with their own boundary and automatic Building footprint download and evaluation,| It will calculate the different metrics (e.g. TP, FP, CSI, F1, Accuracy etc) based on hit and miss of building on different M-FIM and B-FIM. Those all metrics will be saved as CSV format in `output_dir` and finally using that info it prints the counts of building foorpint in each FIMs as well as scenario on the evaluation end via bar plot.|
|
|
140
144
|
|
|
141
145
|
<p align="center">
|
|
142
146
|
<img src="./Images/methodsresults_combined.jpg" width="750" />
|
|
@@ -219,7 +223,9 @@ pip install fimeval
|
|
|
219
223
|
|  | Funding for this project was provided by the National Oceanic & Atmospheric Administration (NOAA), awarded to the Cooperative Institute for Research to Operations in Hydrology (CIROH) through the NOAA Cooperative Agreement with The University of Alabama.
|
|
220
224
|
|
|
221
225
|
### For More Information
|
|
226
|
+
|
|
222
227
|
Contact <a href="https://geography.ua.edu/people/sagy-cohen/" target="_blank">Sagy Cohen</a>
|
|
223
228
|
(sagy.cohen@ua.edu)
|
|
229
|
+
Supath Dhital, (sdhital@crimson.ua.edu)
|
|
224
230
|
Dipsikha Devi, (ddevi@ua.edu)
|
|
225
|
-
|
|
231
|
+
|
|
@@ -8,7 +8,7 @@ from __future__ import annotations
|
|
|
8
8
|
from typing import Optional, Dict, Any, List, Tuple
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
import os
|
|
11
|
-
import json
|
|
11
|
+
import json
|
|
12
12
|
|
|
13
13
|
import rasterio
|
|
14
14
|
from rasterio.warp import transform_bounds
|
|
@@ -24,13 +24,15 @@ from .utilis import (
|
|
|
24
24
|
_to_hour_or_none,
|
|
25
25
|
_record_day,
|
|
26
26
|
_record_hour_or_none,
|
|
27
|
-
_pretty_date_for_print,
|
|
27
|
+
_pretty_date_for_print,
|
|
28
|
+
_ensure_local_gpkg,
|
|
28
29
|
)
|
|
29
30
|
|
|
30
31
|
# Preferred area CRSs for area calculations
|
|
31
|
-
AREA_CRS_US = "EPSG:5070"
|
|
32
|
+
AREA_CRS_US = "EPSG:5070" # for CONUS
|
|
32
33
|
AREA_CRS_GLOBAL = "EPSG:6933" # WGS 84 / NSIDC EASE-Grid 2.0 Global (equal-area) for rest of world: in future if the data is added into the catalog.
|
|
33
34
|
|
|
35
|
+
|
|
34
36
|
# Helper: pretty-print container so that print(response) shows the structured text.
|
|
35
37
|
# If "printable" is empty (e.g., download=True), nothing is printed.
|
|
36
38
|
class PrettyDict(dict):
|
|
@@ -40,6 +42,7 @@ class PrettyDict(dict):
|
|
|
40
42
|
return txt
|
|
41
43
|
# Empty string when we do not want anything printed (e.g., download=True)
|
|
42
44
|
return ""
|
|
45
|
+
|
|
43
46
|
__repr__ = __str__
|
|
44
47
|
|
|
45
48
|
|
|
@@ -106,7 +109,8 @@ def _record_bbox_polygon(rec: Dict[str, Any]) -> Polygon:
|
|
|
106
109
|
minx, miny, maxx, maxy = _get_record_bbox_xy(rec)
|
|
107
110
|
return box(minx, miny, maxx, maxy)
|
|
108
111
|
|
|
109
|
-
|
|
112
|
+
|
|
113
|
+
# Return AOI polygon in WGS84 from a raster file --> this will be useful when user have model predicted raster and looking for the benchmrk FIM into the database
|
|
110
114
|
def _raster_aoi_polygon_wgs84(path: str) -> Polygon:
|
|
111
115
|
with rasterio.open(path) as ds:
|
|
112
116
|
if ds.crs is None:
|
|
@@ -117,24 +121,29 @@ def _raster_aoi_polygon_wgs84(path: str) -> Polygon:
|
|
|
117
121
|
)
|
|
118
122
|
return box(minx, miny, maxx, maxy)
|
|
119
123
|
|
|
120
|
-
|
|
124
|
+
|
|
125
|
+
# Return AOI polygon in WGS84 from a vector file --> this will be useful when user have model predicted vector and looking for the benchmrk FIM into the database
|
|
121
126
|
def _vector_aoi_polygon_wgs84(path: str) -> Polygon:
|
|
122
127
|
gdf = gpd.read_file(path)
|
|
123
128
|
if gdf.empty:
|
|
124
129
|
raise ValueError(f"Vector file {path} contains no features.")
|
|
125
130
|
if gdf.crs is None:
|
|
126
|
-
raise ValueError(
|
|
131
|
+
raise ValueError(
|
|
132
|
+
f"Vector file {path} has no CRS; cannot derive WGS84 geometry."
|
|
133
|
+
)
|
|
127
134
|
gdf = gdf.to_crs("EPSG:4326")
|
|
128
135
|
geom = unary_union(gdf.geometry)
|
|
129
136
|
if geom.is_empty:
|
|
130
137
|
raise ValueError(f"Vector file {path} has empty geometry after union.")
|
|
131
138
|
return geom
|
|
132
139
|
|
|
140
|
+
|
|
133
141
|
def _ensure_dir(path: str | Path) -> Path:
|
|
134
142
|
p = Path(path)
|
|
135
143
|
p.mkdir(parents=True, exist_ok=True)
|
|
136
144
|
return p
|
|
137
145
|
|
|
146
|
+
|
|
138
147
|
# benchmark FIM filtering by date
|
|
139
148
|
def _filter_by_date_exact(
|
|
140
149
|
records: List[Dict[str, Any]],
|
|
@@ -167,7 +176,8 @@ def _filter_by_date_exact(
|
|
|
167
176
|
out.append(r)
|
|
168
177
|
return out
|
|
169
178
|
|
|
170
|
-
|
|
179
|
+
|
|
180
|
+
# Filter available benchmark FIMs by date range
|
|
171
181
|
def _filter_by_date_range(
|
|
172
182
|
records: List[Dict[str, Any]],
|
|
173
183
|
start_date: Optional[str],
|
|
@@ -191,6 +201,7 @@ def _filter_by_date_range(
|
|
|
191
201
|
out.append(r)
|
|
192
202
|
return out
|
|
193
203
|
|
|
204
|
+
|
|
194
205
|
# Dynamic area CRS selection and overlap stats
|
|
195
206
|
def _pick_area_crs_for_bounds(bounds: Tuple[float, float, float, float]) -> str:
|
|
196
207
|
"""
|
|
@@ -219,7 +230,8 @@ def _pick_area_crs_for_bounds(bounds: Tuple[float, float, float, float]) -> str:
|
|
|
219
230
|
return AREA_CRS_US
|
|
220
231
|
return AREA_CRS_GLOBAL
|
|
221
232
|
|
|
222
|
-
|
|
233
|
+
|
|
234
|
+
# Compute the area overlap statistics between user passed AOI/raster and benchmark AOI
|
|
223
235
|
def _compute_area_overlap_stats(
|
|
224
236
|
aoi_geom: Polygon,
|
|
225
237
|
benchmark_geom: Polygon,
|
|
@@ -280,6 +292,7 @@ def _aoi_context_str(
|
|
|
280
292
|
return ", ".join(parts)
|
|
281
293
|
# fall back to the non-AOI context from utils
|
|
282
294
|
from .utilis import _context_str as _ctx
|
|
295
|
+
|
|
283
296
|
return _ctx(
|
|
284
297
|
huc8=huc8,
|
|
285
298
|
date_input=date_input,
|
|
@@ -288,15 +301,28 @@ def _aoi_context_str(
|
|
|
288
301
|
end_date=end_date,
|
|
289
302
|
)
|
|
290
303
|
|
|
304
|
+
def _display_raster_name(rec: Dict[str, Any]) -> str:
|
|
305
|
+
tif_url = rec.get("tif_url")
|
|
306
|
+
if isinstance(tif_url, str) and tif_url.strip():
|
|
307
|
+
# drop querystring if any
|
|
308
|
+
tif_url = tif_url.split("?", 1)[0]
|
|
309
|
+
return os.path.basename(tif_url)
|
|
310
|
+
|
|
311
|
+
# fallback: last path token of id
|
|
312
|
+
rid = rec.get("id")
|
|
313
|
+
if isinstance(rid, str) and rid.strip():
|
|
314
|
+
return rid.strip().split("/")[-1] + ".tif"
|
|
315
|
+
|
|
316
|
+
return "NA"
|
|
291
317
|
|
|
292
318
|
# Build a single printable block, optionally with overlap appended
|
|
293
|
-
def _format_block_with_overlap(
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
tier = rec.get("tier") or rec.get("quality") or "Unknown"
|
|
319
|
+
def _format_block_with_overlap(
|
|
320
|
+
rec: Dict[str, Any], pct: Optional[float], km2: Optional[float]
|
|
321
|
+
) -> str:
|
|
322
|
+
tier = rec.get("tier") or rec.get("HWM") or rec.get("quality") or "Unknown"
|
|
297
323
|
res = rec.get("resolution_m")
|
|
298
324
|
res_txt = f"{res}m" if res is not None else "NA"
|
|
299
|
-
fname = rec
|
|
325
|
+
fname = _display_raster_name(rec)
|
|
300
326
|
|
|
301
327
|
lines = [f"Data Tier: {tier}"]
|
|
302
328
|
|
|
@@ -307,22 +333,27 @@ def _format_block_with_overlap(rec: Dict[str, Any],
|
|
|
307
333
|
date_str = _pretty_date_for_print(rec)
|
|
308
334
|
lines.append(f"Benchmark FIM date: {date_str}")
|
|
309
335
|
|
|
310
|
-
lines.extend(
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
336
|
+
lines.extend(
|
|
337
|
+
[
|
|
338
|
+
f"Spatial Resolution: {res_txt}",
|
|
339
|
+
f"Raster Filename in DB: {fname}",
|
|
340
|
+
]
|
|
341
|
+
)
|
|
314
342
|
|
|
315
343
|
if pct is not None and km2 is not None:
|
|
316
|
-
lines.append(
|
|
344
|
+
lines.append(
|
|
345
|
+
f"Overlap with respect to benchmark FIM: {pct:.1f}% / {km2:.2f} km²"
|
|
346
|
+
)
|
|
317
347
|
|
|
318
348
|
return "\n".join(lines)
|
|
319
349
|
|
|
320
|
-
|
|
350
|
+
|
|
351
|
+
# For Tier-4- adding synthetic event year while reflecting the outcomes
|
|
321
352
|
def _is_synthetic_tier(rec: Dict[str, Any]) -> bool:
|
|
322
353
|
"""Return True when the record is a synthetic (Tier_4) event."""
|
|
323
354
|
tier = str(rec.get("tier") or rec.get("quality") or "").lower()
|
|
324
355
|
return "tier_4" in tier or tier.strip() == "4"
|
|
325
|
-
|
|
356
|
+
|
|
326
357
|
|
|
327
358
|
def _return_period_text(rec: Dict[str, Any]) -> str:
|
|
328
359
|
"""
|
|
@@ -344,13 +375,19 @@ def _return_period_text(rec: Dict[str, Any]) -> str:
|
|
|
344
375
|
except Exception:
|
|
345
376
|
return f"{rp} synthetic flow"
|
|
346
377
|
|
|
347
|
-
|
|
378
|
+
|
|
379
|
+
# helpers to read AOI GPKG geometry directly
|
|
348
380
|
def _storage_options_for_uri(uri: str) -> Optional[Dict[str, Any]]:
|
|
349
381
|
if isinstance(uri, str) and uri.startswith("s3://"):
|
|
350
|
-
anon = str(os.environ.get("AWS_NO_SIGN_REQUEST", "")).upper() in {
|
|
382
|
+
anon = str(os.environ.get("AWS_NO_SIGN_REQUEST", "")).upper() in {
|
|
383
|
+
"YES",
|
|
384
|
+
"TRUE",
|
|
385
|
+
"1",
|
|
386
|
+
}
|
|
351
387
|
return {"anon": anon}
|
|
352
388
|
return None
|
|
353
389
|
|
|
390
|
+
|
|
354
391
|
def _gpkg_urls_from_record(rec: Dict[str, Any]) -> List[str]:
|
|
355
392
|
urls: List[str] = []
|
|
356
393
|
for key in ("aoi_gpkg", "aoi_gpkg_url", "gpkg_url"):
|
|
@@ -382,23 +419,36 @@ def _gpkg_urls_from_record(rec: Dict[str, Any]) -> List[str]:
|
|
|
382
419
|
return out
|
|
383
420
|
|
|
384
421
|
def _read_benchmark_aoi_union_geom(rec: Dict[str, Any]) -> Optional[Polygon]:
|
|
385
|
-
# Read and union AOI geometries referenced by the record
|
|
422
|
+
# Read and union AOI geometries referenced by the record
|
|
386
423
|
urls = _gpkg_urls_from_record(rec)
|
|
387
424
|
if not urls:
|
|
388
425
|
return None
|
|
426
|
+
|
|
389
427
|
geoms: List[Polygon] = []
|
|
390
428
|
for uri in urls:
|
|
391
429
|
try:
|
|
392
430
|
storage_opts = _storage_options_for_uri(uri)
|
|
393
|
-
|
|
431
|
+
|
|
432
|
+
uri_to_read = _ensure_local_gpkg(uri)
|
|
433
|
+
|
|
434
|
+
gdf = (
|
|
435
|
+
gpd.read_file(uri_to_read, storage_options=storage_opts)
|
|
436
|
+
if storage_opts
|
|
437
|
+
else gpd.read_file(uri_to_read)
|
|
438
|
+
)
|
|
394
439
|
if gdf.empty:
|
|
395
440
|
continue
|
|
396
|
-
gdf =
|
|
441
|
+
gdf = (
|
|
442
|
+
gdf.to_crs("EPSG:4326")
|
|
443
|
+
if gdf.crs
|
|
444
|
+
else gdf.set_crs("EPSG:4326", allow_override=True)
|
|
445
|
+
)
|
|
397
446
|
u = unary_union(gdf.geometry)
|
|
398
447
|
if not u.is_empty:
|
|
399
448
|
geoms.append(u)
|
|
400
|
-
except Exception:
|
|
449
|
+
except Exception as e:
|
|
401
450
|
continue
|
|
451
|
+
|
|
402
452
|
if not geoms:
|
|
403
453
|
return None
|
|
404
454
|
uall = unary_union(geoms)
|
|
@@ -419,6 +469,7 @@ class benchFIMquery:
|
|
|
419
469
|
into a local directory
|
|
420
470
|
- or, as a special-case, fetch a specific benchmark by its filename.
|
|
421
471
|
"""
|
|
472
|
+
|
|
422
473
|
def __init__(self, catalog: Optional[Dict[str, Any]] = None) -> None:
|
|
423
474
|
"""
|
|
424
475
|
Create a new service.
|
|
@@ -510,12 +561,14 @@ class benchFIMquery:
|
|
|
510
561
|
)
|
|
511
562
|
|
|
512
563
|
if download and not out_dir:
|
|
513
|
-
return PrettyDict(
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
564
|
+
return PrettyDict(
|
|
565
|
+
{
|
|
566
|
+
"status": "error",
|
|
567
|
+
"message": "When download=True, you must provide out_dir.",
|
|
568
|
+
"matches": [],
|
|
569
|
+
"printable": "",
|
|
570
|
+
}
|
|
571
|
+
)
|
|
519
572
|
|
|
520
573
|
# Direct filename-only workflow (no AOI, no dates)
|
|
521
574
|
if (
|
|
@@ -535,9 +588,7 @@ class benchFIMquery:
|
|
|
535
588
|
]
|
|
536
589
|
if not candidates:
|
|
537
590
|
candidates = [
|
|
538
|
-
r
|
|
539
|
-
for r in recs
|
|
540
|
-
if str(r.get("file_name", "")).strip() == fname
|
|
591
|
+
r for r in recs if str(r.get("file_name", "")).strip() == fname
|
|
541
592
|
]
|
|
542
593
|
else:
|
|
543
594
|
candidates = [
|
|
@@ -545,39 +596,41 @@ class benchFIMquery:
|
|
|
545
596
|
]
|
|
546
597
|
|
|
547
598
|
if not candidates:
|
|
548
|
-
return PrettyDict(
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
599
|
+
return PrettyDict(
|
|
600
|
+
{
|
|
601
|
+
"status": "not_found",
|
|
602
|
+
"message": f"File name {fname!r} not found in catalog.",
|
|
603
|
+
"matches": [],
|
|
604
|
+
"printable": "",
|
|
605
|
+
}
|
|
606
|
+
)
|
|
554
607
|
|
|
555
608
|
target = candidates[0]
|
|
556
609
|
out_dir_path = _ensure_dir(out_dir)
|
|
557
610
|
dl = download_fim_assets(target, str(out_dir_path))
|
|
558
611
|
|
|
559
|
-
return PrettyDict(
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
612
|
+
return PrettyDict(
|
|
613
|
+
{
|
|
614
|
+
"status": "ok",
|
|
615
|
+
"message": f"Downloaded benchmark FIM '{fname}' to '{out_dir_path}'.",
|
|
616
|
+
"matches": [
|
|
617
|
+
{
|
|
618
|
+
"record": target,
|
|
619
|
+
"bbox_intersects": False,
|
|
620
|
+
"intersection_area_pct": None,
|
|
621
|
+
"intersection_area_km2": None,
|
|
622
|
+
"downloads": dl,
|
|
623
|
+
}
|
|
624
|
+
],
|
|
625
|
+
"printable": "",
|
|
626
|
+
}
|
|
627
|
+
)
|
|
573
628
|
|
|
574
629
|
# AOI-based workflows
|
|
575
630
|
records = self.records
|
|
576
631
|
if huc8:
|
|
577
632
|
huc8_str = str(huc8).strip()
|
|
578
|
-
records = [
|
|
579
|
-
r for r in records if str(r.get("huc8", "")).strip() == huc8_str
|
|
580
|
-
]
|
|
633
|
+
records = [r for r in records if str(r.get("huc8", "")).strip() == huc8_str]
|
|
581
634
|
|
|
582
635
|
# Date filters
|
|
583
636
|
if event_date:
|
|
@@ -586,12 +639,14 @@ class benchFIMquery:
|
|
|
586
639
|
records = _filter_by_date_range(records, start_date, end_date)
|
|
587
640
|
|
|
588
641
|
if not records:
|
|
589
|
-
return PrettyDict(
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
642
|
+
return PrettyDict(
|
|
643
|
+
{
|
|
644
|
+
"status": "not_found",
|
|
645
|
+
"message": "No catalog records match the provided filters.",
|
|
646
|
+
"matches": [],
|
|
647
|
+
"printable": "",
|
|
648
|
+
}
|
|
649
|
+
)
|
|
595
650
|
|
|
596
651
|
# If no AOI is provided at all
|
|
597
652
|
if aoi_geom is None:
|
|
@@ -615,14 +670,14 @@ class benchFIMquery:
|
|
|
615
670
|
end_date=end_date,
|
|
616
671
|
file_name=file_name,
|
|
617
672
|
)
|
|
618
|
-
printable = format_records_for_print(
|
|
673
|
+
printable = format_records_for_print(
|
|
674
|
+
[m["record"] for m in matches], context=ctx
|
|
675
|
+
)
|
|
619
676
|
|
|
620
677
|
if download:
|
|
621
678
|
out_dir_path = _ensure_dir(out_dir)
|
|
622
679
|
for m in matches:
|
|
623
|
-
m["downloads"] = download_fim_assets(
|
|
624
|
-
m["record"], str(out_dir_path)
|
|
625
|
-
)
|
|
680
|
+
m["downloads"] = download_fim_assets(m["record"], str(out_dir_path))
|
|
626
681
|
msg = (
|
|
627
682
|
f"Downloaded {len(matches)} benchmark record(s) "
|
|
628
683
|
f"to '{out_dir_path}'."
|
|
@@ -633,12 +688,14 @@ class benchFIMquery:
|
|
|
633
688
|
f"for the provided filters."
|
|
634
689
|
)
|
|
635
690
|
|
|
636
|
-
return PrettyDict(
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
691
|
+
return PrettyDict(
|
|
692
|
+
{
|
|
693
|
+
"status": "ok",
|
|
694
|
+
"message": msg,
|
|
695
|
+
"matches": matches,
|
|
696
|
+
"printable": "" if download else printable,
|
|
697
|
+
}
|
|
698
|
+
)
|
|
642
699
|
|
|
643
700
|
# AOI is present: intersect with bbox
|
|
644
701
|
intersecting: List[Dict[str, Any]] = []
|
|
@@ -652,12 +709,14 @@ class benchFIMquery:
|
|
|
652
709
|
intersecting.append(r)
|
|
653
710
|
|
|
654
711
|
if not intersecting:
|
|
655
|
-
return PrettyDict(
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
712
|
+
return PrettyDict(
|
|
713
|
+
{
|
|
714
|
+
"status": "not_found",
|
|
715
|
+
"message": "No benchmark FIM bbox intersects the provided AOI.",
|
|
716
|
+
"matches": [],
|
|
717
|
+
"printable": "",
|
|
718
|
+
}
|
|
719
|
+
)
|
|
661
720
|
|
|
662
721
|
out_matches: List[Dict[str, Any]] = []
|
|
663
722
|
out_dir_path = _ensure_dir(out_dir) if (download and out_dir) else None
|
|
@@ -725,12 +784,14 @@ class benchFIMquery:
|
|
|
725
784
|
blocks.append(_format_block_with_overlap(rec, None, None))
|
|
726
785
|
printable = header + "\n\n".join(blocks)
|
|
727
786
|
|
|
728
|
-
return PrettyDict(
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
787
|
+
return PrettyDict(
|
|
788
|
+
{
|
|
789
|
+
"status": "ok",
|
|
790
|
+
"message": msg,
|
|
791
|
+
"matches": out_matches,
|
|
792
|
+
"printable": printable,
|
|
793
|
+
}
|
|
794
|
+
)
|
|
734
795
|
|
|
735
796
|
def __call__(
|
|
736
797
|
self,
|
|
@@ -758,4 +819,6 @@ class benchFIMquery:
|
|
|
758
819
|
download=download,
|
|
759
820
|
out_dir=out_dir,
|
|
760
821
|
)
|
|
761
|
-
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
benchFIMquery = benchFIMquery()
|