rashdf 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rashdf-0.5.0 → rashdf-0.6.0}/PKG-INFO +6 -1
- {rashdf-0.5.0 → rashdf-0.6.0}/pyproject.toml +2 -2
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf/base.py +4 -1
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf/plan.py +236 -8
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf/utils.py +32 -2
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf.egg-info/PKG-INFO +6 -1
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf.egg-info/SOURCES.txt +1 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf.egg-info/requires.txt +5 -0
- rashdf-0.6.0/tests/test_base.py +20 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/tests/test_plan.py +196 -10
- {rashdf-0.5.0 → rashdf-0.6.0}/LICENSE +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/README.md +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/setup.cfg +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/src/cli.py +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf/__init__.py +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf/geom.py +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf.egg-info/dependency_links.txt +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf.egg-info/entry_points.txt +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/src/rashdf.egg-info/top_level.txt +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/tests/test_cli.py +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/tests/test_geom.py +0 -0
- {rashdf-0.5.0 → rashdf-0.6.0}/tests/test_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rashdf
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Read data from HEC-RAS HDF files.
|
|
5
5
|
Project-URL: repository, https://github.com/fema-ffrd/rashdf
|
|
6
6
|
Classifier: Development Status :: 4 - Beta
|
|
@@ -23,6 +23,11 @@ Requires-Dist: ruff; extra == "dev"
|
|
|
23
23
|
Requires-Dist: pytest; extra == "dev"
|
|
24
24
|
Requires-Dist: pytest-cov; extra == "dev"
|
|
25
25
|
Requires-Dist: fiona; extra == "dev"
|
|
26
|
+
Requires-Dist: kerchunk; extra == "dev"
|
|
27
|
+
Requires-Dist: zarr; extra == "dev"
|
|
28
|
+
Requires-Dist: dask; extra == "dev"
|
|
29
|
+
Requires-Dist: fsspec; extra == "dev"
|
|
30
|
+
Requires-Dist: s3fs; extra == "dev"
|
|
26
31
|
Provides-Extra: docs
|
|
27
32
|
Requires-Dist: sphinx; extra == "docs"
|
|
28
33
|
Requires-Dist: numpydoc; extra == "docs"
|
|
@@ -12,11 +12,11 @@ classifiers = [
|
|
|
12
12
|
"Programming Language :: Python :: 3.11",
|
|
13
13
|
"Programming Language :: Python :: 3.12",
|
|
14
14
|
]
|
|
15
|
-
version = "0.
|
|
15
|
+
version = "0.6.0"
|
|
16
16
|
dependencies = ["h5py", "geopandas>=1.0,<2.0", "pyarrow", "xarray"]
|
|
17
17
|
|
|
18
18
|
[project.optional-dependencies]
|
|
19
|
-
dev = ["pre-commit", "ruff", "pytest", "pytest-cov", "fiona"]
|
|
19
|
+
dev = ["pre-commit", "ruff", "pytest", "pytest-cov", "fiona", "kerchunk", "zarr", "dask", "fsspec", "s3fs"]
|
|
20
20
|
docs = ["sphinx", "numpydoc", "sphinx_rtd_theme"]
|
|
21
21
|
|
|
22
22
|
[project.urls]
|
|
@@ -19,6 +19,7 @@ class RasHdf(h5py.File):
|
|
|
19
19
|
Additional keyword arguments to pass to h5py.File
|
|
20
20
|
"""
|
|
21
21
|
super().__init__(name, mode="r", **kwargs)
|
|
22
|
+
self._loc = name
|
|
22
23
|
|
|
23
24
|
@classmethod
|
|
24
25
|
def open_uri(
|
|
@@ -49,7 +50,9 @@ class RasHdf(h5py.File):
|
|
|
49
50
|
import fsspec
|
|
50
51
|
|
|
51
52
|
remote_file = fsspec.open(uri, mode="rb", **fsspec_kwargs)
|
|
52
|
-
|
|
53
|
+
result = cls(remote_file.open(), **h5py_kwargs)
|
|
54
|
+
result._loc = uri
|
|
55
|
+
return result
|
|
53
56
|
|
|
54
57
|
def get_attrs(self, attr_path: str) -> Dict:
|
|
55
58
|
"""Convert attributes from a HEC-RAS HDF file into a Python dictionary for a given attribute path.
|
|
@@ -5,6 +5,7 @@ from .utils import (
|
|
|
5
5
|
df_datetimes_to_str,
|
|
6
6
|
ras_timesteps_to_datetimes,
|
|
7
7
|
parse_ras_datetime_ms,
|
|
8
|
+
deprecated,
|
|
8
9
|
)
|
|
9
10
|
|
|
10
11
|
from geopandas import GeoDataFrame
|
|
@@ -585,7 +586,8 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
585
586
|
Returns
|
|
586
587
|
-------
|
|
587
588
|
DataFrame
|
|
588
|
-
A DataFrame with columns 'mesh_name', 'cell_id' or 'face_id', a value column,
|
|
589
|
+
A DataFrame with columns 'mesh_name', 'cell_id' or 'face_id', a value column,
|
|
590
|
+
and a time column if the value corresponds to a specific time.
|
|
589
591
|
"""
|
|
590
592
|
methods_with_times = {
|
|
591
593
|
SummaryOutputVar.MAXIMUM_WATER_SURFACE: self.mesh_max_ws,
|
|
@@ -604,6 +606,76 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
604
606
|
df = other_methods[var]()
|
|
605
607
|
return df
|
|
606
608
|
|
|
609
|
+
def _mesh_summary_outputs_df(
|
|
610
|
+
self,
|
|
611
|
+
cells_or_faces: str,
|
|
612
|
+
output_vars: Optional[List[SummaryOutputVar]] = None,
|
|
613
|
+
round_to: str = "0.1 s",
|
|
614
|
+
) -> DataFrame:
|
|
615
|
+
if cells_or_faces == "cells":
|
|
616
|
+
feature_id_field = "cell_id"
|
|
617
|
+
elif cells_or_faces == "faces":
|
|
618
|
+
feature_id_field = "face_id"
|
|
619
|
+
else:
|
|
620
|
+
raise ValueError('cells_or_faces must be either "cells" or "faces".')
|
|
621
|
+
if output_vars is None:
|
|
622
|
+
summary_output_vars = self._summary_output_vars(
|
|
623
|
+
cells_or_faces=cells_or_faces
|
|
624
|
+
)
|
|
625
|
+
elif isinstance(output_vars, list):
|
|
626
|
+
summary_output_vars = []
|
|
627
|
+
for var in output_vars:
|
|
628
|
+
if not isinstance(var, SummaryOutputVar):
|
|
629
|
+
var = SummaryOutputVar(var)
|
|
630
|
+
summary_output_vars.append(var)
|
|
631
|
+
else:
|
|
632
|
+
raise ValueError(
|
|
633
|
+
"include_output must be a boolean or a list of SummaryOutputVar values."
|
|
634
|
+
)
|
|
635
|
+
df = self.mesh_summary_output(summary_output_vars[0], round_to=round_to)
|
|
636
|
+
for var in summary_output_vars[1:]:
|
|
637
|
+
df_var = self.mesh_summary_output(var, round_to=round_to)
|
|
638
|
+
df = df.merge(df_var, on=["mesh_name", feature_id_field], how="left")
|
|
639
|
+
return df
|
|
640
|
+
|
|
641
|
+
def mesh_cells_summary_output(self, round_to: str = "0.1 s") -> DataFrame:
|
|
642
|
+
"""
|
|
643
|
+
Return a DataFrame with summary output data for each mesh cell in the model.
|
|
644
|
+
|
|
645
|
+
Parameters
|
|
646
|
+
----------
|
|
647
|
+
round_to : str, optional
|
|
648
|
+
The time unit to round the datetimes to. Default: "0.1 s" (seconds).
|
|
649
|
+
See Pandas documentation for valid time units:
|
|
650
|
+
https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html
|
|
651
|
+
|
|
652
|
+
Returns
|
|
653
|
+
-------
|
|
654
|
+
DataFrame
|
|
655
|
+
A DataFrame with columns 'mesh_name', 'cell_id', and columns for each
|
|
656
|
+
summary output variable.
|
|
657
|
+
"""
|
|
658
|
+
return self._mesh_summary_outputs_df("cells", round_to=round_to)
|
|
659
|
+
|
|
660
|
+
def mesh_faces_summary_output(self, round_to: str = "0.1 s") -> DataFrame:
|
|
661
|
+
"""
|
|
662
|
+
Return a DataFrame with summary output data for each mesh face in the model.
|
|
663
|
+
|
|
664
|
+
Parameters
|
|
665
|
+
----------
|
|
666
|
+
round_to : str, optional
|
|
667
|
+
The time unit to round the datetimes to. Default: "0.1 s" (seconds).
|
|
668
|
+
See Pandas documentation for valid time units:
|
|
669
|
+
https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html
|
|
670
|
+
|
|
671
|
+
Returns
|
|
672
|
+
-------
|
|
673
|
+
DataFrame
|
|
674
|
+
A DataFrame with columns 'mesh_name', 'face_id', and columns for each
|
|
675
|
+
summary output variable.
|
|
676
|
+
"""
|
|
677
|
+
return self._mesh_summary_outputs_df("faces", round_to=round_to)
|
|
678
|
+
|
|
607
679
|
def _summary_output_vars(
|
|
608
680
|
self, cells_or_faces: Optional[str] = None
|
|
609
681
|
) -> List[SummaryOutputVar]:
|
|
@@ -812,7 +884,7 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
812
884
|
mesh_name: str,
|
|
813
885
|
var: TimeSeriesOutputVar,
|
|
814
886
|
) -> Tuple[np.ndarray, str]:
|
|
815
|
-
path =
|
|
887
|
+
path = self._mesh_timeseries_output_path(mesh_name, var.value)
|
|
816
888
|
group = self.get(path)
|
|
817
889
|
try:
|
|
818
890
|
import dask.array as da
|
|
@@ -830,6 +902,7 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
830
902
|
self,
|
|
831
903
|
mesh_name: str,
|
|
832
904
|
var: Union[str, TimeSeriesOutputVar],
|
|
905
|
+
truncate: bool = True,
|
|
833
906
|
) -> xr.DataArray:
|
|
834
907
|
"""Return the time series output data for a given variable.
|
|
835
908
|
|
|
@@ -839,6 +912,8 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
839
912
|
The name of the 2D flow area mesh.
|
|
840
913
|
var : TimeSeriesOutputVar
|
|
841
914
|
The time series output variable to retrieve.
|
|
915
|
+
truncate : bool, optional
|
|
916
|
+
If True, truncate the number of cells to the listed cell count.
|
|
842
917
|
|
|
843
918
|
Returns
|
|
844
919
|
-------
|
|
@@ -856,7 +931,10 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
856
931
|
values, units = self._mesh_timeseries_output_values_units(mesh_name, var)
|
|
857
932
|
if var in TIME_SERIES_OUTPUT_VARS_CELLS:
|
|
858
933
|
cell_count = mesh_names_counts[mesh_name]
|
|
859
|
-
|
|
934
|
+
if truncate:
|
|
935
|
+
values = values[:, :cell_count]
|
|
936
|
+
else:
|
|
937
|
+
values = values[:, :]
|
|
860
938
|
id_coord = "cell_id"
|
|
861
939
|
elif var in TIME_SERIES_OUTPUT_VARS_FACES:
|
|
862
940
|
id_coord = "face_id"
|
|
@@ -874,24 +952,28 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
874
952
|
"mesh_name": mesh_name,
|
|
875
953
|
"variable": var.value,
|
|
876
954
|
"units": units,
|
|
955
|
+
"hdf_path": self._mesh_timeseries_output_path(mesh_name, var.value),
|
|
877
956
|
},
|
|
878
957
|
)
|
|
879
958
|
return da
|
|
880
959
|
|
|
960
|
+
def _mesh_timeseries_output_path(self, mesh_name: str, var_name: str) -> str:
|
|
961
|
+
return f"{self.UNSTEADY_TIME_SERIES_PATH}/2D Flow Areas/{mesh_name}/{var_name}"
|
|
962
|
+
|
|
881
963
|
def _mesh_timeseries_outputs(
|
|
882
|
-
self, mesh_name: str, vars: List[TimeSeriesOutputVar]
|
|
964
|
+
self, mesh_name: str, vars: List[TimeSeriesOutputVar], truncate: bool = True
|
|
883
965
|
) -> xr.Dataset:
|
|
884
966
|
datasets = {}
|
|
885
967
|
for var in vars:
|
|
886
968
|
var_path = f"{self.UNSTEADY_TIME_SERIES_PATH}/2D Flow Areas/{mesh_name}/{var.value}"
|
|
887
969
|
if self.get(var_path) is None:
|
|
888
970
|
continue
|
|
889
|
-
da = self.mesh_timeseries_output(mesh_name, var)
|
|
971
|
+
da = self.mesh_timeseries_output(mesh_name, var, truncate=truncate)
|
|
890
972
|
datasets[var.value] = da
|
|
891
973
|
ds = xr.Dataset(datasets, attrs={"mesh_name": mesh_name})
|
|
892
974
|
return ds
|
|
893
975
|
|
|
894
|
-
def
|
|
976
|
+
def mesh_cells_timeseries_output(self, mesh_name: str) -> xr.Dataset:
|
|
895
977
|
"""Return the time series output data for cells in a 2D flow area mesh.
|
|
896
978
|
|
|
897
979
|
Parameters
|
|
@@ -907,7 +989,25 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
907
989
|
ds = self._mesh_timeseries_outputs(mesh_name, TIME_SERIES_OUTPUT_VARS_CELLS)
|
|
908
990
|
return ds
|
|
909
991
|
|
|
910
|
-
|
|
992
|
+
@deprecated
|
|
993
|
+
def mesh_timeseries_output_cells(self, mesh_name: str) -> xr.Dataset:
|
|
994
|
+
"""Return the time series output data for cells in a 2D flow area mesh.
|
|
995
|
+
|
|
996
|
+
Deprecated: use mesh_cells_timeseries_output instead.
|
|
997
|
+
|
|
998
|
+
Parameters
|
|
999
|
+
----------
|
|
1000
|
+
mesh_name : str
|
|
1001
|
+
The name of the 2D flow area mesh.
|
|
1002
|
+
|
|
1003
|
+
Returns
|
|
1004
|
+
-------
|
|
1005
|
+
xr.Dataset
|
|
1006
|
+
An xarray Dataset with DataArrays for each time series output variable.
|
|
1007
|
+
"""
|
|
1008
|
+
return self.mesh_cells_timeseries_output(mesh_name)
|
|
1009
|
+
|
|
1010
|
+
def mesh_faces_timeseries_output(self, mesh_name: str) -> xr.Dataset:
|
|
911
1011
|
"""Return the time series output data for faces in a 2D flow area mesh.
|
|
912
1012
|
|
|
913
1013
|
Parameters
|
|
@@ -923,6 +1023,24 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
923
1023
|
ds = self._mesh_timeseries_outputs(mesh_name, TIME_SERIES_OUTPUT_VARS_FACES)
|
|
924
1024
|
return ds
|
|
925
1025
|
|
|
1026
|
+
@deprecated
|
|
1027
|
+
def mesh_timeseries_output_faces(self, mesh_name: str) -> xr.Dataset:
|
|
1028
|
+
"""Return the time series output data for faces in a 2D flow area mesh.
|
|
1029
|
+
|
|
1030
|
+
Deprecated: use mesh_faces_timeseries_output instead.
|
|
1031
|
+
|
|
1032
|
+
Parameters
|
|
1033
|
+
----------
|
|
1034
|
+
mesh_name : str
|
|
1035
|
+
The name of the 2D flow area mesh.
|
|
1036
|
+
|
|
1037
|
+
Returns
|
|
1038
|
+
-------
|
|
1039
|
+
xr.Dataset
|
|
1040
|
+
An xarray Dataset with DataArrays for each time series output variable.
|
|
1041
|
+
"""
|
|
1042
|
+
return self.mesh_faces_timeseries_output(mesh_name)
|
|
1043
|
+
|
|
926
1044
|
def reference_timeseries_output(self, reftype: str = "lines") -> xr.Dataset:
|
|
927
1045
|
"""Return timeseries output data for reference lines or points from a HEC-RAS HDF plan file.
|
|
928
1046
|
|
|
@@ -984,7 +1102,7 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
984
1102
|
f"{abbrev}_name": (f"{abbrev}_id", names),
|
|
985
1103
|
"mesh_name": (f"{abbrev}_id", mesh_areas),
|
|
986
1104
|
},
|
|
987
|
-
attrs={"
|
|
1105
|
+
attrs={"units": units, "hdf_path": f"{output_path}/{var}"},
|
|
988
1106
|
)
|
|
989
1107
|
das[var] = da
|
|
990
1108
|
return xr.Dataset(das)
|
|
@@ -1317,3 +1435,113 @@ class RasPlanHdf(RasGeomHdf):
|
|
|
1317
1435
|
A DataFrame containing the velocity inside the cross sections
|
|
1318
1436
|
"""
|
|
1319
1437
|
return self.steady_profile_xs_output(XsSteadyOutputVar.VELOCITY_TOTAL)
|
|
1438
|
+
|
|
1439
|
+
def _zmeta(self, ds: xr.Dataset) -> Dict:
|
|
1440
|
+
"""Given a xarray Dataset, return kerchunk-style zarr reference metadata."""
|
|
1441
|
+
from kerchunk.hdf import SingleHdf5ToZarr
|
|
1442
|
+
import zarr
|
|
1443
|
+
import base64
|
|
1444
|
+
|
|
1445
|
+
encoding = {}
|
|
1446
|
+
chunk_meta = {}
|
|
1447
|
+
|
|
1448
|
+
# Loop through each variable / DataArray in the Dataset
|
|
1449
|
+
for var, da in ds.data_vars.items():
|
|
1450
|
+
# The "hdf_path" attribute is the path within the HDF5 file
|
|
1451
|
+
# that the DataArray was read from. This is attribute is inserted
|
|
1452
|
+
# by rashdf (see "mesh_timeseries_output" method).
|
|
1453
|
+
hdf_ds_path = da.attrs["hdf_path"]
|
|
1454
|
+
hdf_ds = self.get(hdf_ds_path)
|
|
1455
|
+
if hdf_ds is None:
|
|
1456
|
+
# If we don't know where in the HDF5 the data came from, we
|
|
1457
|
+
# have to skip it, because we won't be able to generate the
|
|
1458
|
+
# correct metadata for it.
|
|
1459
|
+
continue
|
|
1460
|
+
# Get the filters and storage info for the HDF5 dataset.
|
|
1461
|
+
# Calling private methods from Kerchunk here because
|
|
1462
|
+
# there's not a nice public API for this part. This is hacky
|
|
1463
|
+
# and a bit risky because these private methods are more likely
|
|
1464
|
+
# to change, but short of reimplementing these functions ourselves
|
|
1465
|
+
# it's the best way to get the metadata we need.
|
|
1466
|
+
# TODO: raise an issue in Kerchunk to expose this functionality?
|
|
1467
|
+
filters = SingleHdf5ToZarr._decode_filters(None, hdf_ds)
|
|
1468
|
+
encoding[var] = {"compressor": None, "filters": filters}
|
|
1469
|
+
storage_info = SingleHdf5ToZarr._storage_info(None, hdf_ds)
|
|
1470
|
+
# Generate chunk metadata for the DataArray
|
|
1471
|
+
for key, value in storage_info.items():
|
|
1472
|
+
chunk_number = ".".join([str(k) for k in key])
|
|
1473
|
+
chunk_key = f"{var}/{chunk_number}"
|
|
1474
|
+
chunk_meta[chunk_key] = [str(self._loc), value["offset"], value["size"]]
|
|
1475
|
+
# "Write" the Dataset to a temporary in-memory zarr store (which
|
|
1476
|
+
# is the same a Python dictionary)
|
|
1477
|
+
zarr_tmp = zarr.MemoryStore()
|
|
1478
|
+
# Use compute=False here because we don't _actually_ want to write
|
|
1479
|
+
# the data to the zarr store, we just want to generate the metadata.
|
|
1480
|
+
ds.to_zarr(zarr_tmp, mode="w", compute=False, encoding=encoding)
|
|
1481
|
+
zarr_meta = {"version": 1, "refs": {}}
|
|
1482
|
+
# Loop through the in-memory Zarr store, decode the data to strings,
|
|
1483
|
+
# and add it to the final metadata dictionary.
|
|
1484
|
+
for key, value in zarr_tmp.items():
|
|
1485
|
+
try:
|
|
1486
|
+
value_str = value.decode("utf-8")
|
|
1487
|
+
except UnicodeDecodeError:
|
|
1488
|
+
value_str = "base64:" + base64.b64encode(value).decode("utf-8")
|
|
1489
|
+
zarr_meta["refs"][key] = value_str
|
|
1490
|
+
zarr_meta["refs"].update(chunk_meta)
|
|
1491
|
+
return zarr_meta
|
|
1492
|
+
|
|
1493
|
+
def zmeta_mesh_cells_timeseries_output(self, mesh_name: str) -> Dict:
|
|
1494
|
+
"""Return kerchunk-style zarr reference metadata.
|
|
1495
|
+
|
|
1496
|
+
Requires the 'zarr' and 'kerchunk' packages.
|
|
1497
|
+
|
|
1498
|
+
Returns
|
|
1499
|
+
-------
|
|
1500
|
+
dict
|
|
1501
|
+
Dictionary of kerchunk-style zarr reference metadata.
|
|
1502
|
+
"""
|
|
1503
|
+
ds = self._mesh_timeseries_outputs(
|
|
1504
|
+
mesh_name, TIME_SERIES_OUTPUT_VARS_CELLS, truncate=False
|
|
1505
|
+
)
|
|
1506
|
+
return self._zmeta(ds)
|
|
1507
|
+
|
|
1508
|
+
def zmeta_mesh_faces_timeseries_output(self, mesh_name: str) -> Dict:
|
|
1509
|
+
"""Return kerchunk-style zarr reference metadata.
|
|
1510
|
+
|
|
1511
|
+
Requires the 'zarr' and 'kerchunk' packages.
|
|
1512
|
+
|
|
1513
|
+
Returns
|
|
1514
|
+
-------
|
|
1515
|
+
dict
|
|
1516
|
+
Dictionary of kerchunk-style zarr reference metadata.
|
|
1517
|
+
"""
|
|
1518
|
+
ds = self._mesh_timeseries_outputs(
|
|
1519
|
+
mesh_name, TIME_SERIES_OUTPUT_VARS_FACES, truncate=False
|
|
1520
|
+
)
|
|
1521
|
+
return self._zmeta(ds)
|
|
1522
|
+
|
|
1523
|
+
def zmeta_reference_lines_timeseries_output(self) -> Dict:
|
|
1524
|
+
"""Return kerchunk-style zarr reference metadata.
|
|
1525
|
+
|
|
1526
|
+
Requires the 'zarr' and 'kerchunk' packages.
|
|
1527
|
+
|
|
1528
|
+
Returns
|
|
1529
|
+
-------
|
|
1530
|
+
dict
|
|
1531
|
+
Dictionary of kerchunk-style zarr reference metadata.
|
|
1532
|
+
"""
|
|
1533
|
+
ds = self.reference_lines_timeseries_output()
|
|
1534
|
+
return self._zmeta(ds)
|
|
1535
|
+
|
|
1536
|
+
def zmeta_reference_points_timeseries_output(self) -> Dict:
|
|
1537
|
+
"""Return kerchunk-style zarr reference metadata.
|
|
1538
|
+
|
|
1539
|
+
Requires the 'zarr' and 'kerchunk' packages.
|
|
1540
|
+
|
|
1541
|
+
Returns
|
|
1542
|
+
-------
|
|
1543
|
+
dict
|
|
1544
|
+
Dictionary of kerchunk-style zarr reference metadata.
|
|
1545
|
+
"""
|
|
1546
|
+
ds = self.reference_points_timeseries_output()
|
|
1547
|
+
return self._zmeta(ds)
|
|
@@ -6,8 +6,8 @@ import pandas as pd
|
|
|
6
6
|
|
|
7
7
|
from datetime import datetime, timedelta
|
|
8
8
|
import re
|
|
9
|
-
from typing import Any, List, Tuple, Union, Optional
|
|
10
|
-
|
|
9
|
+
from typing import Any, Callable, List, Tuple, Union, Optional
|
|
10
|
+
import warnings
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def parse_ras_datetime_ms(datetime_str: str) -> datetime:
|
|
@@ -308,3 +308,33 @@ def ras_timesteps_to_datetimes(
|
|
|
308
308
|
start_time + pd.Timedelta(timestep, unit=time_unit).round(round_to)
|
|
309
309
|
for timestep in timesteps.astype(np.float64)
|
|
310
310
|
]
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def deprecated(func) -> Callable:
|
|
314
|
+
"""
|
|
315
|
+
Deprecate a function.
|
|
316
|
+
|
|
317
|
+
This is a decorator which can be used to mark functions as deprecated.
|
|
318
|
+
It will result in a warning being emitted when the function is used.
|
|
319
|
+
|
|
320
|
+
Parameters
|
|
321
|
+
----------
|
|
322
|
+
func: The function to be deprecated.
|
|
323
|
+
|
|
324
|
+
Returns
|
|
325
|
+
-------
|
|
326
|
+
The decorated function.
|
|
327
|
+
"""
|
|
328
|
+
|
|
329
|
+
def new_func(*args, **kwargs):
|
|
330
|
+
warnings.warn(
|
|
331
|
+
f"{func.__name__} is deprecated and will be removed in a future version.",
|
|
332
|
+
category=DeprecationWarning,
|
|
333
|
+
stacklevel=2,
|
|
334
|
+
)
|
|
335
|
+
return func(*args, **kwargs)
|
|
336
|
+
|
|
337
|
+
new_func.__name__ = func.__name__
|
|
338
|
+
new_func.__doc__ = func.__doc__
|
|
339
|
+
new_func.__dict__.update(func.__dict__)
|
|
340
|
+
return new_func
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: rashdf
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Read data from HEC-RAS HDF files.
|
|
5
5
|
Project-URL: repository, https://github.com/fema-ffrd/rashdf
|
|
6
6
|
Classifier: Development Status :: 4 - Beta
|
|
@@ -23,6 +23,11 @@ Requires-Dist: ruff; extra == "dev"
|
|
|
23
23
|
Requires-Dist: pytest; extra == "dev"
|
|
24
24
|
Requires-Dist: pytest-cov; extra == "dev"
|
|
25
25
|
Requires-Dist: fiona; extra == "dev"
|
|
26
|
+
Requires-Dist: kerchunk; extra == "dev"
|
|
27
|
+
Requires-Dist: zarr; extra == "dev"
|
|
28
|
+
Requires-Dist: dask; extra == "dev"
|
|
29
|
+
Requires-Dist: fsspec; extra == "dev"
|
|
30
|
+
Requires-Dist: s3fs; extra == "dev"
|
|
26
31
|
Provides-Extra: docs
|
|
27
32
|
Requires-Dist: sphinx; extra == "docs"
|
|
28
33
|
Requires-Dist: numpydoc; extra == "docs"
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from src.rashdf.base import RasHdf
|
|
2
|
+
from unittest.mock import patch
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_open():
|
|
6
|
+
rasfile = "Muncie.g05.hdf"
|
|
7
|
+
rasfile_path = f"./tests/data/ras/{rasfile}"
|
|
8
|
+
hdf = RasHdf(rasfile_path)
|
|
9
|
+
assert hdf._loc == rasfile_path
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_open_uri():
|
|
13
|
+
rasfile = "Muncie.g05.hdf"
|
|
14
|
+
rasfile_path = f"./tests/data/ras/{rasfile}"
|
|
15
|
+
url = f"s3://mybucket/{rasfile}"
|
|
16
|
+
|
|
17
|
+
# Mock the specific functions used by s3fs
|
|
18
|
+
with patch("s3fs.core.S3FileSystem.open", return_value=open(rasfile_path, "rb")):
|
|
19
|
+
hdf = RasHdf.open_uri(url)
|
|
20
|
+
assert hdf._loc == url
|
|
@@ -5,12 +5,15 @@ from src.rashdf.plan import (
|
|
|
5
5
|
TimeSeriesOutputVar,
|
|
6
6
|
)
|
|
7
7
|
|
|
8
|
+
import filecmp
|
|
9
|
+
import json
|
|
8
10
|
from pathlib import Path
|
|
9
11
|
|
|
10
12
|
import numpy as np
|
|
11
13
|
import pandas as pd
|
|
12
14
|
from pandas.testing import assert_frame_equal
|
|
13
15
|
import pytest
|
|
16
|
+
import xarray as xr
|
|
14
17
|
|
|
15
18
|
from . import (
|
|
16
19
|
_create_hdf_with_group_attrs,
|
|
@@ -193,9 +196,9 @@ def test_mesh_timeseries_output():
|
|
|
193
196
|
plan_hdf.mesh_timeseries_output("BaldEagleCr", "Fake Variable")
|
|
194
197
|
|
|
195
198
|
|
|
196
|
-
def
|
|
199
|
+
def test_mesh_cells_timeseries_output():
|
|
197
200
|
with RasPlanHdf(BALD_EAGLE_P18_TIMESERIES) as plan_hdf:
|
|
198
|
-
ds = plan_hdf.
|
|
201
|
+
ds = plan_hdf.mesh_cells_timeseries_output("BaldEagleCr")
|
|
199
202
|
assert "time" in ds.coords
|
|
200
203
|
assert "cell_id" in ds.coords
|
|
201
204
|
assert "Water Surface" in ds.variables
|
|
@@ -212,7 +215,7 @@ def test_mesh_timeseries_output_cells():
|
|
|
212
215
|
)
|
|
213
216
|
assert_frame_equal(df, valid_df)
|
|
214
217
|
|
|
215
|
-
ds = plan_hdf.
|
|
218
|
+
ds = plan_hdf.mesh_cells_timeseries_output("Upper 2D Area")
|
|
216
219
|
assert "time" in ds.coords
|
|
217
220
|
assert "cell_id" in ds.coords
|
|
218
221
|
assert "Water Surface" in ds.variables
|
|
@@ -230,9 +233,15 @@ def test_mesh_timeseries_output_cells():
|
|
|
230
233
|
assert_frame_equal(df, valid_df)
|
|
231
234
|
|
|
232
235
|
|
|
233
|
-
def
|
|
236
|
+
def test_mesh_timeseries_output_cells():
|
|
237
|
+
with pytest.warns(DeprecationWarning):
|
|
238
|
+
with RasPlanHdf(BALD_EAGLE_P18_TIMESERIES) as plan_hdf:
|
|
239
|
+
plan_hdf.mesh_timeseries_output_cells("BaldEagleCr")
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def test_mesh_faces_timeseries_output():
|
|
234
243
|
with RasPlanHdf(BALD_EAGLE_P18_TIMESERIES) as plan_hdf:
|
|
235
|
-
ds = plan_hdf.
|
|
244
|
+
ds = plan_hdf.mesh_faces_timeseries_output("BaldEagleCr")
|
|
236
245
|
assert "time" in ds.coords
|
|
237
246
|
assert "face_id" in ds.coords
|
|
238
247
|
assert "Face Velocity" in ds.variables
|
|
@@ -249,7 +258,7 @@ def test_mesh_timeseries_output_faces():
|
|
|
249
258
|
)
|
|
250
259
|
assert_frame_equal(df, valid_df)
|
|
251
260
|
|
|
252
|
-
ds = plan_hdf.
|
|
261
|
+
ds = plan_hdf.mesh_faces_timeseries_output("Upper 2D Area")
|
|
253
262
|
assert "time" in ds.coords
|
|
254
263
|
assert "face_id" in ds.coords
|
|
255
264
|
assert "Face Velocity" in ds.variables
|
|
@@ -267,6 +276,12 @@ def test_mesh_timeseries_output_faces():
|
|
|
267
276
|
assert_frame_equal(df, valid_df)
|
|
268
277
|
|
|
269
278
|
|
|
279
|
+
def test_mesh_timeseries_output_faces():
|
|
280
|
+
with pytest.warns(DeprecationWarning):
|
|
281
|
+
with RasPlanHdf(BALD_EAGLE_P18_TIMESERIES) as plan_hdf:
|
|
282
|
+
plan_hdf.mesh_timeseries_output_faces("BaldEagleCr")
|
|
283
|
+
|
|
284
|
+
|
|
270
285
|
def test_reference_lines(tmp_path: Path):
|
|
271
286
|
plan_hdf = RasPlanHdf(BALD_EAGLE_P18_REF)
|
|
272
287
|
gdf = plan_hdf.reference_lines(datetime_to_str=True)
|
|
@@ -291,10 +306,10 @@ def test_reference_lines_timeseries(tmp_path: Path):
|
|
|
291
306
|
|
|
292
307
|
ws = ds["Water Surface"]
|
|
293
308
|
assert ws.shape == (37, 4)
|
|
294
|
-
assert ws.attrs["
|
|
309
|
+
assert ws.attrs["units"] == "ft"
|
|
295
310
|
q = ds["Flow"]
|
|
296
311
|
assert q.shape == (37, 4)
|
|
297
|
-
assert q.attrs["
|
|
312
|
+
assert q.attrs["units"] == "cfs"
|
|
298
313
|
|
|
299
314
|
df = ds.sel(refln_id=2).to_dataframe()
|
|
300
315
|
valid_df = pd.read_csv(
|
|
@@ -330,9 +345,9 @@ def test_reference_points_timeseries():
|
|
|
330
345
|
|
|
331
346
|
ws = ds["Water Surface"]
|
|
332
347
|
assert ws.shape == (37, 3)
|
|
333
|
-
assert ws.attrs["
|
|
348
|
+
assert ws.attrs["units"] == "ft"
|
|
334
349
|
v = ds["Velocity"]
|
|
335
|
-
assert v.attrs["
|
|
350
|
+
assert v.attrs["units"] == "ft/s"
|
|
336
351
|
assert v.shape == (37, 3)
|
|
337
352
|
|
|
338
353
|
df = ds.sel(refpt_id=1).to_dataframe()
|
|
@@ -431,3 +446,174 @@ def test_cross_sections_energy_grade():
|
|
|
431
446
|
assert _gdf_matches_json_alt(
|
|
432
447
|
phdf.cross_sections_energy_grade(), xs_energy_grade_json
|
|
433
448
|
)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _compare_json(json_file1, json_file2) -> bool:
|
|
452
|
+
with open(json_file1) as j1:
|
|
453
|
+
with open(json_file2) as j2:
|
|
454
|
+
return json.load(j1) == json.load(j2)
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def test_zmeta_mesh_cells_timeseries_output(tmp_path):
|
|
458
|
+
with RasPlanHdf(BALD_EAGLE_P18_TIMESERIES) as phdf:
|
|
459
|
+
# Generate Zarr metadata
|
|
460
|
+
zmeta = phdf.zmeta_mesh_cells_timeseries_output("BaldEagleCr")
|
|
461
|
+
|
|
462
|
+
# Write the Zarr metadata to JSON
|
|
463
|
+
zmeta_test_path = tmp_path / "bald-eagle-mesh-cells-zmeta.test.json"
|
|
464
|
+
with open(zmeta_test_path, "w") as f:
|
|
465
|
+
json.dump(zmeta, f, indent=4)
|
|
466
|
+
|
|
467
|
+
# Compare to a validated JSON file
|
|
468
|
+
zmeta_valid_path = TEST_JSON / "bald-eagle-mesh-cells-zmeta.json"
|
|
469
|
+
assert _compare_json(zmeta_test_path, zmeta_valid_path)
|
|
470
|
+
|
|
471
|
+
# Verify that the Zarr metadata can be used to open a dataset
|
|
472
|
+
ds = xr.open_dataset(
|
|
473
|
+
"reference://",
|
|
474
|
+
engine="zarr",
|
|
475
|
+
backend_kwargs={
|
|
476
|
+
"consolidated": False,
|
|
477
|
+
"storage_options": {"fo": str(zmeta_test_path)},
|
|
478
|
+
},
|
|
479
|
+
)
|
|
480
|
+
assert ds["Water Surface"].shape == (37, 3947)
|
|
481
|
+
assert len(ds.coords["time"]) == 37
|
|
482
|
+
assert len(ds.coords["cell_id"]) == 3947
|
|
483
|
+
assert ds.attrs["mesh_name"] == "BaldEagleCr"
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def test_zmeta_mesh_faces_timeseries_output(tmp_path):
|
|
487
|
+
with RasPlanHdf(BALD_EAGLE_P18_TIMESERIES) as phdf:
|
|
488
|
+
# Generate Zarr metadata
|
|
489
|
+
zmeta = phdf.zmeta_mesh_faces_timeseries_output("BaldEagleCr")
|
|
490
|
+
|
|
491
|
+
# Write the Zarr metadata to JSON
|
|
492
|
+
zmeta_test_path = tmp_path / "bald-eagle-mesh-faces-zmeta.test.json"
|
|
493
|
+
with open(zmeta_test_path, "w") as f:
|
|
494
|
+
json.dump(zmeta, f, indent=4)
|
|
495
|
+
|
|
496
|
+
# Compare to a validated JSON file
|
|
497
|
+
zmeta_valid_path = TEST_JSON / "bald-eagle-mesh-faces-zmeta.json"
|
|
498
|
+
assert _compare_json(zmeta_test_path, zmeta_valid_path)
|
|
499
|
+
|
|
500
|
+
# Verify that the Zarr metadata can be used to open a dataset
|
|
501
|
+
ds = xr.open_dataset(
|
|
502
|
+
"reference://",
|
|
503
|
+
engine="zarr",
|
|
504
|
+
backend_kwargs={
|
|
505
|
+
"consolidated": False,
|
|
506
|
+
"storage_options": {"fo": str(zmeta_test_path)},
|
|
507
|
+
},
|
|
508
|
+
)
|
|
509
|
+
assert ds["Face Velocity"].shape == (37, 7295)
|
|
510
|
+
assert len(ds.coords["time"]) == 37
|
|
511
|
+
assert len(ds.coords["face_id"]) == 7295
|
|
512
|
+
assert ds.attrs["mesh_name"] == "BaldEagleCr"
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def test_zmeta_reference_lines_timeseries_output(tmp_path):
|
|
516
|
+
with RasPlanHdf(BALD_EAGLE_P18_REF) as phdf:
|
|
517
|
+
# Generate Zarr metadata
|
|
518
|
+
zmeta = phdf.zmeta_reference_lines_timeseries_output()
|
|
519
|
+
|
|
520
|
+
# Write the Zarr metadata to JSON
|
|
521
|
+
zmeta_test_path = tmp_path / "bald-eagle-reflines-zmeta.test.json"
|
|
522
|
+
with open(zmeta_test_path, "w") as f:
|
|
523
|
+
json.dump(zmeta, f, indent=4)
|
|
524
|
+
|
|
525
|
+
# Compare to a validated JSON file
|
|
526
|
+
zmeta_valid_path = TEST_JSON / "bald-eagle-reflines-zmeta.json"
|
|
527
|
+
assert _compare_json(zmeta_test_path, zmeta_valid_path)
|
|
528
|
+
|
|
529
|
+
# Verify that the Zarr metadata can be used to open a dataset
|
|
530
|
+
ds = xr.open_dataset(
|
|
531
|
+
"reference://",
|
|
532
|
+
engine="zarr",
|
|
533
|
+
backend_kwargs={
|
|
534
|
+
"consolidated": False,
|
|
535
|
+
"storage_options": {"fo": str(zmeta_test_path)},
|
|
536
|
+
},
|
|
537
|
+
)
|
|
538
|
+
assert ds["Flow"].shape == (37, 4)
|
|
539
|
+
assert len(ds.coords["time"]) == 37
|
|
540
|
+
assert len(ds.coords["refln_id"]) == 4
|
|
541
|
+
assert ds.attrs == {}
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def test_zmeta_reference_points_timeseries_output(tmp_path):
|
|
545
|
+
with RasPlanHdf(BALD_EAGLE_P18_REF) as phdf:
|
|
546
|
+
# Generate Zarr metadata
|
|
547
|
+
zmeta = phdf.zmeta_reference_points_timeseries_output()
|
|
548
|
+
|
|
549
|
+
# Write the Zarr metadata to JSON
|
|
550
|
+
zmeta_test_path = tmp_path / "bald-eagle-refpoints-zmeta.test.json"
|
|
551
|
+
with open(zmeta_test_path, "w") as f:
|
|
552
|
+
json.dump(zmeta, f, indent=4)
|
|
553
|
+
|
|
554
|
+
# Compare to a validated JSON file
|
|
555
|
+
zmeta_valid_path = TEST_JSON / "bald-eagle-refpoints-zmeta.json"
|
|
556
|
+
assert _compare_json(zmeta_test_path, zmeta_valid_path)
|
|
557
|
+
|
|
558
|
+
# Verify that the Zarr metadata can be used to open a dataset
|
|
559
|
+
ds = xr.open_dataset(
|
|
560
|
+
"reference://",
|
|
561
|
+
engine="zarr",
|
|
562
|
+
backend_kwargs={
|
|
563
|
+
"consolidated": False,
|
|
564
|
+
"storage_options": {"fo": str(zmeta_test_path)},
|
|
565
|
+
},
|
|
566
|
+
)
|
|
567
|
+
assert ds["Water Surface"].shape == (37, 3)
|
|
568
|
+
assert ds["Velocity"].shape == (37, 3)
|
|
569
|
+
assert len(ds.coords["time"]) == 37
|
|
570
|
+
assert len(ds.coords["refpt_id"]) == 3
|
|
571
|
+
assert ds.attrs == {}
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def test_mesh_cells_summary_output(tmp_path):
|
|
575
|
+
with RasPlanHdf(BALD_EAGLE_P18) as phdf:
|
|
576
|
+
df = phdf.mesh_cells_summary_output()
|
|
577
|
+
test_csv = tmp_path / "BaldEagleDamBrk.summary-cells.test.csv"
|
|
578
|
+
df.to_csv(test_csv)
|
|
579
|
+
filecmp.cmp(
|
|
580
|
+
test_csv,
|
|
581
|
+
TEST_CSV / "BaldEagleDamBrk.summary-cells.csv",
|
|
582
|
+
shallow=False,
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def test_mesh_faces_summary_output(tmp_path):
|
|
587
|
+
with RasPlanHdf(BALD_EAGLE_P18) as phdf:
|
|
588
|
+
df = phdf.mesh_faces_summary_output()
|
|
589
|
+
test_csv = tmp_path / "BaldEagleDamBrk.summary-faces.test.csv"
|
|
590
|
+
df.to_csv(test_csv)
|
|
591
|
+
filecmp.cmp(
|
|
592
|
+
test_csv,
|
|
593
|
+
TEST_CSV / "BaldEagleDamBrk.summary-faces.csv",
|
|
594
|
+
shallow=False,
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def test__mesh_summary_outputs_df(tmp_path):
|
|
599
|
+
with RasPlanHdf(BALD_EAGLE_P18) as phdf:
|
|
600
|
+
with pytest.raises(ValueError):
|
|
601
|
+
phdf._mesh_summary_outputs_df("neither")
|
|
602
|
+
|
|
603
|
+
with pytest.raises(ValueError):
|
|
604
|
+
phdf._mesh_summary_outputs_df(cells_or_faces="cells", output_vars="wrong")
|
|
605
|
+
|
|
606
|
+
df = phdf._mesh_summary_outputs_df(
|
|
607
|
+
cells_or_faces="cells",
|
|
608
|
+
output_vars=[
|
|
609
|
+
SummaryOutputVar.MAXIMUM_WATER_SURFACE,
|
|
610
|
+
SummaryOutputVar.MINIMUM_WATER_SURFACE,
|
|
611
|
+
],
|
|
612
|
+
)
|
|
613
|
+
test_csv = tmp_path / "BaldEagleDamBrk.summary-cells-selectvars.test.csv"
|
|
614
|
+
df.to_csv(test_csv)
|
|
615
|
+
filecmp.cmp(
|
|
616
|
+
test_csv,
|
|
617
|
+
TEST_CSV / "BaldEagleDamBrk.summary-cells-selectvars.csv",
|
|
618
|
+
shallow=False,
|
|
619
|
+
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|