toolbox-utils 5.2.4__tar.gz → 5.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/.pre-commit-config.yaml +5 -5
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/CHANGELOG.md +20 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/PKG-INFO +1 -1
- toolbox_utils-5.4.0/VERSION +1 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/pyproject.toml +1 -1
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils/readers/hbn.py +32 -172
- toolbox_utils-5.4.0/src/toolbox_utils/readers/utils.py +179 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils/tsutils.py +31 -18
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils/utils.py +52 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils.egg-info/PKG-INFO +1 -1
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils.egg-info/SOURCES.txt +1 -0
- toolbox_utils-5.2.4/VERSION +0 -1
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/.deepsource.toml +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/.github/dependabot.yml +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/.github/workflows/clean-workflow-runs.yml +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/.github/workflows/pypi-package.yml +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/.github/workflows/tests.yml +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/.gitignore +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/.sourcery.yaml +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/AUTHORS.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/BADGES.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/CONTRIBUTING.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/LICENSE.txt +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/README.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/docs/Makefile +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/docs/authors.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/docs/conf.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/docs/contributing.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/docs/index.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/docs/license.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/docs/make.bat +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/docs/readme.rst +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/setup.cfg +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils/__init__.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils/readers/__init__.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils/readers/plotgen.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils/readers/wdm.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils.egg-info/dependency_links.txt +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils.egg-info/requires.txt +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/src/toolbox_utils.egg-info/top_level.txt +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data.wdm +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_bi_daily.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_bivl.hbn +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_end.bivl.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_end.daily.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_end.monthly.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_end.yearly.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_flow_stage.xlsx +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_period.bivl.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_period.daily.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_period.monthly.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_period.yearly.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_plotgen.plt +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_simple.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_start.bivl.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_start.daily.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_start.monthly.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_start.yearly.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_wdm_1.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_wdm_2.csv +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/data_yearly.hbn +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_asbestfreq.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_date_slice.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_dateparse.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_extract.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_hbn.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_make_list.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_period_timestamp.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_plotgen.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_range_to_numlist.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_read.py +0 -0
- {toolbox_utils-5.2.4 → toolbox_utils-5.4.0}/tests/test_wdm.py +0 -0
|
@@ -29,7 +29,7 @@ repos:
|
|
|
29
29
|
|
|
30
30
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
31
31
|
# Ruff version.
|
|
32
|
-
rev: v0.15.
|
|
32
|
+
rev: v0.15.16
|
|
33
33
|
hooks:
|
|
34
34
|
# Run the linter.
|
|
35
35
|
- id: ruff
|
|
@@ -40,7 +40,7 @@ repos:
|
|
|
40
40
|
types_or: [python, pyi, jupyter]
|
|
41
41
|
|
|
42
42
|
- repo: https://github.com/pycqa/isort
|
|
43
|
-
rev:
|
|
43
|
+
rev: 9.0.0a3
|
|
44
44
|
hooks:
|
|
45
45
|
- id: isort
|
|
46
46
|
name: isort (python)
|
|
@@ -52,7 +52,7 @@ repos:
|
|
|
52
52
|
types: [pyi]
|
|
53
53
|
|
|
54
54
|
- repo: https://github.com/pappasam/toml-sort
|
|
55
|
-
rev: v0.24.
|
|
55
|
+
rev: v0.24.4
|
|
56
56
|
hooks:
|
|
57
57
|
- id: toml-sort-fix
|
|
58
58
|
args: [--in-place, --spaces-indent-inline-array, '4']
|
|
@@ -75,7 +75,7 @@ repos:
|
|
|
75
75
|
args: [-s, bash]
|
|
76
76
|
|
|
77
77
|
- repo: https://github.com/lovesegfault/beautysh
|
|
78
|
-
rev: v6.4.
|
|
78
|
+
rev: v6.4.3
|
|
79
79
|
hooks:
|
|
80
80
|
- id: beautysh
|
|
81
81
|
args: [--indent-size, '4']
|
|
@@ -91,7 +91,7 @@ repos:
|
|
|
91
91
|
- id: pyupgrade
|
|
92
92
|
|
|
93
93
|
- repo: https://github.com/commitizen-tools/commitizen
|
|
94
|
-
rev: v4.
|
|
94
|
+
rev: v4.16.3
|
|
95
95
|
hooks:
|
|
96
96
|
- id: commitizen
|
|
97
97
|
stages: [commit-msg]
|
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
## v5.4.0 (2026-06-09)
|
|
2
|
+
|
|
3
|
+
### Feat
|
|
4
|
+
|
|
5
|
+
- add "rename_index=True" to printiso function for functions in hspf_utils to name the index "BALANCE TERM"
|
|
6
|
+
|
|
7
|
+
## v5.3.0 (2026-06-08)
|
|
8
|
+
|
|
9
|
+
### Feat
|
|
10
|
+
|
|
11
|
+
- added pandas_period_by_version to adjust period alias code for pandas<2.2 vs pandas>=2.2
|
|
12
|
+
|
|
13
|
+
## v5.2.5 (2026-06-07)
|
|
14
|
+
|
|
15
|
+
### Fix
|
|
16
|
+
|
|
17
|
+
- fix and refactor hbn reader to make hdf reader easier to implement
|
|
18
|
+
- fix the csv_nos and tsc_nos to be more robust and not remove spaces in the middle of values
|
|
19
|
+
- remove spaces either side of column names and index
|
|
20
|
+
|
|
1
21
|
## v5.2.4 (2026-03-27)
|
|
2
22
|
|
|
3
23
|
### Fix
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
5.4.0
|
|
@@ -1,23 +1,26 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""function to read HSPF binary files."""
|
|
2
2
|
|
|
3
3
|
import datetime
|
|
4
4
|
import struct
|
|
5
5
|
import sys
|
|
6
|
-
|
|
7
|
-
try:
|
|
8
|
-
from typing import Literal
|
|
9
|
-
except ImportError:
|
|
10
|
-
from typing import Literal
|
|
6
|
+
from typing import Literal
|
|
11
7
|
|
|
12
8
|
import pandas as pd
|
|
13
9
|
|
|
14
10
|
from .. import tsutils
|
|
11
|
+
from ..utils import pandas_period_by_version
|
|
12
|
+
from . import utils
|
|
15
13
|
|
|
16
14
|
code2intervalmap = {5: "yearly", 4: "monthly", 3: "daily", 2: "bivl"}
|
|
17
15
|
|
|
18
16
|
interval2codemap = {"yearly": 5, "monthly": 4, "daily": 3, "bivl": 2}
|
|
19
17
|
|
|
20
|
-
code2freqmap = {
|
|
18
|
+
code2freqmap = {
|
|
19
|
+
5: pandas_period_by_version("Y"),
|
|
20
|
+
4: "M",
|
|
21
|
+
3: "D",
|
|
22
|
+
2: None,
|
|
23
|
+
}
|
|
21
24
|
|
|
22
25
|
|
|
23
26
|
_LOCAL_DOCSTRINGS = {
|
|
@@ -27,168 +30,20 @@ _LOCAL_DOCSTRINGS = {
|
|
|
27
30
|
}
|
|
28
31
|
|
|
29
32
|
|
|
30
|
-
def tuple_match(findme, hay):
|
|
31
|
-
"""Part of partial ordered matching.
|
|
32
|
-
See http://stackoverflow.com/a/4559604
|
|
33
|
-
"""
|
|
34
|
-
return len(findme) == len(hay) and all(
|
|
35
|
-
i is None or j is None or i == j for i, j in zip(findme, hay)
|
|
36
|
-
)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def tuple_combine(findme, hay):
|
|
40
|
-
"""Part of partial ordered matching.
|
|
41
|
-
See http://stackoverflow.com/a/4559604
|
|
42
|
-
"""
|
|
43
|
-
return tuple(i is None and j or i for i, j in zip(findme, hay))
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def tuple_search(findme, haystack):
|
|
47
|
-
"""Partial ordered matching with 'None' as wildcard
|
|
48
|
-
See http://stackoverflow.com/a/4559604
|
|
49
|
-
"""
|
|
50
|
-
return [
|
|
51
|
-
(index, tuple_combine(findme, hay))
|
|
52
|
-
for index, hay in enumerate(haystack)
|
|
53
|
-
if tuple_match(findme, hay)
|
|
54
|
-
]
|
|
55
|
-
|
|
56
|
-
|
|
57
33
|
def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
|
|
58
34
|
"""Underlying function to read from the binary file. Used by
|
|
59
35
|
'extract', 'catalog'.
|
|
60
36
|
"""
|
|
61
|
-
if labels is None:
|
|
62
|
-
labels = [",,,"]
|
|
63
|
-
testem = {
|
|
64
|
-
"PERLND": [
|
|
65
|
-
"ATEMP",
|
|
66
|
-
"SNOW",
|
|
67
|
-
"PWATER",
|
|
68
|
-
"SEDMNT",
|
|
69
|
-
"PSTEMP",
|
|
70
|
-
"PWTGAS",
|
|
71
|
-
"PQUAL",
|
|
72
|
-
"MSTLAY",
|
|
73
|
-
"PEST",
|
|
74
|
-
"NITR",
|
|
75
|
-
"PHOS",
|
|
76
|
-
"TRACER",
|
|
77
|
-
"",
|
|
78
|
-
],
|
|
79
|
-
"IMPLND": ["ATEMP", "SNOW", "IWATER", "SOLIDS", "IWTGAS", "IQUAL", ""],
|
|
80
|
-
"RCHRES": [
|
|
81
|
-
"HYDR",
|
|
82
|
-
"CONS",
|
|
83
|
-
"HTRCH",
|
|
84
|
-
"SEDTRN",
|
|
85
|
-
"GQUAL",
|
|
86
|
-
"OXRX",
|
|
87
|
-
"NUTRX",
|
|
88
|
-
"PLANK",
|
|
89
|
-
"PHCARB",
|
|
90
|
-
"INFLOW",
|
|
91
|
-
"OFLOW",
|
|
92
|
-
"ROFLOW",
|
|
93
|
-
"",
|
|
94
|
-
],
|
|
95
|
-
"BMPRAC": [""],
|
|
96
|
-
"": [""],
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
collect_dict = {}
|
|
100
|
-
lablist = []
|
|
101
|
-
|
|
102
37
|
# Normalize interval code
|
|
103
38
|
try:
|
|
104
39
|
intervalcode = interval2codemap[interval.lower()]
|
|
105
40
|
except AttributeError:
|
|
106
41
|
intervalcode = None
|
|
107
42
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
# turn into a list of lists
|
|
112
|
-
nlabels = []
|
|
113
|
-
for label in labels:
|
|
114
|
-
if isinstance(label, str):
|
|
115
|
-
nlabels.append(label.split(","))
|
|
116
|
-
else:
|
|
117
|
-
nlabels.append(label)
|
|
118
|
-
labels = nlabels
|
|
119
|
-
|
|
120
|
-
# Check the list members for valid values
|
|
121
|
-
for label in labels:
|
|
122
|
-
if len(label) != 4:
|
|
123
|
-
raise ValueError(
|
|
124
|
-
tsutils.error_wrapper(
|
|
125
|
-
f"""The label '{label}' has the wrong number of entries.
|
|
126
|
-
"""
|
|
127
|
-
)
|
|
128
|
-
)
|
|
129
|
-
|
|
130
|
-
# replace empty fields with None
|
|
131
|
-
# operation,lue_number,group,variable
|
|
132
|
-
words = [None if (i in ("", "None")) else i for i in label]
|
|
133
|
-
|
|
134
|
-
# first word must be a valid operation type or None
|
|
135
|
-
if words[0] is not None:
|
|
136
|
-
# force uppercase before comparison
|
|
137
|
-
words[0] = words[0].upper()
|
|
138
|
-
if words[0] not in testem:
|
|
139
|
-
raise ValueError(
|
|
140
|
-
tsutils.error_wrapper(
|
|
141
|
-
f"""Operation type must be one of 'PERLND', 'IMPLND',
|
|
142
|
-
'RCHRES', or 'BMPRAC', or missing (to get all) instead
|
|
143
|
-
of {words[0]}.
|
|
144
|
-
"""
|
|
145
|
-
)
|
|
146
|
-
)
|
|
147
|
-
|
|
148
|
-
# second word must be integer 1-999 or None or range to parse
|
|
149
|
-
if words[1] is not None:
|
|
150
|
-
try:
|
|
151
|
-
words[1] = int(words[1])
|
|
152
|
-
luelist = [words[1]]
|
|
153
|
-
except ValueError:
|
|
154
|
-
luelist = tsutils.range_to_numlist(words[1])
|
|
155
|
-
for luenum in luelist:
|
|
156
|
-
if luenum < 1 or luenum > 999:
|
|
157
|
-
raise ValueError(
|
|
158
|
-
tsutils.error_wrapper(
|
|
159
|
-
f"""The land use element must be an integer from
|
|
160
|
-
1 to 999 inclusive, instead of {luenum}.
|
|
161
|
-
"""
|
|
162
|
-
)
|
|
163
|
-
)
|
|
164
|
-
else:
|
|
165
|
-
luelist = [None]
|
|
166
|
-
|
|
167
|
-
# third word must be a valid group name or None
|
|
168
|
-
if words[2] is not None:
|
|
169
|
-
words[2] = words[2].upper()
|
|
170
|
-
if words[2] not in testem[words[0]]:
|
|
171
|
-
raise ValueError(
|
|
172
|
-
tsutils.error_wrapper(
|
|
173
|
-
f"""The {words[0]} operation type only allows the
|
|
174
|
-
variable groups: {testem[words[0]][:-1]}, instead you
|
|
175
|
-
gave {words[2]}.
|
|
176
|
-
"""
|
|
177
|
-
)
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
# fourth word is currently not checked - assumed to be a variable name
|
|
181
|
-
# if not, it will simply never be found in the file, so ok
|
|
182
|
-
# but no warning for the user - add check?
|
|
183
|
-
|
|
184
|
-
# add interval code as fifth word in list
|
|
185
|
-
words.append(intervalcode)
|
|
186
|
-
|
|
187
|
-
# add to new list of checked and expanded lists
|
|
188
|
-
for luenum in luelist:
|
|
189
|
-
words[1] = luenum
|
|
190
|
-
lablist.append(list(words))
|
|
43
|
+
lablist = utils.normalize_labels(labels)
|
|
44
|
+
lablist = [i + [intervalcode] for i in lablist]
|
|
191
45
|
|
|
46
|
+
collect_dict = {}
|
|
192
47
|
# Now read through the binary file and collect the data matching the labels
|
|
193
48
|
with open(binfilename, "rb") as binfp:
|
|
194
49
|
labeltest = set()
|
|
@@ -200,7 +55,8 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
|
|
|
200
55
|
# not a valid HSPF binary file
|
|
201
56
|
raise ValueError(
|
|
202
57
|
tsutils.error_wrapper(
|
|
203
|
-
f"""
|
|
58
|
+
f"""
|
|
59
|
+
{binfilename} is not a valid HSPF binary output file
|
|
204
60
|
(.hbn), The first byte must be FD hexadecimal, but it was
|
|
205
61
|
{magicbyte}.
|
|
206
62
|
"""
|
|
@@ -251,9 +107,7 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
|
|
|
251
107
|
# read the variable name
|
|
252
108
|
variable_name = struct.unpack(f"{length}s", binfp.read(length))[0]
|
|
253
109
|
|
|
254
|
-
# add variable name to the
|
|
255
|
-
# why a set instead of a list? There should never be
|
|
256
|
-
# a duplicate anyway
|
|
110
|
+
# add variable name to the vnames list
|
|
257
111
|
vnames.setdefault((lue, group), []).append(variable_name)
|
|
258
112
|
|
|
259
113
|
# update how far along the record we are
|
|
@@ -275,11 +129,15 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
|
|
|
275
129
|
vals = struct.unpack(f"{numvals}f", binfp.read(4 * numvals))
|
|
276
130
|
recpos += 4 * numvals
|
|
277
131
|
|
|
132
|
+
# delta accounts for HSPF's use of hour 24 to represent
|
|
133
|
+
# the end of the last interval of the day.
|
|
278
134
|
delta = datetime.timedelta(hours=0)
|
|
279
|
-
if
|
|
280
|
-
|
|
135
|
+
if level == interval2codemap["bivl"]:
|
|
136
|
+
delta = datetime.timedelta(hours=hour) + datetime.timedelta(
|
|
137
|
+
minutes=minute
|
|
138
|
+
)
|
|
281
139
|
|
|
282
|
-
ndate = datetime.datetime(year, month, day
|
|
140
|
+
ndate = datetime.datetime(year, month, day) + delta
|
|
283
141
|
|
|
284
142
|
# Go through labels to see if these values need to be
|
|
285
143
|
# collected
|
|
@@ -291,9 +149,8 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
|
|
|
291
149
|
vname.decode("ascii"),
|
|
292
150
|
level,
|
|
293
151
|
)
|
|
294
|
-
|
|
295
152
|
for lbl in lablist:
|
|
296
|
-
res = tuple_search(tmpkey, [lbl])
|
|
153
|
+
res = utils.tuple_search(tmpkey, [lbl])
|
|
297
154
|
if not res:
|
|
298
155
|
continue
|
|
299
156
|
labeltest.add(tuple(lbl))
|
|
@@ -322,8 +179,9 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
|
|
|
322
179
|
if not collect_dict:
|
|
323
180
|
raise ValueError(
|
|
324
181
|
tsutils.error_wrapper(
|
|
325
|
-
f"""
|
|
326
|
-
binary
|
|
182
|
+
f"""
|
|
183
|
+
The label specifications below matched no records in the binary
|
|
184
|
+
file.
|
|
327
185
|
|
|
328
186
|
{lablist}
|
|
329
187
|
"""
|
|
@@ -337,8 +195,9 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
|
|
|
337
195
|
if tuple(lbl) not in labeltest:
|
|
338
196
|
sys.stderr.write(
|
|
339
197
|
tsutils.error_wrapper(
|
|
340
|
-
f"""
|
|
341
|
-
|
|
198
|
+
f"""
|
|
199
|
+
Warning: The label '{lbl}' matched no records in the
|
|
200
|
+
binary file.
|
|
342
201
|
"""
|
|
343
202
|
)
|
|
344
203
|
)
|
|
@@ -365,7 +224,8 @@ def hbn_extract(
|
|
|
365
224
|
if interval not in ("bivl", "daily", "monthly", "yearly"):
|
|
366
225
|
raise ValueError(
|
|
367
226
|
tsutils.error_wrapper(
|
|
368
|
-
f"""
|
|
227
|
+
f"""
|
|
228
|
+
The "interval" argument must be one of "bivl", "daily",
|
|
369
229
|
"monthly", or "yearly". You supplied "{interval}".
|
|
370
230
|
"""
|
|
371
231
|
)
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""A collection of functions used by toolbox_utils, wdmtoolbox, ...etc."""
|
|
2
|
+
|
|
3
|
+
from typing import List, Optional, Union
|
|
4
|
+
|
|
5
|
+
import pint_pandas # not used directly, but required to use pint in pandas
|
|
6
|
+
|
|
7
|
+
from .. import tsutils
|
|
8
|
+
|
|
9
|
+
# This is here so that linters don't remove the pint_pandas import which is
|
|
10
|
+
# needed to use pint in pandas
|
|
11
|
+
_ = pint_pandas.version("pint")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def normalize_labels(labels: Optional[Union[str, List[str]]]) -> List[str]:
|
|
15
|
+
"""
|
|
16
|
+
Process labels for the hbn function.
|
|
17
|
+
|
|
18
|
+
Parameters
|
|
19
|
+
----------
|
|
20
|
+
labels
|
|
21
|
+
The labels to be processed.
|
|
22
|
+
|
|
23
|
+
Returns
|
|
24
|
+
-------
|
|
25
|
+
process_labels
|
|
26
|
+
A list of processed labels.
|
|
27
|
+
"""
|
|
28
|
+
if labels is None:
|
|
29
|
+
labels = [",,,"]
|
|
30
|
+
|
|
31
|
+
testem = {
|
|
32
|
+
"PERLND": [
|
|
33
|
+
"ATEMP",
|
|
34
|
+
"SNOW",
|
|
35
|
+
"PWATER",
|
|
36
|
+
"SEDMNT",
|
|
37
|
+
"PSTEMP",
|
|
38
|
+
"PWTGAS",
|
|
39
|
+
"PQUAL",
|
|
40
|
+
"MSTLAY",
|
|
41
|
+
"PEST",
|
|
42
|
+
"NITR",
|
|
43
|
+
"PHOS",
|
|
44
|
+
"TRACER",
|
|
45
|
+
"",
|
|
46
|
+
],
|
|
47
|
+
"IMPLND": ["ATEMP", "SNOW", "IWATER", "SOLIDS", "IWTGAS", "IQUAL", ""],
|
|
48
|
+
"RCHRES": [
|
|
49
|
+
"HYDR",
|
|
50
|
+
"CONS",
|
|
51
|
+
"HTRCH",
|
|
52
|
+
"SEDTRN",
|
|
53
|
+
"GQUAL",
|
|
54
|
+
"OXRX",
|
|
55
|
+
"NUTRX",
|
|
56
|
+
"PLANK",
|
|
57
|
+
"PHCARB",
|
|
58
|
+
"INFLOW",
|
|
59
|
+
"OFLOW",
|
|
60
|
+
"ROFLOW",
|
|
61
|
+
"",
|
|
62
|
+
],
|
|
63
|
+
"BMPRAC": [""],
|
|
64
|
+
"": [""],
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
lablist = []
|
|
68
|
+
|
|
69
|
+
# convert label tuples to lists
|
|
70
|
+
labels = list(labels)
|
|
71
|
+
|
|
72
|
+
# turn into a list of lists
|
|
73
|
+
nlabels = []
|
|
74
|
+
for label in labels:
|
|
75
|
+
if isinstance(label, str):
|
|
76
|
+
nlabels.append(label.split(","))
|
|
77
|
+
else:
|
|
78
|
+
nlabels.append(label)
|
|
79
|
+
labels = nlabels
|
|
80
|
+
|
|
81
|
+
# Check the list members for valid values
|
|
82
|
+
for label in labels:
|
|
83
|
+
if len(label) != 4:
|
|
84
|
+
raise ValueError(
|
|
85
|
+
tsutils.error_wrapper(
|
|
86
|
+
f"""
|
|
87
|
+
The label '{label}' has the wrong number of entries.
|
|
88
|
+
"""
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# replace empty fields with None
|
|
93
|
+
words = [None if i == "" else i for i in label]
|
|
94
|
+
|
|
95
|
+
# first word must be a valid operation type or None
|
|
96
|
+
if words[0] is not None:
|
|
97
|
+
# force uppercase before comparison
|
|
98
|
+
words[0] = words[0].upper()
|
|
99
|
+
if words[0] not in testem:
|
|
100
|
+
raise ValueError(
|
|
101
|
+
tsutils.error_wrapper(
|
|
102
|
+
f"""
|
|
103
|
+
Operation type must be one of 'PERLND', 'IMPLND',
|
|
104
|
+
'RCHRES', or 'BMPRAC', or missing (to get all) instead
|
|
105
|
+
of {words[0]}.
|
|
106
|
+
"""
|
|
107
|
+
)
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# second word must be integer 1-999 or None or range to parse
|
|
111
|
+
if words[1] is not None:
|
|
112
|
+
try:
|
|
113
|
+
words[1] = int(words[1])
|
|
114
|
+
luelist = [words[1]]
|
|
115
|
+
except ValueError:
|
|
116
|
+
luelist = tsutils.range_to_numlist(words[1])
|
|
117
|
+
for luenum in luelist:
|
|
118
|
+
if luenum < 1 or luenum > 999:
|
|
119
|
+
raise ValueError(
|
|
120
|
+
tsutils.error_wrapper(
|
|
121
|
+
f"""
|
|
122
|
+
The land use element must be an integer from 1 to
|
|
123
|
+
999 inclusive, instead of {luenum}.
|
|
124
|
+
"""
|
|
125
|
+
)
|
|
126
|
+
)
|
|
127
|
+
else:
|
|
128
|
+
luelist = [None]
|
|
129
|
+
|
|
130
|
+
# third word must be a valid group name or None
|
|
131
|
+
if words[2] is not None:
|
|
132
|
+
words[2] = words[2].upper()
|
|
133
|
+
if (words[0] is not None) and (words[2] not in testem[words[0]]):
|
|
134
|
+
raise ValueError(
|
|
135
|
+
tsutils.error_wrapper(
|
|
136
|
+
f"""
|
|
137
|
+
The {words[0]} operation type only allows the variable
|
|
138
|
+
groups: {testem[words[0]][:-1]},
|
|
139
|
+
instead you gave {words[2]}.
|
|
140
|
+
"""
|
|
141
|
+
)
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
# fourth word is currently not checked - assumed to be a variable name
|
|
145
|
+
# if not, it will simply never be found in the file, so ok
|
|
146
|
+
# but no warning for the user - add check?
|
|
147
|
+
|
|
148
|
+
# add to new list of checked and expanded lists
|
|
149
|
+
for luenum in luelist:
|
|
150
|
+
words[1] = luenum
|
|
151
|
+
lablist.append(list(words))
|
|
152
|
+
return lablist
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def tuple_match(findme, hay):
|
|
156
|
+
"""Part of partial ordered matching.
|
|
157
|
+
See http://stackoverflow.com/a/4559604
|
|
158
|
+
"""
|
|
159
|
+
return len(findme) == len(hay) and all(
|
|
160
|
+
i is None or j is None or i == j for i, j in zip(findme, hay)
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def tuple_combine(findme, hay):
|
|
165
|
+
"""Part of partial ordered matching.
|
|
166
|
+
See http://stackoverflow.com/a/4559604
|
|
167
|
+
"""
|
|
168
|
+
return tuple(i is None and j or i for i, j in zip(findme, hay))
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def tuple_search(findme, haystack):
|
|
172
|
+
"""Partial ordered matching with 'None' as wildcard
|
|
173
|
+
See http://stackoverflow.com/a/4559604
|
|
174
|
+
"""
|
|
175
|
+
return [
|
|
176
|
+
(index, tuple_combine(findme, hay))
|
|
177
|
+
for index, hay in enumerate(haystack)
|
|
178
|
+
if tuple_match(findme, hay)
|
|
179
|
+
]
|
|
@@ -28,12 +28,6 @@ from numpy import int64, ndarray
|
|
|
28
28
|
from pandas.core.frame import DataFrame
|
|
29
29
|
from pandas.core.indexes.base import Index
|
|
30
30
|
from pandas.tseries.frequencies import to_offset
|
|
31
|
-
|
|
32
|
-
try:
|
|
33
|
-
from pydantic import validate_call
|
|
34
|
-
except ImportError:
|
|
35
|
-
from pydantic import validate_arguments as validate_call
|
|
36
|
-
|
|
37
31
|
from scipy.stats.distributions import lognorm, norm
|
|
38
32
|
from tabulate import simple_separated_format
|
|
39
33
|
from tabulate import tabulate as tb
|
|
@@ -43,6 +37,11 @@ from .readers.plotgen import plotgen_extract as plotgen
|
|
|
43
37
|
from .readers.wdm import wdm_extract as wdm
|
|
44
38
|
from .utils import pandas_offset_by_version
|
|
45
39
|
|
|
40
|
+
try:
|
|
41
|
+
from pydantic import validate_call
|
|
42
|
+
except ImportError:
|
|
43
|
+
from pydantic import validate_arguments as validate_call
|
|
44
|
+
|
|
46
45
|
# This is here so that linters don't remove the pint_pandas import which is
|
|
47
46
|
# needed to use pint in pandas
|
|
48
47
|
_ = pint_pandas.version("pint")
|
|
@@ -723,7 +722,7 @@ def copy_doc(source: Callable) -> Callable:
|
|
|
723
722
|
|
|
724
723
|
def wrapper_copy_doc(func: Callable) -> Callable:
|
|
725
724
|
if source.__doc__:
|
|
726
|
-
func.__doc__ = source.__doc__
|
|
725
|
+
func.__doc__ = source.__doc__
|
|
727
726
|
|
|
728
727
|
return func
|
|
729
728
|
|
|
@@ -1477,7 +1476,9 @@ def common_kwds(
|
|
|
1477
1476
|
|
|
1478
1477
|
ntsd = _date_slice(ntsd, start_date=start_date, end_date=end_date, por=por)
|
|
1479
1478
|
|
|
1480
|
-
if
|
|
1479
|
+
if (
|
|
1480
|
+
not ntsd.index.name or "Datetime" not in ntsd.index.name
|
|
1481
|
+
) and ntsd.index.inferred_type == "datetime64":
|
|
1481
1482
|
ntsd.index.name = "Datetime"
|
|
1482
1483
|
|
|
1483
1484
|
if dropna in ("any", "all"):
|
|
@@ -1494,6 +1495,8 @@ def common_kwds(
|
|
|
1494
1495
|
return ntsd.resample(groupby)
|
|
1495
1496
|
|
|
1496
1497
|
ntsd[ntsd.isna()] = np.nan
|
|
1498
|
+
ntsd.columns = [i.strip() for i in ntsd.columns]
|
|
1499
|
+
ntsd.index.name = ntsd.index.name.strip()
|
|
1497
1500
|
return ntsd
|
|
1498
1501
|
|
|
1499
1502
|
|
|
@@ -1984,8 +1987,9 @@ def _printiso(
|
|
|
1984
1987
|
showindex: Union[str, bool] = True,
|
|
1985
1988
|
headers: str = "keys",
|
|
1986
1989
|
tablefmt: Optional[str] = "csv",
|
|
1990
|
+
rename_index: bool = True,
|
|
1987
1991
|
) -> None:
|
|
1988
|
-
"""
|
|
1992
|
+
"""Print data. If time series data, print in ISO format."""
|
|
1989
1993
|
showindex = {"always": True, "never": False, True: True, False: False}[showindex]
|
|
1990
1994
|
|
|
1991
1995
|
if isinstance(tsd, (pd.DataFrame, pd.Series)):
|
|
@@ -1995,11 +1999,20 @@ def _printiso(
|
|
|
1995
1999
|
if tsd.columns.empty:
|
|
1996
2000
|
tsd = pd.DataFrame(index=tsd.index)
|
|
1997
2001
|
|
|
1998
|
-
if
|
|
2002
|
+
if rename_index and (
|
|
2003
|
+
(not tsd.index.name) or ("Datetime" not in tsd.index.name)
|
|
2004
|
+
):
|
|
1999
2005
|
tsd.index.name = "UniqueID"
|
|
2000
2006
|
|
|
2001
|
-
|
|
2002
|
-
|
|
2007
|
+
if isinstance(tsd.index, pd.DatetimeIndex):
|
|
2008
|
+
timezone = tsd.index.tz
|
|
2009
|
+
if timezone:
|
|
2010
|
+
tsd.index.name = f"Datetime:{timezone}"
|
|
2011
|
+
else:
|
|
2012
|
+
tsd.index.name = "Datetime"
|
|
2013
|
+
|
|
2014
|
+
if isinstance(tsd.index, pd.PeriodIndex):
|
|
2015
|
+
tsd.index.name = "Period"
|
|
2003
2016
|
|
|
2004
2017
|
elif isinstance(tsd, (int, float, tuple, np.ndarray)):
|
|
2005
2018
|
tablefmt = None
|
|
@@ -2018,7 +2031,6 @@ def _printiso(
|
|
|
2018
2031
|
sep=sep,
|
|
2019
2032
|
index=showindex,
|
|
2020
2033
|
)
|
|
2021
|
-
|
|
2022
2034
|
return
|
|
2023
2035
|
except OSError:
|
|
2024
2036
|
return
|
|
@@ -2027,6 +2039,7 @@ def _printiso(
|
|
|
2027
2039
|
|
|
2028
2040
|
if tablefmt is None:
|
|
2029
2041
|
print(str(list(tsd))[1:-1])
|
|
2042
|
+
return
|
|
2030
2043
|
|
|
2031
2044
|
if ntablefmt is None:
|
|
2032
2045
|
all_table = tb(
|
|
@@ -2046,7 +2059,7 @@ def _printiso(
|
|
|
2046
2059
|
)
|
|
2047
2060
|
|
|
2048
2061
|
if tablefmt in ("csv_nos", "tsv_nos"):
|
|
2049
|
-
print(
|
|
2062
|
+
print(re.sub(r" *, *", ",", all_table))
|
|
2050
2063
|
else:
|
|
2051
2064
|
print(all_table)
|
|
2052
2065
|
|
|
@@ -2135,7 +2148,6 @@ def memory_optimize(tsd: DataFrame) -> DataFrame:
|
|
|
2135
2148
|
# TypeError: Not datetime like index
|
|
2136
2149
|
# ValueError: Less than three rows
|
|
2137
2150
|
tsd.index.freq = pd.infer_freq(tsd.index)
|
|
2138
|
-
|
|
2139
2151
|
return tsd
|
|
2140
2152
|
|
|
2141
2153
|
|
|
@@ -2365,7 +2377,7 @@ def read_iso_ts(
|
|
|
2365
2377
|
res = res.join(hbn(fname, interval, labels), how="outer")
|
|
2366
2378
|
elif ext.lower() == ".plt":
|
|
2367
2379
|
res = plotgen(fname)
|
|
2368
|
-
elif ext.lower()
|
|
2380
|
+
elif ext.lower() in (".h5", ".hdf5"):
|
|
2369
2381
|
if args:
|
|
2370
2382
|
res = pd.DataFrame()
|
|
2371
2383
|
|
|
@@ -2572,9 +2584,10 @@ def read_iso_ts(
|
|
|
2572
2584
|
result = lresult_list[0]
|
|
2573
2585
|
|
|
2574
2586
|
# Assign names to the index and columns.
|
|
2575
|
-
|
|
2576
2587
|
if names is not None:
|
|
2577
|
-
|
|
2588
|
+
possible_index_name = names.pop(0)
|
|
2589
|
+
if not result.index.name:
|
|
2590
|
+
result.index.name = possible_index_name
|
|
2578
2591
|
result.columns = names
|
|
2579
2592
|
|
|
2580
2593
|
result.sort_index(inplace=True)
|
|
@@ -1,6 +1,58 @@
|
|
|
1
1
|
import pandas as pd
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
def pandas_period_by_version(new_offset: str) -> str:
|
|
5
|
+
"""
|
|
6
|
+
Convert the time offset code to match the version of pandas.
|
|
7
|
+
|
|
8
|
+
+------------------------+--------+---------+-----------------------------+
|
|
9
|
+
| DateOffset | less | greater | Generic offset class, |
|
|
10
|
+
| | 2.2 | equal | defaults to absolute 24 |
|
|
11
|
+
| | | 2.2 | hours |
|
|
12
|
+
+========================+========+=========+=============================+
|
|
13
|
+
| Year | 'A' | 'Y ' | calendar year |
|
|
14
|
+
+------------------------+--------+---------+-----------------------------+
|
|
15
|
+
| Hour | 'H' | 'h' | one hour |
|
|
16
|
+
+------------------------+--------+---------+-----------------------------+
|
|
17
|
+
| Minute | 'T' or | 'min' | one minute |
|
|
18
|
+
| | 'min' | | |
|
|
19
|
+
+------------------------+--------+---------+-----------------------------+
|
|
20
|
+
| Second | 'S' | 's' | one second |
|
|
21
|
+
+------------------------+--------+---------+-----------------------------+
|
|
22
|
+
| Milli | 'L' or | 'ms' | one millisecond |
|
|
23
|
+
| | 'ms' | | |
|
|
24
|
+
+------------------------+--------+---------+-----------------------------+
|
|
25
|
+
| Micro | 'U' or | 'us' | one microsecond |
|
|
26
|
+
| | 'us' | | |
|
|
27
|
+
+------------------------+--------+---------+-----------------------------+
|
|
28
|
+
| Nano | 'N' | 'ns' | one nanosecond |
|
|
29
|
+
+------------------------+--------+---------+-----------------------------+
|
|
30
|
+
|
|
31
|
+
Parameters
|
|
32
|
+
----------
|
|
33
|
+
offset
|
|
34
|
+
The new style offset to convert if needed for older pandas version.
|
|
35
|
+
|
|
36
|
+
Returns
|
|
37
|
+
-------
|
|
38
|
+
offset_by_version
|
|
39
|
+
The offset for the installed version of pandas.
|
|
40
|
+
"""
|
|
41
|
+
new_to_old_period = {}
|
|
42
|
+
major, minor = pd.__version__.split(".")[:2]
|
|
43
|
+
if (int(major) + int(minor) / 10) < 2.2:
|
|
44
|
+
new_to_old_freq = {
|
|
45
|
+
"Y": "A",
|
|
46
|
+
"h": "H",
|
|
47
|
+
"min": "T",
|
|
48
|
+
"s": "S",
|
|
49
|
+
"ms": "L",
|
|
50
|
+
"us": "U",
|
|
51
|
+
"ns": "N",
|
|
52
|
+
}
|
|
53
|
+
return new_to_old_period.get(new_offset, new_offset)
|
|
54
|
+
|
|
55
|
+
|
|
4
56
|
def pandas_offset_by_version(new_offset: str) -> str:
|
|
5
57
|
"""
|
|
6
58
|
Convert the time offset code to match the version of pandas.
|
|
@@ -33,6 +33,7 @@ src/toolbox_utils.egg-info/top_level.txt
|
|
|
33
33
|
src/toolbox_utils/readers/__init__.py
|
|
34
34
|
src/toolbox_utils/readers/hbn.py
|
|
35
35
|
src/toolbox_utils/readers/plotgen.py
|
|
36
|
+
src/toolbox_utils/readers/utils.py
|
|
36
37
|
src/toolbox_utils/readers/wdm.py
|
|
37
38
|
tests/data.wdm
|
|
38
39
|
tests/data_bi_daily.csv
|
toolbox_utils-5.2.4/VERSION
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
5.2.4
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|