toolbox-utils 5.2.3__tar.gz → 5.2.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/.github/workflows/pypi-package.yml +1 -1
  2. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/.pre-commit-config.yaml +5 -5
  3. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/CHANGELOG.md +15 -0
  4. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/PKG-INFO +1 -1
  5. toolbox_utils-5.2.5/VERSION +1 -0
  6. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/pyproject.toml +1 -1
  7. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils/readers/hbn.py +28 -168
  8. toolbox_utils-5.2.5/src/toolbox_utils/readers/utils.py +179 -0
  9. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils/tsutils.py +55 -50
  10. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils.egg-info/PKG-INFO +1 -1
  11. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils.egg-info/SOURCES.txt +1 -0
  12. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_asbestfreq.py +5 -3
  13. toolbox_utils-5.2.3/VERSION +0 -1
  14. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/.deepsource.toml +0 -0
  15. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/.github/dependabot.yml +0 -0
  16. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/.github/workflows/clean-workflow-runs.yml +0 -0
  17. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/.github/workflows/tests.yml +0 -0
  18. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/.gitignore +0 -0
  19. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/.sourcery.yaml +0 -0
  20. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/AUTHORS.rst +0 -0
  21. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/BADGES.rst +0 -0
  22. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/CONTRIBUTING.rst +0 -0
  23. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/LICENSE.txt +0 -0
  24. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/README.rst +0 -0
  25. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/docs/Makefile +0 -0
  26. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/docs/authors.rst +0 -0
  27. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/docs/conf.py +0 -0
  28. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/docs/contributing.rst +0 -0
  29. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/docs/index.rst +0 -0
  30. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/docs/license.rst +0 -0
  31. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/docs/make.bat +0 -0
  32. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/docs/readme.rst +0 -0
  33. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/setup.cfg +0 -0
  34. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils/__init__.py +0 -0
  35. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils/readers/__init__.py +0 -0
  36. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils/readers/plotgen.py +0 -0
  37. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils/readers/wdm.py +0 -0
  38. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils/utils.py +0 -0
  39. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils.egg-info/dependency_links.txt +0 -0
  40. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils.egg-info/requires.txt +0 -0
  41. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/src/toolbox_utils.egg-info/top_level.txt +0 -0
  42. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data.wdm +0 -0
  43. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_bi_daily.csv +0 -0
  44. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_bivl.hbn +0 -0
  45. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_end.bivl.csv +0 -0
  46. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_end.daily.csv +0 -0
  47. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_end.monthly.csv +0 -0
  48. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_end.yearly.csv +0 -0
  49. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_flow_stage.xlsx +0 -0
  50. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_period.bivl.csv +0 -0
  51. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_period.daily.csv +0 -0
  52. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_period.monthly.csv +0 -0
  53. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_period.yearly.csv +0 -0
  54. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_plotgen.plt +0 -0
  55. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_simple.csv +0 -0
  56. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_start.bivl.csv +0 -0
  57. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_start.daily.csv +0 -0
  58. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_start.monthly.csv +0 -0
  59. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_start.yearly.csv +0 -0
  60. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_wdm_1.csv +0 -0
  61. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_wdm_2.csv +0 -0
  62. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/data_yearly.hbn +0 -0
  63. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_date_slice.py +0 -0
  64. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_dateparse.py +0 -0
  65. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_extract.py +0 -0
  66. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_hbn.py +0 -0
  67. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_make_list.py +0 -0
  68. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_period_timestamp.py +0 -0
  69. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_plotgen.py +0 -0
  70. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_range_to_numlist.py +0 -0
  71. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_read.py +0 -0
  72. {toolbox_utils-5.2.3 → toolbox_utils-5.2.5}/tests/test_wdm.py +0 -0
@@ -38,7 +38,7 @@ jobs:
38
38
  python -m build
39
39
 
40
40
  - name: Download artifact
41
- uses: actions/download-artifact@v7
41
+ uses: actions/download-artifact@v8
42
42
  with:
43
43
  path: dist
44
44
  merge-multiple: true
@@ -29,7 +29,7 @@ repos:
29
29
 
30
30
  - repo: https://github.com/astral-sh/ruff-pre-commit
31
31
  # Ruff version.
32
- rev: v0.15.2
32
+ rev: v0.15.8
33
33
  hooks:
34
34
  # Run the linter.
35
35
  - id: ruff
@@ -40,7 +40,7 @@ repos:
40
40
  types_or: [python, pyi, jupyter]
41
41
 
42
42
  - repo: https://github.com/pycqa/isort
43
- rev: 8.0.0
43
+ rev: 8.0.1
44
44
  hooks:
45
45
  - id: isort
46
46
  name: isort (python)
@@ -52,7 +52,7 @@ repos:
52
52
  types: [pyi]
53
53
 
54
54
  - repo: https://github.com/pappasam/toml-sort
55
- rev: v0.24.3
55
+ rev: v0.24.4
56
56
  hooks:
57
57
  - id: toml-sort-fix
58
58
  args: [--in-place, --spaces-indent-inline-array, '4']
@@ -75,7 +75,7 @@ repos:
75
75
  args: [-s, bash]
76
76
 
77
77
  - repo: https://github.com/lovesegfault/beautysh
78
- rev: v6.4.2
78
+ rev: v6.4.3
79
79
  hooks:
80
80
  - id: beautysh
81
81
  args: [--indent-size, '4']
@@ -91,7 +91,7 @@ repos:
91
91
  - id: pyupgrade
92
92
 
93
93
  - repo: https://github.com/commitizen-tools/commitizen
94
- rev: v4.13.8
94
+ rev: v4.13.9
95
95
  hooks:
96
96
  - id: commitizen
97
97
  stages: [commit-msg]
@@ -1,3 +1,18 @@
1
+ ## v5.2.5 (2026-06-07)
2
+
3
+ ### Fix
4
+
5
+ - fix and refactor hbn reader to make hdf reader easier to implement
6
+ - fix the csv_nos and tsc_nos to be more robust and not remove spaces in the middle of values
7
+ - remove spaces either side of column names and index
8
+
9
+ ## v5.2.4 (2026-03-27)
10
+
11
+ ### Fix
12
+
13
+ - reordered calculation of best frequency in asbestfreq
14
+ - fix column suffixes to join
15
+
1
16
  ## v5.2.3 (2026-03-06)
2
17
 
3
18
  ### Fix
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: toolbox_utils
3
- Version: 5.2.3
3
+ Version: 5.2.5
4
4
  Summary: Python toolbox for common utilities
5
5
  Author-email: Tim Cera <tim@cerazone.net>
6
6
  License: BSD-3-Clause
@@ -0,0 +1 @@
1
+ 5.2.5
@@ -61,7 +61,7 @@ github = "https://github.com/timcera/toolbox_utils"
61
61
  name = "cz_conventional_commits"
62
62
  tag_format = "v$version"
63
63
  update_changelog_on_bump = true
64
- version = "5.2.3"
64
+ version = "5.2.5"
65
65
  version_files = ["VERSION"]
66
66
 
67
67
  [tool.isort]
@@ -3,21 +3,24 @@
3
3
  import datetime
4
4
  import struct
5
5
  import sys
6
-
7
- try:
8
- from typing import Literal
9
- except ImportError:
10
- from typing import Literal
6
+ from typing import Literal
11
7
 
12
8
  import pandas as pd
13
9
 
14
10
  from .. import tsutils
11
+ from ..utils import pandas_offset_by_version
12
+ from . import utils
15
13
 
16
14
  code2intervalmap = {5: "yearly", 4: "monthly", 3: "daily", 2: "bivl"}
17
15
 
18
16
  interval2codemap = {"yearly": 5, "monthly": 4, "daily": 3, "bivl": 2}
19
17
 
20
- code2freqmap = {5: "A", 4: "M", 3: "D", 2: None}
18
+ code2freqmap = {
19
+ 5: pandas_offset_by_version("YE"),
20
+ 4: pandas_offset_by_version("ME"),
21
+ 3: "D",
22
+ 2: None,
23
+ }
21
24
 
22
25
 
23
26
  _LOCAL_DOCSTRINGS = {
@@ -27,168 +30,20 @@ _LOCAL_DOCSTRINGS = {
27
30
  }
28
31
 
29
32
 
30
- def tuple_match(findme, hay):
31
- """Part of partial ordered matching.
32
- See http://stackoverflow.com/a/4559604
33
- """
34
- return len(findme) == len(hay) and all(
35
- i is None or j is None or i == j for i, j in zip(findme, hay)
36
- )
37
-
38
-
39
- def tuple_combine(findme, hay):
40
- """Part of partial ordered matching.
41
- See http://stackoverflow.com/a/4559604
42
- """
43
- return tuple(i is None and j or i for i, j in zip(findme, hay))
44
-
45
-
46
- def tuple_search(findme, haystack):
47
- """Partial ordered matching with 'None' as wildcard
48
- See http://stackoverflow.com/a/4559604
49
- """
50
- return [
51
- (index, tuple_combine(findme, hay))
52
- for index, hay in enumerate(haystack)
53
- if tuple_match(findme, hay)
54
- ]
55
-
56
-
57
33
  def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
58
34
  """Underlying function to read from the binary file. Used by
59
35
  'extract', 'catalog'.
60
36
  """
61
- if labels is None:
62
- labels = [",,,"]
63
- testem = {
64
- "PERLND": [
65
- "ATEMP",
66
- "SNOW",
67
- "PWATER",
68
- "SEDMNT",
69
- "PSTEMP",
70
- "PWTGAS",
71
- "PQUAL",
72
- "MSTLAY",
73
- "PEST",
74
- "NITR",
75
- "PHOS",
76
- "TRACER",
77
- "",
78
- ],
79
- "IMPLND": ["ATEMP", "SNOW", "IWATER", "SOLIDS", "IWTGAS", "IQUAL", ""],
80
- "RCHRES": [
81
- "HYDR",
82
- "CONS",
83
- "HTRCH",
84
- "SEDTRN",
85
- "GQUAL",
86
- "OXRX",
87
- "NUTRX",
88
- "PLANK",
89
- "PHCARB",
90
- "INFLOW",
91
- "OFLOW",
92
- "ROFLOW",
93
- "",
94
- ],
95
- "BMPRAC": [""],
96
- "": [""],
97
- }
98
-
99
- collect_dict = {}
100
- lablist = []
101
-
102
37
  # Normalize interval code
103
38
  try:
104
39
  intervalcode = interval2codemap[interval.lower()]
105
40
  except AttributeError:
106
41
  intervalcode = None
107
42
 
108
- # convert label tuples to lists
109
- labels = list(labels)
110
-
111
- # turn into a list of lists
112
- nlabels = []
113
- for label in labels:
114
- if isinstance(label, str):
115
- nlabels.append(label.split(","))
116
- else:
117
- nlabels.append(label)
118
- labels = nlabels
119
-
120
- # Check the list members for valid values
121
- for label in labels:
122
- if len(label) != 4:
123
- raise ValueError(
124
- tsutils.error_wrapper(
125
- f"""The label '{label}' has the wrong number of entries.
126
- """
127
- )
128
- )
129
-
130
- # replace empty fields with None
131
- # operation,lue_number,group,variable
132
- words = [None if (i in ("", "None")) else i for i in label]
133
-
134
- # first word must be a valid operation type or None
135
- if words[0] is not None:
136
- # force uppercase before comparison
137
- words[0] = words[0].upper()
138
- if words[0] not in testem:
139
- raise ValueError(
140
- tsutils.error_wrapper(
141
- f"""Operation type must be one of 'PERLND', 'IMPLND',
142
- 'RCHRES', or 'BMPRAC', or missing (to get all) instead
143
- of {words[0]}.
144
- """
145
- )
146
- )
147
-
148
- # second word must be integer 1-999 or None or range to parse
149
- if words[1] is not None:
150
- try:
151
- words[1] = int(words[1])
152
- luelist = [words[1]]
153
- except ValueError:
154
- luelist = tsutils.range_to_numlist(words[1])
155
- for luenum in luelist:
156
- if luenum < 1 or luenum > 999:
157
- raise ValueError(
158
- tsutils.error_wrapper(
159
- f"""The land use element must be an integer from
160
- 1 to 999 inclusive, instead of {luenum}.
161
- """
162
- )
163
- )
164
- else:
165
- luelist = [None]
166
-
167
- # third word must be a valid group name or None
168
- if words[2] is not None:
169
- words[2] = words[2].upper()
170
- if words[2] not in testem[words[0]]:
171
- raise ValueError(
172
- tsutils.error_wrapper(
173
- f"""The {words[0]} operation type only allows the
174
- variable groups: {testem[words[0]][:-1]}, instead you
175
- gave {words[2]}.
176
- """
177
- )
178
- )
179
-
180
- # fourth word is currently not checked - assumed to be a variable name
181
- # if not, it will simply never be found in the file, so ok
182
- # but no warning for the user - add check?
183
-
184
- # add interval code as fifth word in list
185
- words.append(intervalcode)
186
-
187
- # add to new list of checked and expanded lists
188
- for luenum in luelist:
189
- words[1] = luenum
190
- lablist.append(list(words))
43
+ lablist = utils.normalize_labels(labels)
44
+ lablist = [i + [intervalcode] for i in lablist]
191
45
 
46
+ collect_dict = {}
192
47
  # Now read through the binary file and collect the data matching the labels
193
48
  with open(binfilename, "rb") as binfp:
194
49
  labeltest = set()
@@ -200,7 +55,8 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
200
55
  # not a valid HSPF binary file
201
56
  raise ValueError(
202
57
  tsutils.error_wrapper(
203
- f"""{binfilename} is not a valid HSPF binary output file
58
+ f"""
59
+ {binfilename} is not a valid HSPF binary output file
204
60
  (.hbn), The first byte must be FD hexadecimal, but it was
205
61
  {magicbyte}.
206
62
  """
@@ -276,10 +132,12 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
276
132
  recpos += 4 * numvals
277
133
 
278
134
  delta = datetime.timedelta(hours=0)
279
- if hour == 24:
280
- hour = 0
135
+ if level == interval2codemap["bivl"]:
136
+ delta = datetime.timedelta(hours=hour) + datetime.timedelta(
137
+ minutes=minute
138
+ )
281
139
 
282
- ndate = datetime.datetime(year, month, day, hour, minute) + delta
140
+ ndate = datetime.datetime(year, month, day) + delta
283
141
 
284
142
  # Go through labels to see if these values need to be
285
143
  # collected
@@ -291,9 +149,8 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
291
149
  vname.decode("ascii"),
292
150
  level,
293
151
  )
294
-
295
152
  for lbl in lablist:
296
- res = tuple_search(tmpkey, [lbl])
153
+ res = utils.tuple_search(tmpkey, [lbl])
297
154
  if not res:
298
155
  continue
299
156
  labeltest.add(tuple(lbl))
@@ -322,8 +179,9 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
322
179
  if not collect_dict:
323
180
  raise ValueError(
324
181
  tsutils.error_wrapper(
325
- f"""The label specifications below matched no records in the
326
- binary file.
182
+ f"""
183
+ The label specifications below matched no records in the binary
184
+ file.
327
185
 
328
186
  {lablist}
329
187
  """
@@ -337,8 +195,9 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
337
195
  if tuple(lbl) not in labeltest:
338
196
  sys.stderr.write(
339
197
  tsutils.error_wrapper(
340
- f"""Warning: The label '{lbl}' matched no records in
341
- the binary file.
198
+ f"""
199
+ Warning: The label '{lbl}' matched no records in the
200
+ binary file.
342
201
  """
343
202
  )
344
203
  )
@@ -365,7 +224,8 @@ def hbn_extract(
365
224
  if interval not in ("bivl", "daily", "monthly", "yearly"):
366
225
  raise ValueError(
367
226
  tsutils.error_wrapper(
368
- f"""The "interval" argument must be one of "bivl", "daily",
227
+ f"""
228
+ The "interval" argument must be one of "bivl", "daily",
369
229
  "monthly", or "yearly". You supplied "{interval}".
370
230
  """
371
231
  )
@@ -0,0 +1,179 @@
1
+ """A collection of functions used by toolbox_utils, wdmtoolbox, ...etc."""
2
+
3
+ from typing import List, Optional, Union
4
+
5
+ import pint_pandas # not used directly, but required to use pint in pandas
6
+
7
+ from .. import tsutils
8
+
9
+ # This is here so that linters don't remove the pint_pandas import which is
10
+ # needed to use pint in pandas
11
+ _ = pint_pandas.version("pint")
12
+
13
+
14
+ def normalize_labels(labels: Optional[Union[str, List[str]]]) -> List[str]:
15
+ """
16
+ Process labels for the hbn function.
17
+
18
+ Parameters
19
+ ----------
20
+ labels
21
+ The labels to be processed.
22
+
23
+ Returns
24
+ -------
25
+ process_labels
26
+ A list of processed labels.
27
+ """
28
+ if labels is None:
29
+ labels = [",,,"]
30
+
31
+ testem = {
32
+ "PERLND": [
33
+ "ATEMP",
34
+ "SNOW",
35
+ "PWATER",
36
+ "SEDMNT",
37
+ "PSTEMP",
38
+ "PWTGAS",
39
+ "PQUAL",
40
+ "MSTLAY",
41
+ "PEST",
42
+ "NITR",
43
+ "PHOS",
44
+ "TRACER",
45
+ "",
46
+ ],
47
+ "IMPLND": ["ATEMP", "SNOW", "IWATER", "SOLIDS", "IWTGAS", "IQUAL", ""],
48
+ "RCHRES": [
49
+ "HYDR",
50
+ "CONS",
51
+ "HTRCH",
52
+ "SEDTRN",
53
+ "GQUAL",
54
+ "OXRX",
55
+ "NUTRX",
56
+ "PLANK",
57
+ "PHCARB",
58
+ "INFLOW",
59
+ "OFLOW",
60
+ "ROFLOW",
61
+ "",
62
+ ],
63
+ "BMPRAC": [""],
64
+ "": [""],
65
+ }
66
+
67
+ lablist = []
68
+
69
+ # convert label tuples to lists
70
+ labels = list(labels)
71
+
72
+ # turn into a list of lists
73
+ nlabels = []
74
+ for label in labels:
75
+ if isinstance(label, str):
76
+ nlabels.append(label.split(","))
77
+ else:
78
+ nlabels.append(label)
79
+ labels = nlabels
80
+
81
+ # Check the list members for valid values
82
+ for label in labels:
83
+ if len(label) != 4:
84
+ raise ValueError(
85
+ tsutils.error_wrapper(
86
+ f"""
87
+ The label '{label}' has the wrong number of entries.
88
+ """
89
+ )
90
+ )
91
+
92
+ # replace empty fields with None
93
+ words = [None if i == "" else i for i in label]
94
+
95
+ # first word must be a valid operation type or None
96
+ if words[0] is not None:
97
+ # force uppercase before comparison
98
+ words[0] = words[0].upper()
99
+ if words[0] not in testem:
100
+ raise ValueError(
101
+ tsutils.error_wrapper(
102
+ f"""
103
+ Operation type must be one of 'PERLND', 'IMPLND',
104
+ 'RCHRES', or 'BMPRAC', or missing (to get all) instead
105
+ of {words[0]}.
106
+ """
107
+ )
108
+ )
109
+
110
+ # second word must be integer 1-999 or None or range to parse
111
+ if words[1] is not None:
112
+ try:
113
+ words[1] = int(words[1])
114
+ luelist = [words[1]]
115
+ except ValueError:
116
+ luelist = tsutils.range_to_numlist(words[1])
117
+ for luenum in luelist:
118
+ if luenum < 1 or luenum > 999:
119
+ raise ValueError(
120
+ tsutils.error_wrapper(
121
+ f"""
122
+ The land use element must be an integer from 1 to
123
+ 999 inclusive, instead of {luenum}.
124
+ """
125
+ )
126
+ )
127
+ else:
128
+ luelist = [None]
129
+
130
+ # third word must be a valid group name or None
131
+ if words[2] is not None:
132
+ words[2] = words[2].upper()
133
+ if (words[0] is not None) and (words[2] not in testem[words[0]]):
134
+ raise ValueError(
135
+ tsutils.error_wrapper(
136
+ f"""
137
+ The {words[0]} operation type only allows the variable
138
+ groups: {testem[words[0]][:-1]},
139
+ instead you gave {words[2]}.
140
+ """
141
+ )
142
+ )
143
+
144
+ # fourth word is currently not checked - assumed to be a variable name
145
+ # if not, it will simply never be found in the file, so ok
146
+ # but no warning for the user - add check?
147
+
148
+ # add to new list of checked and expanded lists
149
+ for luenum in luelist:
150
+ words[1] = luenum
151
+ lablist.append(list(words))
152
+ return lablist
153
+
154
+
155
+ def tuple_match(findme, hay):
156
+ """Part of partial ordered matching.
157
+ See http://stackoverflow.com/a/4559604
158
+ """
159
+ return len(findme) == len(hay) and all(
160
+ i is None or j is None or i == j for i, j in zip(findme, hay)
161
+ )
162
+
163
+
164
+ def tuple_combine(findme, hay):
165
+ """Part of partial ordered matching.
166
+ See http://stackoverflow.com/a/4559604
167
+ """
168
+ return tuple(i is None and j or i for i, j in zip(findme, hay))
169
+
170
+
171
+ def tuple_search(findme, haystack):
172
+ """Partial ordered matching with 'None' as wildcard
173
+ See http://stackoverflow.com/a/4559604
174
+ """
175
+ return [
176
+ (index, tuple_combine(findme, hay))
177
+ for index, hay in enumerate(haystack)
178
+ if tuple_match(findme, hay)
179
+ ]
@@ -28,12 +28,6 @@ from numpy import int64, ndarray
28
28
  from pandas.core.frame import DataFrame
29
29
  from pandas.core.indexes.base import Index
30
30
  from pandas.tseries.frequencies import to_offset
31
-
32
- try:
33
- from pydantic import validate_call
34
- except ImportError:
35
- from pydantic import validate_arguments as validate_call
36
-
37
31
  from scipy.stats.distributions import lognorm, norm
38
32
  from tabulate import simple_separated_format
39
33
  from tabulate import tabulate as tb
@@ -43,6 +37,11 @@ from .readers.plotgen import plotgen_extract as plotgen
43
37
  from .readers.wdm import wdm_extract as wdm
44
38
  from .utils import pandas_offset_by_version
45
39
 
40
+ try:
41
+ from pydantic import validate_call
42
+ except ImportError:
43
+ from pydantic import validate_arguments as validate_call
44
+
46
45
  # This is here so that linters don't remove the pint_pandas import which is
47
46
  # needed to use pint in pandas
48
47
  _ = pint_pandas.version("pint")
@@ -723,7 +722,7 @@ def copy_doc(source: Callable) -> Callable:
723
722
 
724
723
  def wrapper_copy_doc(func: Callable) -> Callable:
725
724
  if source.__doc__:
726
- func.__doc__ = source.__doc__ # noqa: WPS125
725
+ func.__doc__ = source.__doc__
727
726
 
728
727
  return func
729
728
 
@@ -1027,7 +1026,6 @@ def make_list(
1027
1026
  )
1028
1027
 
1029
1028
  # At this point 'strorlist' variable should be a list or tuple.
1030
-
1031
1029
  if n is None:
1032
1030
  n = len(strorlist)
1033
1031
 
@@ -1478,7 +1476,9 @@ def common_kwds(
1478
1476
 
1479
1477
  ntsd = _date_slice(ntsd, start_date=start_date, end_date=end_date, por=por)
1480
1478
 
1481
- if ntsd.index.inferred_type == "datetime64":
1479
+ if (
1480
+ not ntsd.index.name or "Datetime" not in ntsd.index.name
1481
+ ) and ntsd.index.inferred_type == "datetime64":
1482
1482
  ntsd.index.name = "Datetime"
1483
1483
 
1484
1484
  if dropna in ("any", "all"):
@@ -1495,6 +1495,8 @@ def common_kwds(
1495
1495
  return ntsd.resample(groupby)
1496
1496
 
1497
1497
  ntsd[ntsd.isna()] = np.nan
1498
+ ntsd.columns = [i.strip() for i in ntsd.columns]
1499
+ ntsd.index.name = ntsd.index.name.strip()
1498
1500
  return ntsd
1499
1501
 
1500
1502
 
@@ -1696,9 +1698,9 @@ def asbestfreq(data: DataFrame, force_freq: Optional[str] = None) -> DataFrame:
1696
1698
  2. If data.index.freq is not None, just return.
1697
1699
  3. If data.index.inferred_freq is set use .asfreq.
1698
1700
  4. Use pd.infer_freq - fails if any missing
1699
- 5. Use .is_* functions to establish YE, YS, Y-*, YS-*, Q, QS, M, MS
1700
- 6. Use minimum interval to establish the fixed time periods up to
1701
+ 5. Use minimum interval to establish the fixed time periods up to
1701
1702
  weekly
1703
+ 6. Use .is_* functions to establish YE, YS, Y-*, YS-*, Q, QS, M, MS
1702
1704
  7. Gives up returning None for PANDAS offset string
1703
1705
 
1704
1706
  Parameters
@@ -1737,32 +1739,6 @@ def asbestfreq(data: DataFrame, force_freq: Optional[str] = None) -> DataFrame:
1737
1739
  if infer_freq:
1738
1740
  return _replace_nan_with_na(data, freq=infer_freq)
1739
1741
 
1740
- # At this point pd.infer_freq failed probably because of missing values.
1741
- # The following algorithm would not capture things like BQ, BQS
1742
- # ...etc.
1743
- if np.all(data.index.is_year_end):
1744
- infer_freq = pandas_offset_by_version("YE")
1745
- elif np.all(data.index.is_year_start):
1746
- infer_freq = pandas_offset_by_version("YS")
1747
- elif np.all(data.index.is_quarter_end):
1748
- infer_freq = pandas_offset_by_version("QE")
1749
- elif np.all(data.index.is_quarter_start):
1750
- infer_freq = pandas_offset_by_version("QS")
1751
- elif np.all(data.index.is_month_end):
1752
- if np.all(data.index.month == data.index[0].month):
1753
- # Actually yearly with different ends
1754
- infer_freq = f"YE-{_ANNUALS[data.index[0].month]}"
1755
- else:
1756
- infer_freq = "ME"
1757
- elif np.all(data.index.is_month_start):
1758
- if np.all(data.index.month == data.index[0].month):
1759
- # Actually yearly with different start
1760
- infer_freq = f"YE-{_ANNUALS[data.index[0].month - 1]}"
1761
- else:
1762
- infer_freq = "MS"
1763
- if infer_freq:
1764
- return _replace_nan_with_na(data, freq=infer_freq)
1765
-
1766
1742
  data.index = data.index.astype("datetime64[ns]")
1767
1743
  ndiff = (
1768
1744
  data.index.astype("int64").values[1:] - data.index.astype("int64").values[:-1]
@@ -1810,8 +1786,33 @@ def asbestfreq(data: DataFrame, force_freq: Optional[str] = None) -> DataFrame:
1810
1786
  infer_freq = f"{infer_freq}-{_WEEKLIES[data.index[0].dayofweek]}"
1811
1787
  else:
1812
1788
  infer_freq = "D"
1789
+ if infer_freq:
1790
+ return _replace_nan_with_na(data, freq=infer_freq)
1791
+
1792
+ # At this point pd.infer_freq failed probably because of missing values.
1793
+ # The following algorithm would not capture things like BQ, BQS
1794
+ # ...etc.
1795
+ if np.all(data.index.is_month_start):
1796
+ if np.all(data.index.month == data.index[0].month):
1797
+ # Actually yearly with different start
1798
+ infer_freq = f"YE-{_ANNUALS[data.index[0].month - 1]}"
1799
+ else:
1800
+ infer_freq = "MS"
1801
+ elif np.all(data.index.is_month_end):
1802
+ if np.all(data.index.month == data.index[0].month):
1803
+ # Actually yearly with different ends
1804
+ infer_freq = f"YE-{_ANNUALS[data.index[0].month]}"
1805
+ else:
1806
+ infer_freq = "ME"
1807
+ elif np.all(data.index.is_quarter_end):
1808
+ infer_freq = pandas_offset_by_version("QE")
1809
+ elif np.all(data.index.is_quarter_start):
1810
+ infer_freq = pandas_offset_by_version("QS")
1811
+ elif np.all(data.index.is_year_end):
1812
+ infer_freq = pandas_offset_by_version("YE")
1813
+ elif np.all(data.index.is_year_start):
1814
+ infer_freq = pandas_offset_by_version("YS")
1813
1815
 
1814
- data.index = data.index.astype("datetime64[ns]")
1815
1816
  return _replace_nan_with_na(data, freq=infer_freq)
1816
1817
 
1817
1818
 
@@ -1987,7 +1988,7 @@ def _printiso(
1987
1988
  headers: str = "keys",
1988
1989
  tablefmt: Optional[str] = "csv",
1989
1990
  ) -> None:
1990
- """Separate this function so can use in tests."""
1991
+ """Print data. If time series data, print in ISO format."""
1991
1992
  showindex = {"always": True, "never": False, True: True, False: False}[showindex]
1992
1993
 
1993
1994
  if isinstance(tsd, (pd.DataFrame, pd.Series)):
@@ -1997,11 +1998,11 @@ def _printiso(
1997
1998
  if tsd.columns.empty:
1998
1999
  tsd = pd.DataFrame(index=tsd.index)
1999
2000
 
2000
- if not tsd.index.name:
2001
+ if not tsd.index.name or "Datetime" not in tsd.index.name:
2001
2002
  tsd.index.name = "UniqueID"
2002
2003
 
2003
- if isinstance(tsd.index, (pd.DatetimeIndex, pd.PeriodIndex)):
2004
- tsd.index.name = "Datetime"
2004
+ if isinstance(tsd.index, (pd.DatetimeIndex, pd.PeriodIndex)):
2005
+ tsd.index.name = "Datetime"
2005
2006
 
2006
2007
  elif isinstance(tsd, (int, float, tuple, np.ndarray)):
2007
2008
  tablefmt = None
@@ -2020,7 +2021,6 @@ def _printiso(
2020
2021
  sep=sep,
2021
2022
  index=showindex,
2022
2023
  )
2023
-
2024
2024
  return
2025
2025
  except OSError:
2026
2026
  return
@@ -2029,6 +2029,7 @@ def _printiso(
2029
2029
 
2030
2030
  if tablefmt is None:
2031
2031
  print(str(list(tsd))[1:-1])
2032
+ return
2032
2033
 
2033
2034
  if ntablefmt is None:
2034
2035
  all_table = tb(
@@ -2048,7 +2049,7 @@ def _printiso(
2048
2049
  )
2049
2050
 
2050
2051
  if tablefmt in ("csv_nos", "tsv_nos"):
2051
- print(all_table.replace(" ", ""))
2052
+ print(re.sub(r" *, *", ",", all_table))
2052
2053
  else:
2053
2054
  print(all_table)
2054
2055
 
@@ -2137,7 +2138,6 @@ def memory_optimize(tsd: DataFrame) -> DataFrame:
2137
2138
  # TypeError: Not datetime like index
2138
2139
  # ValueError: Less than three rows
2139
2140
  tsd.index.freq = pd.infer_freq(tsd.index)
2140
-
2141
2141
  return tsd
2142
2142
 
2143
2143
 
@@ -2559,20 +2559,25 @@ def read_iso_ts(
2559
2559
 
2560
2560
  result = pd.DataFrame()
2561
2561
 
2562
- for lres in lresult_list:
2562
+ for counter, lres in enumerate(lresult_list):
2563
+ if counter < 2:
2564
+ lcounter = "_r"
2565
+ else:
2566
+ lcounter = f"_r{counter}"
2563
2567
  if len(offset_set) < 2:
2564
- result = result.join(lres, how="outer", rsuffix="_r")
2568
+ result = result.join(lres, how="outer", rsuffix=f"{lcounter}")
2565
2569
  else:
2566
2570
  result = result.join(
2567
- lres.asfreq(moffset - epoch), how="outer", rsuffix="_r"
2571
+ lres.asfreq(moffset - epoch), how="outer", rsuffix=f"{lcounter}"
2568
2572
  )
2569
2573
  else:
2570
2574
  result = lresult_list[0]
2571
2575
 
2572
2576
  # Assign names to the index and columns.
2573
-
2574
2577
  if names is not None:
2575
- result.index.name = names.pop(0)
2578
+ possible_index_name = names.pop(0)
2579
+ if not result.index.name:
2580
+ result.index.name = possible_index_name
2576
2581
  result.columns = names
2577
2582
 
2578
2583
  result.sort_index(inplace=True)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: toolbox_utils
3
- Version: 5.2.3
3
+ Version: 5.2.5
4
4
  Summary: Python toolbox for common utilities
5
5
  Author-email: Tim Cera <tim@cerazone.net>
6
6
  License: BSD-3-Clause
@@ -33,6 +33,7 @@ src/toolbox_utils.egg-info/top_level.txt
33
33
  src/toolbox_utils/readers/__init__.py
34
34
  src/toolbox_utils/readers/hbn.py
35
35
  src/toolbox_utils/readers/plotgen.py
36
+ src/toolbox_utils/readers/utils.py
36
37
  src/toolbox_utils/readers/wdm.py
37
38
  tests/data.wdm
38
39
  tests/data_bi_daily.csv
@@ -34,7 +34,11 @@ def create_test_data(start, periods, freq, columns=["value"]):
34
34
  @pytest.mark.parametrize(
35
35
  "test_input, expected",
36
36
  [
37
- # Happy path tests
37
+ pytest.param(
38
+ create_test_data("2010-01-01", 16, pandas_offset_by_version("h")),
39
+ "(H|h)",
40
+ id="hourly_freq_on_jan_01",
41
+ ),
38
42
  pytest.param(create_test_data("2021-01-01", 10, "D"), "D", id="daily_freq"),
39
43
  pytest.param(
40
44
  create_test_data("2021-01-01", 10, pandas_offset_by_version("ME")),
@@ -46,7 +50,6 @@ def create_test_data(start, periods, freq, columns=["value"]):
46
50
  "(YE-DEC|A-DEC)",
47
51
  id="annual_freq",
48
52
  ),
49
- # Edge cases
50
53
  pytest.param(
51
54
  create_test_data("2021-01-01", 10, f"5{pandas_offset_by_version('h')}"),
52
55
  "(5h|5H)",
@@ -57,7 +60,6 @@ def create_test_data(start, periods, freq, columns=["value"]):
57
60
  "(15min|15T)",
58
61
  id="15_minute_freq",
59
62
  ),
60
- # Error cases
61
63
  pytest.param(
62
64
  create_test_data("2021-01-01", 10, "D").iloc[::2], "2D", id="2_day"
63
65
  ),
@@ -1 +0,0 @@
1
- 5.2.3
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes