toolbox-utils 5.2.4__tar.gz → 5.2.5__tar.gz

{toolbox_utils-5.2.4 → toolbox_utils-5.2.5}/.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
 
     - repo: https://github.com/astral-sh/ruff-pre-commit
       # Ruff version.
-      rev: v0.15.2
+      rev: v0.15.8
       hooks:
           # Run the linter.
           - id: ruff
@@ -40,7 +40,7 @@ repos:
             types_or: [python, pyi, jupyter]
 
     - repo: https://github.com/pycqa/isort
-      rev: 8.0.0
+      rev: 8.0.1
       hooks:
           - id: isort
             name: isort (python)
@@ -52,7 +52,7 @@ repos:
             types: [pyi]
 
     - repo: https://github.com/pappasam/toml-sort
-      rev: v0.24.3
+      rev: v0.24.4
       hooks:
           - id: toml-sort-fix
             args: [--in-place, --spaces-indent-inline-array, '4']
@@ -75,7 +75,7 @@ repos:
             args: [-s, bash]
 
     - repo: https://github.com/lovesegfault/beautysh
-      rev: v6.4.2
+      rev: v6.4.3
       hooks:
           - id: beautysh
             args: [--indent-size, '4']
@@ -91,7 +91,7 @@ repos:
           - id: pyupgrade
 
     - repo: https://github.com/commitizen-tools/commitizen
-      rev: v4.13.8
+      rev: v4.13.9
       hooks:
           - id: commitizen
             stages: [commit-msg]

{toolbox_utils-5.2.4 → toolbox_utils-5.2.5}/CHANGELOG.md

@@ -1,3 +1,11 @@
+## v5.2.5 (2026-06-07)
+
+### Fix
+
+- fix and refactor hbn reader to make hdf reader easier to implement
+- fix the csv_nos and tsc_nos to be more robust and not remove spaces in the middle of values
+- remove spaces either side of column names and index
+
 ## v5.2.4 (2026-03-27)
 
 ### Fix

{toolbox_utils-5.2.4 → toolbox_utils-5.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolbox_utils
-Version: 5.2.4
+Version: 5.2.5
 Summary: Python toolbox for common utilities
 Author-email: Tim Cera <tim@cerazone.net>
 License: BSD-3-Clause

toolbox_utils-5.2.5/VERSION

@@ -0,0 +1 @@
+5.2.5

{toolbox_utils-5.2.4 → toolbox_utils-5.2.5}/pyproject.toml

@@ -61,7 +61,7 @@ github = "https://github.com/timcera/toolbox_utils"
 name = "cz_conventional_commits"
 tag_format = "v$version"
 update_changelog_on_bump = true
-version = "5.2.4"
+version = "5.2.5"
 version_files = ["VERSION"]
 
 [tool.isort]

{toolbox_utils-5.2.4 → toolbox_utils-5.2.5}/src/toolbox_utils/readers/hbn.py

@@ -3,21 +3,24 @@
 import datetime
 import struct
 import sys
-
-try:
-    from typing import Literal
-except ImportError:
-    from typing import Literal
+from typing import Literal
 
 import pandas as pd
 
 from .. import tsutils
+from ..utils import pandas_offset_by_version
+from . import utils
 
 code2intervalmap = {5: "yearly", 4: "monthly", 3: "daily", 2: "bivl"}
 
 interval2codemap = {"yearly": 5, "monthly": 4, "daily": 3, "bivl": 2}
 
-code2freqmap = {5: "A", 4: "M", 3: "D", 2: None}
+code2freqmap = {
+    5: pandas_offset_by_version("YE"),
+    4: pandas_offset_by_version("ME"),
+    3: "D",
+    2: None,
+}
 
 
 _LOCAL_DOCSTRINGS = {
@@ -27,168 +30,20 @@ _LOCAL_DOCSTRINGS = {
 }
 
 
-def tuple_match(findme, hay):
-    """Part of partial ordered matching.
-    See http://stackoverflow.com/a/4559604
-    """
-    return len(findme) == len(hay) and all(
-        i is None or j is None or i == j for i, j in zip(findme, hay)
-    )
-
-
-def tuple_combine(findme, hay):
-    """Part of partial ordered matching.
-    See http://stackoverflow.com/a/4559604
-    """
-    return tuple(i is None and j or i for i, j in zip(findme, hay))
-
-
-def tuple_search(findme, haystack):
-    """Partial ordered matching with 'None' as wildcard
-    See http://stackoverflow.com/a/4559604
-    """
-    return [
-        (index, tuple_combine(findme, hay))
-        for index, hay in enumerate(haystack)
-        if tuple_match(findme, hay)
-    ]
-
-
 def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
     """Underlying function to read from the binary file.  Used by
     'extract', 'catalog'.
     """
-    if labels is None:
-        labels = [",,,"]
-    testem = {
-        "PERLND": [
-            "ATEMP",
-            "SNOW",
-            "PWATER",
-            "SEDMNT",
-            "PSTEMP",
-            "PWTGAS",
-            "PQUAL",
-            "MSTLAY",
-            "PEST",
-            "NITR",
-            "PHOS",
-            "TRACER",
-            "",
-        ],
-        "IMPLND": ["ATEMP", "SNOW", "IWATER", "SOLIDS", "IWTGAS", "IQUAL", ""],
-        "RCHRES": [
-            "HYDR",
-            "CONS",
-            "HTRCH",
-            "SEDTRN",
-            "GQUAL",
-            "OXRX",
-            "NUTRX",
-            "PLANK",
-            "PHCARB",
-            "INFLOW",
-            "OFLOW",
-            "ROFLOW",
-            "",
-        ],
-        "BMPRAC": [""],
-        "": [""],
-    }
-
-    collect_dict = {}
-    lablist = []
-
     # Normalize interval code
     try:
         intervalcode = interval2codemap[interval.lower()]
     except AttributeError:
         intervalcode = None
 
-    # convert label tuples to lists
-    labels = list(labels)
-
-    # turn into a list of lists
-    nlabels = []
-    for label in labels:
-        if isinstance(label, str):
-            nlabels.append(label.split(","))
-        else:
-            nlabels.append(label)
-    labels = nlabels
-
-    # Check the list members for valid values
-    for label in labels:
-        if len(label) != 4:
-            raise ValueError(
-                tsutils.error_wrapper(
-                    f"""The label '{label}' has the wrong number of entries.
-                    """
-                )
-            )
-
-        # replace empty fields with None
-        # operation,lue_number,group,variable
-        words = [None if (i in ("", "None")) else i for i in label]
-
-        # first word must be a valid operation type or None
-        if words[0] is not None:
-            # force uppercase before comparison
-            words[0] = words[0].upper()
-            if words[0] not in testem:
-                raise ValueError(
-                    tsutils.error_wrapper(
-                        f"""Operation type must be one of 'PERLND', 'IMPLND',
-                        'RCHRES', or 'BMPRAC', or missing (to get all) instead
-                        of {words[0]}.
-                        """
-                    )
-                )
-
-        # second word must be integer 1-999 or None or range to parse
-        if words[1] is not None:
-            try:
-                words[1] = int(words[1])
-                luelist = [words[1]]
-            except ValueError:
-                luelist = tsutils.range_to_numlist(words[1])
-            for luenum in luelist:
-                if luenum < 1 or luenum > 999:
-                    raise ValueError(
-                        tsutils.error_wrapper(
-                            f"""The land use element must be an integer from
-                            1 to 999 inclusive, instead of {luenum}.
-                            """
-                        )
-                    )
-        else:
-            luelist = [None]
-
-        # third word must be a valid group name or None
-        if words[2] is not None:
-            words[2] = words[2].upper()
-            if words[2] not in testem[words[0]]:
-                raise ValueError(
-                    tsutils.error_wrapper(
-                        f"""The {words[0]} operation type only allows the
-                        variable groups: {testem[words[0]][:-1]}, instead you
-                        gave {words[2]}.
-                        """
-                    )
-                )
-
-        # fourth word is currently not checked - assumed to be a variable name
-        # if not, it will simply never be found in the file, so ok
-        # but no warning for the user - add check?
-
-        # add interval code as fifth word in list
-        words.append(intervalcode)
-
-        # add to new list of checked and expanded lists
-        for luenum in luelist:
-            words[1] = luenum
-            lablist.append(list(words))
+    lablist = utils.normalize_labels(labels)
+    lablist = [i + [intervalcode] for i in lablist]
 
+    collect_dict = {}
     # Now read through the binary file and collect the data matching the labels
     with open(binfilename, "rb") as binfp:
         labeltest = set()
@@ -200,7 +55,8 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
             # not a valid HSPF binary file
             raise ValueError(
                 tsutils.error_wrapper(
-                    f"""{binfilename} is not a valid HSPF binary output file
+                    f"""
+                    {binfilename} is not a valid HSPF binary output file
                     (.hbn),  The first byte must be FD hexadecimal, but it was
                     {magicbyte}.
                     """
@@ -276,10 +132,12 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
                 recpos += 4 * numvals
 
                 delta = datetime.timedelta(hours=0)
-                if hour == 24:
-                    hour = 0
+                if level == interval2codemap["bivl"]:
+                    delta = datetime.timedelta(hours=hour) + datetime.timedelta(
+                        minutes=minute
+                    )
 
-                ndate = datetime.datetime(year, month, day, hour, minute) + delta
+                ndate = datetime.datetime(year, month, day) + delta
 
                 #  Go through labels to see if these values need to be
                 #  collected
@@ -291,9 +149,8 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
                         vname.decode("ascii"),
                         level,
                     )
-
                     for lbl in lablist:
-                        res = tuple_search(tmpkey, [lbl])
+                        res = utils.tuple_search(tmpkey, [lbl])
                         if not res:
                             continue
                         labeltest.add(tuple(lbl))
@@ -322,8 +179,9 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
     if not collect_dict:
         raise ValueError(
             tsutils.error_wrapper(
-                f"""The label specifications below matched no records in the
-                binary file.
+                f"""
+                The label specifications below matched no records in the binary
+                file.
 
                 {lablist}
                 """
@@ -337,8 +195,9 @@ def _get_data(binfilename, interval="daily", labels=None, catalog_only=True):
             if tuple(lbl) not in labeltest:
                 sys.stderr.write(
                     tsutils.error_wrapper(
-                        f"""Warning: The label '{lbl}' matched no records in
-                        the binary file.
+                        f"""
+                        Warning: The label '{lbl}' matched no records in the
+                        binary file.
                         """
                     )
                 )
@@ -365,7 +224,8 @@ def hbn_extract(
     if interval not in ("bivl", "daily", "monthly", "yearly"):
         raise ValueError(
             tsutils.error_wrapper(
-                f"""The "interval" argument must be one of "bivl", "daily",
+                f"""
+                The "interval" argument must be one of "bivl", "daily",
                 "monthly", or "yearly".  You supplied "{interval}".
                 """
             )

toolbox_utils-5.2.5/src/toolbox_utils/readers/utils.py

@@ -0,0 +1,179 @@
+"""A collection of functions used by toolbox_utils, wdmtoolbox, ...etc."""
+
+from typing import List, Optional, Union
+
+import pint_pandas  # not used directly, but required to use pint in pandas
+
+from .. import tsutils
+
+# This is here so that linters don't remove the pint_pandas import which is
+# needed to use pint in pandas
+_ = pint_pandas.version("pint")
+
+
+def normalize_labels(labels: Optional[Union[str, List[str]]]) -> List[str]:
+    """
+    Process labels for the hbn function.
+
+    Parameters
+    ----------
+    labels
+        The labels to be processed.
+
+    Returns
+    -------
+    process_labels
+        A list of processed labels.
+    """
+    if labels is None:
+        labels = [",,,"]
+
+    testem = {
+        "PERLND": [
+            "ATEMP",
+            "SNOW",
+            "PWATER",
+            "SEDMNT",
+            "PSTEMP",
+            "PWTGAS",
+            "PQUAL",
+            "MSTLAY",
+            "PEST",
+            "NITR",
+            "PHOS",
+            "TRACER",
+            "",
+        ],
+        "IMPLND": ["ATEMP", "SNOW", "IWATER", "SOLIDS", "IWTGAS", "IQUAL", ""],
+        "RCHRES": [
+            "HYDR",
+            "CONS",
+            "HTRCH",
+            "SEDTRN",
+            "GQUAL",
+            "OXRX",
+            "NUTRX",
+            "PLANK",
+            "PHCARB",
+            "INFLOW",
+            "OFLOW",
+            "ROFLOW",
+            "",
+        ],
+        "BMPRAC": [""],
+        "": [""],
+    }
+
+    lablist = []
+
+    # convert label tuples to lists
+    labels = list(labels)
+
+    # turn into a list of lists
+    nlabels = []
+    for label in labels:
+        if isinstance(label, str):
+            nlabels.append(label.split(","))
+        else:
+            nlabels.append(label)
+    labels = nlabels
+
+    # Check the list members for valid values
+    for label in labels:
+        if len(label) != 4:
+            raise ValueError(
+                tsutils.error_wrapper(
+                    f"""
+                    The label '{label}' has the wrong number of entries.
+                    """
+                )
+            )
+
+        # replace empty fields with None
+        words = [None if i == "" else i for i in label]
+
+        # first word must be a valid operation type or None
+        if words[0] is not None:
+            # force uppercase before comparison
+            words[0] = words[0].upper()
+            if words[0] not in testem:
+                raise ValueError(
+                    tsutils.error_wrapper(
+                        f"""
+                        Operation type must be one of 'PERLND', 'IMPLND',
+                        'RCHRES', or 'BMPRAC', or missing (to get all) instead
+                        of {words[0]}.
+                        """
+                    )
+                )
+
+        # second word must be integer 1-999 or None or range to parse
+        if words[1] is not None:
+            try:
+                words[1] = int(words[1])
+                luelist = [words[1]]
+            except ValueError:
+                luelist = tsutils.range_to_numlist(words[1])
+            for luenum in luelist:
+                if luenum < 1 or luenum > 999:
+                    raise ValueError(
+                        tsutils.error_wrapper(
+                            f"""
+                            The land use element must be an integer from 1 to
+                            999 inclusive, instead of {luenum}.
+                            """
+                        )
+                    )
+        else:
+            luelist = [None]
+
+        # third word must be a valid group name or None
+        if words[2] is not None:
+            words[2] = words[2].upper()
+            if (words[0] is not None) and (words[2] not in testem[words[0]]):
+                raise ValueError(
+                    tsutils.error_wrapper(
+                        f"""
+                        The {words[0]} operation type only allows the variable
+                        groups: {testem[words[0]][:-1]},
+                        instead you gave {words[2]}.
+                        """
+                    )
+                )
+
+        # fourth word is currently not checked - assumed to be a variable name
+        # if not, it will simply never be found in the file, so ok
+        # but no warning for the user - add check?
+
+        # add to new list of checked and expanded lists
+        for luenum in luelist:
+            words[1] = luenum
+            lablist.append(list(words))
+    return lablist
+
+
+def tuple_match(findme, hay):
+    """Part of partial ordered matching.
+    See http://stackoverflow.com/a/4559604
+    """
+    return len(findme) == len(hay) and all(
+        i is None or j is None or i == j for i, j in zip(findme, hay)
+    )
+
+
+def tuple_combine(findme, hay):
+    """Part of partial ordered matching.
+    See http://stackoverflow.com/a/4559604
+    """
+    return tuple(i is None and j or i for i, j in zip(findme, hay))
+
+
+def tuple_search(findme, haystack):
+    """Partial ordered matching with 'None' as wildcard
+    See http://stackoverflow.com/a/4559604
+    """
+    return [
+        (index, tuple_combine(findme, hay))
+        for index, hay in enumerate(haystack)
+        if tuple_match(findme, hay)
+    ]

{toolbox_utils-5.2.4 → toolbox_utils-5.2.5}/src/toolbox_utils/tsutils.py

@@ -28,12 +28,6 @@ from numpy import int64, ndarray
 from pandas.core.frame import DataFrame
 from pandas.core.indexes.base import Index
 from pandas.tseries.frequencies import to_offset
-
-try:
-    from pydantic import validate_call
-except ImportError:
-    from pydantic import validate_arguments as validate_call
-
 from scipy.stats.distributions import lognorm, norm
 from tabulate import simple_separated_format
 from tabulate import tabulate as tb
@@ -43,6 +37,11 @@ from .readers.plotgen import plotgen_extract as plotgen
 from .readers.wdm import wdm_extract as wdm
 from .utils import pandas_offset_by_version
 
+try:
+    from pydantic import validate_call
+except ImportError:
+    from pydantic import validate_arguments as validate_call
+
 # This is here so that linters don't remove the pint_pandas import which is
 # needed to use pint in pandas
 _ = pint_pandas.version("pint")
@@ -723,7 +722,7 @@ def copy_doc(source: Callable) -> Callable:
 
     def wrapper_copy_doc(func: Callable) -> Callable:
         if source.__doc__:
-            func.__doc__ = source.__doc__  # noqa: WPS125
+            func.__doc__ = source.__doc__
 
         return func
 
@@ -1477,7 +1476,9 @@ def common_kwds(
 
     ntsd = _date_slice(ntsd, start_date=start_date, end_date=end_date, por=por)
 
-    if ntsd.index.inferred_type == "datetime64":
+    if (
+        not ntsd.index.name or "Datetime" not in ntsd.index.name
+    ) and ntsd.index.inferred_type == "datetime64":
         ntsd.index.name = "Datetime"
 
     if dropna in ("any", "all"):
@@ -1494,6 +1495,8 @@ def common_kwds(
         return ntsd.resample(groupby)
 
     ntsd[ntsd.isna()] = np.nan
+    ntsd.columns = [i.strip() for i in ntsd.columns]
+    ntsd.index.name = ntsd.index.name.strip()
     return ntsd
 
 
@@ -1985,7 +1988,7 @@ def _printiso(
     headers: str = "keys",
     tablefmt: Optional[str] = "csv",
 ) -> None:
-    """Separate this function so can use in tests."""
+    """Print data.  If time series data, print in ISO format."""
     showindex = {"always": True, "never": False, True: True, False: False}[showindex]
 
     if isinstance(tsd, (pd.DataFrame, pd.Series)):
@@ -1995,11 +1998,11 @@ def _printiso(
         if tsd.columns.empty:
             tsd = pd.DataFrame(index=tsd.index)
 
-        if not tsd.index.name:
+        if not tsd.index.name or "Datetime" not in tsd.index.name:
             tsd.index.name = "UniqueID"
 
-        if isinstance(tsd.index, (pd.DatetimeIndex, pd.PeriodIndex)):
-            tsd.index.name = "Datetime"
+            if isinstance(tsd.index, (pd.DatetimeIndex, pd.PeriodIndex)):
+                tsd.index.name = "Datetime"
 
     elif isinstance(tsd, (int, float, tuple, np.ndarray)):
         tablefmt = None
@@ -2018,7 +2021,6 @@ def _printiso(
                     sep=sep,
                     index=showindex,
                 )
-
                 return
             except OSError:
                 return
@@ -2027,6 +2029,7 @@ def _printiso(
 
     if tablefmt is None:
         print(str(list(tsd))[1:-1])
+        return
 
     if ntablefmt is None:
         all_table = tb(
@@ -2046,7 +2049,7 @@ def _printiso(
         )
 
     if tablefmt in ("csv_nos", "tsv_nos"):
-        print(all_table.replace(" ", ""))
+        print(re.sub(r" *, *", ",", all_table))
     else:
         print(all_table)
 
@@ -2135,7 +2138,6 @@ def memory_optimize(tsd: DataFrame) -> DataFrame:
         # TypeError: Not datetime like index
         # ValueError: Less than three rows
         tsd.index.freq = pd.infer_freq(tsd.index)
-
     return tsd
 
 
@@ -2572,9 +2574,10 @@ def read_iso_ts(
         result = lresult_list[0]
 
     # Assign names to the index and columns.
-
     if names is not None:
-        result.index.name = names.pop(0)
+        possible_index_name = names.pop(0)
+        if not result.index.name:
+            result.index.name = possible_index_name
         result.columns = names
 
     result.sort_index(inplace=True)

{toolbox_utils-5.2.4 → toolbox_utils-5.2.5}/src/toolbox_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolbox_utils
-Version: 5.2.4
+Version: 5.2.5
 Summary: Python toolbox for common utilities
 Author-email: Tim Cera <tim@cerazone.net>
 License: BSD-3-Clause

{toolbox_utils-5.2.4 → toolbox_utils-5.2.5}/src/toolbox_utils.egg-info/SOURCES.txt

@@ -33,6 +33,7 @@ src/toolbox_utils.egg-info/top_level.txt
 src/toolbox_utils/readers/__init__.py
 src/toolbox_utils/readers/hbn.py
 src/toolbox_utils/readers/plotgen.py
+src/toolbox_utils/readers/utils.py
 src/toolbox_utils/readers/wdm.py
 tests/data.wdm
 tests/data_bi_daily.csv

toolbox_utils-5.2.4/VERSION

@@ -1 +0,0 @@
-5.2.4