hydrobot 0.7.2__tar.gz → 0.7.4__tar.gz

{hydrobot-0.7.2 → hydrobot-0.7.4}/HISTORY.rst

@@ -146,3 +146,13 @@ History
 ----------------------------------
 
 * Accurately representing the inaccurate recorder totals.
+
+0.7.3 (2024-09-19)
+----------------------------------
+
+* Fixing the defusedxml dependency version
+
+0.7.4 (2024-09-19)
+----------------------------------
+
+* Making manual tip filter work better

{hydrobot-0.7.2 → hydrobot-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hydrobot
-Version: 0.7.2
+Version: 0.7.4
 Summary: A suite of processing tools for Hilltop hydrological data.
 Author-email: Nic Mostert <nicolas.mostert@horizons.govt.nz>, Sam Irvine <sam.irvine@horizons.govt.nz>
 License: GNU General Public License v3
@@ -21,7 +21,7 @@ License-File: AUTHORS.rst
 Requires-Dist: hilltop-py>=2.3.1
 Requires-Dist: data-annalist>=0.4.1
 Requires-Dist: matplotlib>=3.8.0
-Requires-Dist: defusedxml>=0.7.2
+Requires-Dist: defusedxml>=0.7.1
 Requires-Dist: plotly>=5.20.0
 Requires-Dist: streamlit>=1.33.0
 Requires-Dist: pyyaml>=6.0.1
@@ -96,7 +96,7 @@ some external tools are still required to do a full processing. Importantly,
 the hybrid workflow relies on some R scripts to obtain check data from sources
 other than Hilltop. Further processing using Hilltop manager is also supported.
 
-NOTE: Hydrobot 0.7.2 supports only Water Temperature processing at the moment,
+NOTE: Hydrobot 0.7.4 supports only Water Temperature processing at the moment,
 but more measurements will be supported in patches as the processing
 progresses.
 
@@ -107,16 +107,16 @@ Initial Setup (Repeat for each release)
 
 #. In your favourite shell (if you don't know what that is, use powershell -
    it's already installed on windows), create a new virtual environment using
-   this python interpreter and name it "hydrobot0.7.2". It's important that
+   this python interpreter and name it "hydrobot0.7.4". It's important that
    this is stored somewhere locally. For example, it could be stored in a
    "Hydrobot" folder in the C: drive, which would need the command::
 
-    python -m venv C:/Hydrobot/hydrobot0.7.2/
+    python -m venv C:/Hydrobot/hydrobot0.7.4/
 
 #. Activate this virtual environment. In powershell this should be something
    like::
 
-    C:/Hydrobot/hydrobot0.7.2/Scripts/Activate.ps1
+    C:/Hydrobot/hydrobot0.7.4/Scripts/Activate.ps1
 
 #. With your venv active, install the latest version of Hydrobot using pip::
 
@@ -158,7 +158,7 @@ Processing Steps
    instructions) and activate it. To activate, in your shell type the location
    of the "Activate.ps1" script in the venv/Scripts folder, e.g.::
 
-    C:/Hydrobot/hydrobot0.7.2/Scripts/Activate.ps1
+    C:/Hydrobot/hydrobot0.7.4/Scripts/Activate.ps1
 
    You can ensure it is active by typing `gcm python` and confirm that your
    python interpreter (under "Source") is running from your venv folder.

{hydrobot-0.7.2 → hydrobot-0.7.4}/README.rst

@@ -49,7 +49,7 @@ some external tools are still required to do a full processing. Importantly,
 the hybrid workflow relies on some R scripts to obtain check data from sources
 other than Hilltop. Further processing using Hilltop manager is also supported.
 
-NOTE: Hydrobot 0.7.2 supports only Water Temperature processing at the moment,
+NOTE: Hydrobot 0.7.4 supports only Water Temperature processing at the moment,
 but more measurements will be supported in patches as the processing
 progresses.
 
@@ -60,16 +60,16 @@ Initial Setup (Repeat for each release)
 
 #. In your favourite shell (if you don't know what that is, use powershell -
    it's already installed on windows), create a new virtual environment using
-   this python interpreter and name it "hydrobot0.7.2". It's important that
+   this python interpreter and name it "hydrobot0.7.4". It's important that
    this is stored somewhere locally. For example, it could be stored in a
    "Hydrobot" folder in the C: drive, which would need the command::
 
-    python -m venv C:/Hydrobot/hydrobot0.7.2/
+    python -m venv C:/Hydrobot/hydrobot0.7.4/
 
 #. Activate this virtual environment. In powershell this should be something
    like::
 
-    C:/Hydrobot/hydrobot0.7.2/Scripts/Activate.ps1
+    C:/Hydrobot/hydrobot0.7.4/Scripts/Activate.ps1
 
 #. With your venv active, install the latest version of Hydrobot using pip::
 
@@ -111,7 +111,7 @@ Processing Steps
    instructions) and activate it. To activate, in your shell type the location
    of the "Activate.ps1" script in the venv/Scripts folder, e.g.::
 
-    C:/Hydrobot/hydrobot0.7.2/Scripts/Activate.ps1
+    C:/Hydrobot/hydrobot0.7.4/Scripts/Activate.ps1
 
    You can ensure it is active by typing `gcm python` and confirm that your
    python interpreter (under "Source") is running from your venv folder.

{hydrobot-0.7.2 → hydrobot-0.7.4}/hydrobot/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Nic Mostert"""
 __email__ = "nicolas.mostert@horizons.govt.nz"
-__version__ = "0.7.2"
+__version__ = "0.7.4"

{hydrobot-0.7.2 → hydrobot-0.7.4}/hydrobot/measurement_specific_functions/rainfall.py

@@ -368,7 +368,7 @@ def manual_tip_filter(
         Issue to report, if any
     """
     std_series = std_series.copy()
-    threshold = pd.Timedelta("00:00:20")
+    mode = std_series.astype(float).replace(0, np.nan).mode().item()
 
     if not isinstance(std_series.index, pd.DatetimeIndex):
         warnings.warn(
@@ -381,7 +381,7 @@ def manual_tip_filter(
         (std_series.index > arrival_time) & (std_series.index < departure_time)
     ]
 
-    if len(inspection_data) < manual_tips:
+    if inspection_data.sum() < (manual_tips * mode):
         # Manual tips presumed to be in inspection mode, no further action
         return std_series, None
     elif manual_tips == 0:
@@ -389,10 +389,9 @@ def manual_tip_filter(
         return std_series, None
     else:
         if weather in ["Fine", "Overcast"]:
-            if abs(manual_tips - len(inspection_data)) <= 1:
+            if abs(manual_tips * mode - inspection_data.sum()) <= mode:
                 # Off by 1 is probably just a typo, delete it all
                 std_series[inspection_data.index] = 0
-
                 return std_series, None
             else:
                 issue = {
@@ -410,12 +409,11 @@ def manual_tip_filter(
                 # All this does is find the first element of the shortest period
                 first_manual_tip_index = pd.DataFrame(differences).idxmin().iloc[0]
 
-                if differences[first_manual_tip_index] < threshold:
-                    # Sufficiently intense
-                    inspection_data[
-                        first_manual_tip_index : first_manual_tip_index + manual_tips
-                    ] = 0
-                    std_series[inspection_data.index] = inspection_data
+                # Sufficiently intense
+                inspection_data[
+                    first_manual_tip_index : first_manual_tip_index + manual_tips
+                ] = 0
+                std_series[inspection_data.index] = inspection_data
 
                 return std_series, issue
         else:
@@ -437,11 +435,10 @@ def manual_tip_filter(
             # All this does is find the first element of the shortest period
             first_manual_tip_index = pd.DataFrame(differences).idxmin().iloc[0]
 
-            if differences[first_manual_tip_index] < threshold or manual_tips > 30:
-                # Sufficiently intense or calibration
-                inspection_data[
-                    first_manual_tip_index : first_manual_tip_index + manual_tips
-                ] = 0
-                std_series[inspection_data.index] = inspection_data
+            # Sufficiently intense or calibration
+            inspection_data[
+                first_manual_tip_index : first_manual_tip_index + manual_tips
+            ] = 0
+            std_series[inspection_data.index] = inspection_data
 
             return std_series, issue

{hydrobot-0.7.2 → hydrobot-0.7.4}/hydrobot/rf_processor.py

@@ -434,6 +434,7 @@ class RFProcessor(Processor):
                 check["start_time"],
                 check["end_time"],
                 check["primary_manual_tips"],
+                check["weather"],
             )
             if issue is not None:
                 self.report_processing_issue(**issue)

{hydrobot-0.7.2 → hydrobot-0.7.4}/hydrobot.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hydrobot
-Version: 0.7.2
+Version: 0.7.4
 Summary: A suite of processing tools for Hilltop hydrological data.
 Author-email: Nic Mostert <nicolas.mostert@horizons.govt.nz>, Sam Irvine <sam.irvine@horizons.govt.nz>
 License: GNU General Public License v3
@@ -21,7 +21,7 @@ License-File: AUTHORS.rst
 Requires-Dist: hilltop-py>=2.3.1
 Requires-Dist: data-annalist>=0.4.1
 Requires-Dist: matplotlib>=3.8.0
-Requires-Dist: defusedxml>=0.7.2
+Requires-Dist: defusedxml>=0.7.1
 Requires-Dist: plotly>=5.20.0
 Requires-Dist: streamlit>=1.33.0
 Requires-Dist: pyyaml>=6.0.1
@@ -96,7 +96,7 @@ some external tools are still required to do a full processing. Importantly,
 the hybrid workflow relies on some R scripts to obtain check data from sources
 other than Hilltop. Further processing using Hilltop manager is also supported.
 
-NOTE: Hydrobot 0.7.2 supports only Water Temperature processing at the moment,
+NOTE: Hydrobot 0.7.4 supports only Water Temperature processing at the moment,
 but more measurements will be supported in patches as the processing
 progresses.
 
@@ -107,16 +107,16 @@ Initial Setup (Repeat for each release)
 
 #. In your favourite shell (if you don't know what that is, use powershell -
    it's already installed on windows), create a new virtual environment using
-   this python interpreter and name it "hydrobot0.7.2". It's important that
+   this python interpreter and name it "hydrobot0.7.4". It's important that
    this is stored somewhere locally. For example, it could be stored in a
    "Hydrobot" folder in the C: drive, which would need the command::
 
-    python -m venv C:/Hydrobot/hydrobot0.7.2/
+    python -m venv C:/Hydrobot/hydrobot0.7.4/
 
 #. Activate this virtual environment. In powershell this should be something
    like::
 
-    C:/Hydrobot/hydrobot0.7.2/Scripts/Activate.ps1
+    C:/Hydrobot/hydrobot0.7.4/Scripts/Activate.ps1
 
 #. With your venv active, install the latest version of Hydrobot using pip::
 
@@ -158,7 +158,7 @@ Processing Steps
    instructions) and activate it. To activate, in your shell type the location
    of the "Activate.ps1" script in the venv/Scripts folder, e.g.::
 
-    C:/Hydrobot/hydrobot0.7.2/Scripts/Activate.ps1
+    C:/Hydrobot/hydrobot0.7.4/Scripts/Activate.ps1
 
    You can ensure it is active by typing `gcm python` and confirm that your
    python interpreter (under "Source") is running from your venv folder.

{hydrobot-0.7.2 → hydrobot-0.7.4}/hydrobot.egg-info/requires.txt

@@ -1,7 +1,7 @@
 hilltop-py>=2.3.1
 data-annalist>=0.4.1
 matplotlib>=3.8.0
-defusedxml>=0.7.2
+defusedxml>=0.7.1
 plotly>=5.20.0
 streamlit>=1.33.0
 pyyaml>=6.0.1

{hydrobot-0.7.2 → hydrobot-0.7.4}/prototypes/rainfall/rain_config.yaml

@@ -1,10 +1,15 @@
+site: "Tiraumea at Ngaturi"
+analyst_name: "Sam Irvine"
+from_date: "2023-03-16 08:54"
+to_date: "2024-09-19 14:00"
+
 # Hilltop Server Configuration
 base_url: "http://hilltopdev.horizons.govt.nz/"
 standard_hts_filename: "RawLoggerNet.hts"
 check_hts_filename: "HydrobotCheckData.hts"
 
 # Data Source Configuration
-site: "Ahuahu at Te Tuhi Junction"
+
 standard_measurement_name: "Rainfall [SCADA Rainfall]"
 check_measurement_name: "Check Gauge Total [Rainfall]"
 standard_backup: "Rainfall [SCADA Rainfall]"
@@ -14,15 +19,14 @@ check_backup: "Check Gauge Total [Rainfall]"
 export_file_name: "processed_rf.xml"
 
 # Processing Configuration
-from_date: "2023-06-01 10:42"
-to_date: "2024-03-02 10:00"
+
 defaults:
   high_clip: 2.1
   low_clip: -0.1
   max_qc: 600
 
 # Annalist Configuration
-analyst_name: "default"
+
 logfile: "hydrobot_logs.csv"
 format:
   file: "%(asctime)s, %(analyst_name)s, %(function_name)s, %(site)s, %(measurement)s, %(from_date)s, %(to_date)s, %(message)s"

{hydrobot-0.7.2 → hydrobot-0.7.4}/prototypes/rainfall/rain_script.py

@@ -13,6 +13,7 @@ import pandas as pd
 import sqlalchemy as db
 from sqlalchemy.engine import URL
 
+from hydrobot import utils
 from hydrobot.filters import trim_series
 from hydrobot.rf_processor import RFProcessor
 
@@ -87,7 +88,7 @@ check_data = pd.DataFrame(
     rainfall_checks[["arrival_time", "flask", "notes", "primary_total"]].copy()
 )
 
-check_data["Recorder Total"] = check_data.loc[:, "primary_total"]
+check_data["Recorder Total"] = check_data.loc[:, "primary_total"] * 1000
 check_data["Recorder Time"] = check_data.loc[:, "arrival_time"]
 check_data = check_data.set_index("arrival_time")
 check_data.index = pd.to_datetime(check_data.index)
@@ -114,7 +115,7 @@ check_data = check_data[
     ]
 ]
 
-data.check_data = check_data
+data.check_data = utils.series_rounder(check_data)
 
 all_checks = rainfall_checks.rename(
     columns={"primary_total": "Logger", "flask": "Value"}

{hydrobot-0.7.2 → hydrobot-0.7.4}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "hydrobot"
 description = "A suite of processing tools for Hilltop hydrological data."
-version = "0.7.2"
+version = "0.7.4"
 authors = [
     { name = "Nic Mostert", email = "nicolas.mostert@horizons.govt.nz" },
     { name = "Sam Irvine", email = "sam.irvine@horizons.govt.nz" }
@@ -15,7 +15,7 @@ dependencies = [
     "hilltop-py>=2.3.1",
     "data-annalist>=0.4.1",
     "matplotlib>=3.8.0",
-    "defusedxml>=0.7.2",
+    "defusedxml>=0.7.1",
     "plotly>=5.20.0",
     "streamlit>=1.33.0",
     "pyyaml>=6.0.1",
@@ -106,7 +106,7 @@ convention = "numpy"
 "prototypes" = ["D"]
 
 [tool.bumpversion]
-current_version = "0.7.2"
+current_version = "0.7.4"
 commit = true
 tag = true
 tag_name = "{new_version}"