hydrobot 0.2.2__tar.gz → 0.3.0__tar.gz

{hydrobot-0.2.2 → hydrobot-0.3.0}/CONTRIBUTING.rst

@@ -141,7 +141,6 @@ Before you submit a pull request, check that it meets these guidelines:
 2. If the pull request adds functionality, the docs should be updated. Put
    your new functionality into a function with a docstring, and add the
    feature to the appropriate location in the documentation.
-3. The pull request should work for Python 3.5, 3.6, 3.7 and 3.8, and for PyPy. [*NOTE: This is a matter of trust for now, but I will figure out how to enforce this with "tox" at some point*]
 
 Tips
 ----
@@ -151,13 +150,61 @@ To run a subset of tests::
 $ pytest tests.test_hydrobot
 
 
-Deploying
----------
+Releasing to PyPI
+------------------
 
 A reminder for the maintainers on how to deploy.
-Make sure all your changes are committed (including an entry in HISTORY.rst).
-Then run::
 
-$ bump2version patch # possible: major / minor / patch
-$ git push
-$ git push --tags
+1. Make sure all your changes are committed (including an entry in HISTORY.rst, documentation, etc.).
+
+2. Then run `bump2version` to increment the release tags in the appropriate places. Consider running `bump2version --dry-run` to make sure there are no errors first::
+
+    $ bump2version --dry-run --verbose patch # Optional, just to test if it runs without errors
+    $ bump2version patch # For real this time. Possible values: major / minor / patch
+
+3. Install the local development version of the package (make sure you're in the package root directory where setup.py is). You should see the package install with the correct version number.::
+
+    $ pip install -e .
+
+4. Run the tests to see that they still work with this local install::
+
+    $ pytest
+
+5. Push the commit::
+
+    $ git push
+
+6. Push the tags to GitHub. (Note that we don't actually release on GitHub though. We want to keep the releases to PyPI so there's less ambiguity about how to install it.)::
+
+    $ git push --tags
+
+7. Do the release.
+
+    * If using the Makefile (i.e. you have `make` installed and can run `make help` without errors) you can simply run::
+
+        $ make release
+
+    * Otherwise, you would have to do the release manually.
+
+        a. Clean up all the artifact files::
+
+            $ rm -fr build/
+            $ rm -fr dist/
+            $ rm -fr .eggs/
+            $ find . -name '*.egg-info' -exec rm -fr {} +
+            $ find . -name '*.egg' -exec rm -f {} +
+            $ find . -name '*.pyc' -exec rm -f {} +
+            $ find . -name '*.pyo' -exec rm -f {} +
+            $ find . -name '*~' -exec rm -f {} +
+            $ find . -name '__pycache__' -exec rm -fr {} +
+	        $ rm -fr .pytest_cache
+
+        b. Build the source and wheel packages::
+
+            $ python setup.py sdist
+            $ python setup.py bdist_wheel
+            $ ls -l dist
+
+        c. Use twine to release to PyPI. You'll be asked for authentication. Use the username `__token__`, along with the API key I gave you.::
+
+            $ twine upload dist/*

{hydrobot-0.2.2 → hydrobot-0.3.0}/HISTORY.rst

@@ -21,3 +21,8 @@ History
 ------------------
 
 * Ok, actually including the config files (promise, but my fingers are crossed behind my back)
+
+0.2.3 (2023-11-30)
+------------------
+
+* Fixed some tests and hopefully fixed the config file issue this time.

{hydrobot-0.2.2 → hydrobot-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hydrobot
-Version: 0.2.2
+Version: 0.3.0
 Summary: Python Package providing a suite of processing tools        and utilities for Hilltop hydrological data.
 Home-page: https://github.com/nicmostert/hydrobot
 Author: Nic Mostert
@@ -12,7 +12,7 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: ==3.11
+Requires-Python: ==3.11.6
 License-File: LICENSE
 License-File: AUTHORS.rst
 
@@ -80,3 +80,8 @@ History
 ------------------
 
 * Ok, actually including the config files (promise, but my fingers are crossed behind my back)
+
+0.2.3 (2023-11-30)
+------------------
+
+* Fixed some tests and hopefully fixed the config file issue this time.

{hydrobot-0.2.2 → hydrobot-0.3.0}/docs/installation.rst

@@ -49,3 +49,23 @@ Once you have a copy of the source, you can install it with:
 
 .. _Github repo: https://github.com/nicmostert/hydrobot
 .. _tarball: https://github.com/nicmostert/hydrobot/tarball/master
+
+
+Usage
+-----
+
+For purposes of auditing data changes, you may want to track what changes to
+the data are made, and what version of hydrobot (and it's dependencies) are
+used.
+
+    Windows
+
+In Powershell:
+
+.. code-block:: console
+
+    cd path/to/Hydrobot_install
+    python -m venv /path/to/Hydrobot_install/venv
+    path/to/Hydrobot_install/venv/Scripts/Activate.ps1
+    pip install hydrobot
+    pip freeze > requirements.txt

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Nic Mostert"""
 __email__ = "nicolas.mostert@horizons.govt.nz"
-__version__ = "0.2.2"
+__version__ = "0.3.0"

hydrobot-0.3.0/hydrobot/config/TwoLevelMeasurement_QC_config.csv

@@ -0,0 +1 @@
+Water level statistics: Point Sample,10,3,0.5,0.2,2000

hydrobot-0.3.0/hydrobot/config/measurement_QC_config.csv

@@ -0,0 +1,2 @@
+Atmospheric Pressure,5,2.5
+Water Temperature [Dissolved Oxygen sensor],1.2,0.8

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot/data_acquisition.py

@@ -103,4 +103,5 @@ def get_series(
         data = pd.Series(data["Value"].values, data["Time"])
         data.index.name = "Time"
         data.name = "Value"
+        data.index = pd.to_datetime(data.index)
     return data

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot/data_sources.py

@@ -196,7 +196,10 @@ def get_measurement(measurement_name):
     if measurement_name in m_dict:
         return m_dict[measurement_name]
     else:
-        raise Exception("Measurement not found in the config file")
+        raise Exception(
+            f"Measurement {measurement_name} not found in the config file. "
+            f"Available measurements are {list(m_dict.keys())}."
+        )
 
 
 def series_export_to_csv(

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot/evaluator.py

@@ -311,10 +311,10 @@ def diagnose_data(base_series, check_series, qc_series, frequency):
     split_data = splitter(base_series, qc_series, frequency)
     for qc in split_data:
         print(
-            f"Data that is QC{qc} makes up {len(split_data[qc].dropna()) / len(base_series.dropna()) * 100}% of the "
-            f"workable data and {len(split_data[qc].dropna()) / len(base_series) * 100}% of the time period"
+            f"Data that is QC{qc} makes up {len(split_data[qc].dropna()) / len(base_series.dropna()) * 100:.2f}% of "
+            f"the "
+            f"workable data and {len(split_data[qc].dropna()) / len(base_series) * 100:.2f}% of the time period"
         )
-    print("Now it's your job to figure out if that's good enough")
 
 
 def splitter(base_series, qc_series, frequency):

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot/filters.py

@@ -23,7 +23,8 @@ def clip(unclipped, low_clip: float, high_clip: float):
     Returns
     -------
     pandas.Series
-        A Series containing the clipped values with the same index as the input Series.
+        A Series containing the clipped values with the same index as the
+        input Series.
     """
     unclipped_arr = unclipped.values
 
@@ -33,11 +34,11 @@ def clip(unclipped, low_clip: float, high_clip: float):
     # Use pandas' where function to clip values to NaN where the condition is
     # True
     clipped_series = unclipped.where(~clip_cond, np.nan)
-    clipped_series = unclipped.where(~clip_cond, np.nan)
 
     return clipped_series
 
 
+# noinspection SpellCheckingInspection
 def fbewma(input_data, span: int):
     """
     Calculate the Forward-Backward Exponentially Weighted Moving Average (FBEWMA) of a pandas Series.
@@ -52,7 +53,8 @@ def fbewma(input_data, span: int):
     Returns
     -------
     pandas.Series
-        A Series containing the FBEWMA values with the same index as the input Series.
+        A Series containing the FBEWMA values with the same index as the
+        input Series.
     """
     # Calculate the Forward EWMA.
     fwd = input_data.ewm(span=span).mean()
@@ -74,14 +76,14 @@ def remove_outliers(input_data, span: int, delta: float):
     Remove outliers.
 
     Remove outliers from a time series by comparing it to the
-    Forward-Backward Exponentially Weighted Moving Average (FB-EWMA).
+    Forward-Backward Exponentially Weighted Moving Average (FBEWMA).
 
     Parameters
     ----------
     input_data : pandas.Series
         Input time series data.
     span : int
-        Span parameter for exponential weighting used in the FB-EWMA.
+        Span parameter for exponential weighting used in the FBEWMA.
     delta : float
         Threshold for identifying outliers. Values greater than this
         threshold will be set to NaN.
@@ -92,7 +94,7 @@ def remove_outliers(input_data, span: int, delta: float):
         A Series containing the time series with outliers removed with
         the same index as the input Series.
     """
-    # Calculate the FB-EWMA of the time series
+    # Calculate the FBEWMA of the time series
     fbewma_series = fbewma(input_data, span)
 
     # Create a condition to identify outliers based on the absolute difference
@@ -143,3 +145,28 @@ def remove_spikes(
     # interp_series = gaps_series.interpolate()
 
     return gaps_series
+
+
+def remove_range(input_series: pd.Series, from_date, to_date):
+    """
+    Remove data from series in given range.
+
+    Returns the input series without data between from_date and to_date
+    inclusive
+
+    Parameters
+    ----------
+    input_series : pd.Series
+        The series to have a section removed
+    from_date : str
+        Start of removed section
+    to_date : str
+        End of removed section
+
+    Returns
+    -------
+    pd.Series
+        The series with relevant slice removed
+    """
+    slice_to_remove = input_series.loc[from_date:to_date]
+    return input_series.drop(slice_to_remove.index)

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot/plotter.py

@@ -165,6 +165,7 @@ def qc_plotter(base_series, check_series, qc_series, frequency, show=True):
         color="black",
         linestyle="None",
     )
+    plt.xticks(rotation=45, ha="right")
 
     plt.legend()
     if show:

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot/processor.py

@@ -16,14 +16,16 @@ def stale_warning(method):
     Decorate dangerous functions.
 
     Check whether the data is stale, and warn user if so.
-    Warning will then take input form user to determine whether to proceed or cancel.
-    Cancelling will return a null function, which returns None with no side effects no matter what the input
+    Warning will then take input form user to determine whether to proceed
+    or cancel.
+    Cancelling will return a null function, which returns None with no side
+    effects no matter what the input
 
     Parameters
     ----------
     method : function
-        A function that might have some problems if the parameters have been changed but the data hasn't been
-        updated
+        A function that might have some problems if the parameters have been
+        changed but the data hasn't been updated
 
     Returns
     -------
@@ -35,7 +37,8 @@ def stale_warning(method):
     def _impl(self, *method_args, **method_kwargs):
         if self._stale:
             warnings.warn(
-                "Warning: a key parameter of the data has changed but the data itself has not been reloaded."
+                "Warning: a key parameter of the data has changed but the "
+                "data itself has not been reloaded."
             )
             while True:
                 user_input = input("Do you want to continue? y/n: ")
@@ -71,7 +74,7 @@ class Processor:
         to_date: str | None = None,
         check_hts: str | None = None,
         check_measurement: str | None = None,
-        defaults: dict = {},
+        defaults: dict | None = None,
         **kwargs,
     ):
         """Initialize a Processor instance."""
@@ -90,27 +93,30 @@ class Processor:
         else:
             raise ValueError(
                 f"Site '{site}' not found for both base_url and hts combos."
-                f"Available sites in standard_hts are {[s for s in standard_hilltop.available_sites]}"
-                f"Available sites in check_hts are {[s for s in check_hilltop.available_sites]}"
+                f"Available sites in standard_hts are: "
+                f"{[s for s in standard_hilltop.available_sites]}"
+                f"Available sites in check_hts are: "
+                f"{[s for s in check_hilltop.available_sites]}"
             )
 
-        self._standard_measurement_list = standard_hilltop.get_measurement_list(site)
-        if standard_measurement in self._standard_measurement_list.values:
+        standard_measurement_list = standard_hilltop.get_measurement_list(site)
+        if standard_measurement in list(standard_measurement_list.MeasurementName):
             self._standard_measurement = standard_measurement
         else:
             raise ValueError(
-                f"Standard measurement '{standard_measurement}' not found at site '{site}'. "
+                f"Standard measurement '{standard_measurement}' not found at "
+                f"site '{site}'. "
                 "Available measurements are "
-                f"{[str(m[0]) for m in self._standard_measurement_list.values]}"
+                f"{list(standard_measurement_list.MeasurementName)}"
             )
-        self._check_measurement_list = check_hilltop.get_measurement_list(site)
-        if check_measurement in self._check_measurement_list.values:
+        check_measurement_list = check_hilltop.get_measurement_list(site)
+        if check_measurement in list(check_measurement_list.MeasurementName):
             self._check_measurement = check_measurement
         else:
             raise ValueError(
                 f"Check measurement '{check_measurement}' not found at site '{site}'. "
                 "Available measurements are "
-                f"{[str(m[0]) for m in self._check_measurement_list.values]}"
+                f"{list(check_measurement_list.MeasurementName)}"
             )
 
         self._base_url = base_url
@@ -281,10 +287,9 @@ class Processor:
                 tstype="Standard",
             )
             insert_series = insert_series.asfreq(self.frequency)
-            slice_to_remove = self.standard_series.loc[
-                insert_series.index[0] : insert_series.index[-1]
-            ]
-            cleaned_series = self.standard_series.drop(slice_to_remove.index)
+            cleaned_series = filters.remove_range(
+                self.standard_series, insert_series.index[0], insert_series.index[-1]
+            )
             self.standard_series = pd.concat(
                 [cleaned_series, insert_series]
             ).sort_index()
@@ -328,6 +333,9 @@ class Processor:
         quality: bool = False,
     ):
         """Import data using class parameter range."""
+        self.standard_series = pd.Series({})
+        self.check_series = pd.Series({})
+        self.quality_series = pd.Series({})
         self.import_range(self._from_date, self._to_date, standard, check, quality)
         self._stale = False
 
@@ -410,6 +418,34 @@ class Processor:
             self._standard_series, span, low_clip, high_clip, delta
         )
 
+    @ClassLogger
+    def delete_range(
+        self,
+        from_date,
+        to_date,
+        tstype_standard=True,
+        tstype_check=False,
+        tstype_quality=False,
+    ):
+        """Delete range of data a la remove_range."""
+        if tstype_standard:
+            self.standard_series = filters.remove_range(
+                self.standard_series, from_date, to_date
+            )
+        if tstype_check:
+            self.standard_series = filters.remove_range(
+                self.standard_series, from_date, to_date
+            )
+        if tstype_quality:
+            self.standard_series = filters.remove_range(
+                self.standard_series, from_date, to_date
+            )
+
+    @ClassLogger
+    def insert_missing_nans(self):
+        """Set the data to the correct frequency, filled with NaNs as appropriate."""
+        self.standard_series = self.standard_series.asfreq(self.frequency)
+
     @ClassLogger
     def data_exporter(self, file_location):
         """Export data to csv."""

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hydrobot
-Version: 0.2.2
+Version: 0.3.0
 Summary: Python Package providing a suite of processing tools        and utilities for Hilltop hydrological data.
 Home-page: https://github.com/nicmostert/hydrobot
 Author: Nic Mostert
@@ -12,7 +12,7 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Natural Language :: English
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: ==3.11
+Requires-Python: ==3.11.6
 License-File: LICENSE
 License-File: AUTHORS.rst
 
@@ -80,3 +80,8 @@ History
 ------------------
 
 * Ok, actually including the config files (promise, but my fingers are crossed behind my back)
+
+0.2.3 (2023-11-30)
+------------------
+
+* Fixed some tests and hopefully fixed the config file issue this time.

{hydrobot-0.2.2 → hydrobot-0.3.0}/hydrobot.egg-info/SOURCES.txt

@@ -34,6 +34,8 @@ hydrobot.egg-info/dependency_links.txt
 hydrobot.egg-info/not-zip-safe
 hydrobot.egg-info/requires.txt
 hydrobot.egg-info/top_level.txt
+hydrobot/config/TwoLevelMeasurement_QC_config.csv
+hydrobot/config/measurement_QC_config.csv
 tests/__init__.py
 tests/site_list_response.xml
 tests/test_data_sources.py

{hydrobot-0.2.2 → hydrobot-0.3.0}/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.2
+current_version = 0.3.0
 commit = True
 tag = True
 

{hydrobot-0.2.2 → hydrobot-0.3.0}/setup.py

@@ -12,12 +12,17 @@ with open("HISTORY.rst") as history_file:
 
 requirements: list = ["hilltop-py>=2.3.1", "data-annalist>=0.3.6", "matplotlib>=3.8.0"]
 
-test_requirements = ["pytest>=3", "data-annalist>=0.3.6"]
+test_requirements = [
+    "pytest>=3",
+    "data-annalist>=0.3.6",
+    "pytest-mock>=3.12.0",
+    "pytest-dependency>=0.5.1",
+]
 
 setup(
     author="Nic Mostert",
     author_email="nicolas.mostert@horizons.govt.nz",
-    python_requires="==3.11",
+    python_requires="==3.11.6",
     classifiers=[
         "Development Status :: 2 - Pre-Alpha",
         "Intended Audience :: Developers",
@@ -42,6 +47,7 @@ setup(
     test_suite="tests",
     tests_require=test_requirements,
     url="https://github.com/nicmostert/hydrobot",
-    version="0.2.2",
+    version="0.3.0",
     zip_safe=False,
+    package_data={"hydrobot": ["config/*"]},
 )

{hydrobot-0.2.2 → hydrobot-0.3.0}/tests/test_filters.py

@@ -36,9 +36,10 @@ def raw_data():
 
 
 def insert_raw_data_gaps(gaps):
+    gap_data_dict = dict(raw_data_dict)
     for gap in gaps:
-        raw_data_dict[gap] = np.nan
-    data_series = pd.Series(raw_data_dict)
+        gap_data_dict[gap] = np.nan
+    data_series = pd.Series(gap_data_dict)
     return data_series
 
 
@@ -81,10 +82,10 @@ def test_fbewma(raw_data, fbewma_data):
     span = 3
 
     # Testing
-    fbewmadf = filters.fbewma(raw_data, span)
+    fbewma_df = filters.fbewma(raw_data, span)
 
     # pytest.approx accounts for floating point errors and such
-    assert fbewmadf.values == pytest.approx(fbewma_data.values), "FBEWMA failed!"
+    assert fbewma_df.values == pytest.approx(fbewma_data.values), "FBEWMA failed!"
 
 
 def test_remove_outliers(raw_data, fbewma_data, mocker, span=2, delta=2):
@@ -127,3 +128,14 @@ def test_remove_spike(raw_data, fbewma_data, mocker):
 
     spike_removed = filters.remove_spikes(raw_data, span, high_clip, low_clip, delta)
     assert math.isnan(spike_removed["2021-01-01 00:10"]), "Spike not removed!"
+
+
+def test_remove_range(raw_data):
+    a = filters.remove_range(raw_data, "2021-01-01 00:05", "2021-01-01 00:10")
+    assert len(a) == 3, "Incorrect number of values removed"
+    assert a["2021-01-01 00:00"] == 1.0, "first value compromised"
+    assert a["2021-01-01 00:15"] == 4.0, "after value compromised"
+    assert a["2021-01-01 00:20"] == 5.0, "end value compromised"
+
+    b = filters.remove_range(raw_data, "2021-01-01 00:03", "2021-01-01 00:14")
+    assert b.equals(a), "time rounding error"

{hydrobot-0.2.2 → hydrobot-0.3.0}/tests/test_processor.py

@@ -1,4 +1,3 @@
-'''
 import random
 from datetime import datetime, timedelta
 
@@ -20,6 +19,7 @@ SITES = [
 
 MEASUREMENTS = [
     "General Nastiness (out of 10)",
+    "Atmospheric Pressure",
     "Number of Actual Whole Turds Floating By (t/s)",
     "Dead Cow Concentration (ppm)",
 ]
@@ -27,14 +27,13 @@ MEASUREMENTS = [
 
 @pytest.fixture(autouse=True)
 def no_requests(monkeypatch):
-    """Don't allow requests to make requests"""
+    """Don't allow requests to make requests."""
     monkeypatch.delattr("requests.sessions.Session.request")
 
 
 @pytest.fixture
 def mock_site_list():
-    """Mock response from SiteList server call method"""
-
+    """Mock response from SiteList server call method."""
     data = {
         "SiteName": SITES,
     }
@@ -45,8 +44,7 @@ def mock_site_list():
 
 @pytest.fixture
 def mock_measurement_list():
-    """Mock response from MeasurementList server call method"""
-
+    """Mock response from MeasurementList server call method."""
     data = {
         "MeasurementName": MEASUREMENTS,
     }
@@ -78,8 +76,9 @@ def mock_dataset():
     return df
 
 
-
 def test_processor(monkeypatch, mock_site_list, mock_measurement_list, mock_dataset):
+    """Test the processor function."""
+
     def get_mock_site_list(*args, **kwargs):
         return mock_site_list
 
@@ -95,8 +94,11 @@ def test_processor(monkeypatch, mock_site_list, mock_measurement_list, mock_data
 
     pr = processor.Processor(
         "https://greenwashed.and.pleasant/",
-        "GreenPasturesAreNaturalAndEcoFriendlyISwear.hts",
         SITES[1],
+        "GreenPasturesAreNaturalAndEcoFriendlyISwear.hts",
         MEASUREMENTS[1],
+        "5T",
     )
-    assert pr.dataset.loc[0].Time == "2020-10-01 08:00:00"'''
+    pr.import_data()
+    print(pr.standard_series)
+    assert pr.standard_series.loc["2020-10-01 08:00:00"]