shadoof 0.1.0__tar.gz

shadoof-0.1.0/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: shadoof
+Version: 0.1.0
+Summary: Command line utility for writing to hilltop from xmls and dsns of xmls
+License: GNU General Public License v3
+Author: SIrvine1994
+Author-email: sam.irvine@horizons.govt.nz
+Requires-Python: >=3.14.1,<4.0
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Requires-Dist: pyodbc (>=5.3.0,<6.0.0)
+Requires-Dist: pywin32 (>=312,<313)
+Requires-Dist: typer (>=0.27.1,<0.28.0)
+Requires-Dist: whurl (>=0.2.1,<0.3.0)

shadoof-0.1.0/pyproject.toml

@@ -0,0 +1,44 @@
+[project]
+name = "shadoof"
+version = "0.1.0"
+description = "Command line utility for writing to hilltop from xmls and dsns of xmls"
+authors = [
+    {name = "SIrvine1994", email = "sam.irvine@horizons.govt.nz"}
+]
+license = "GNU General Public License v3"
+requires-python = ">=3.14.1,<4.0"
+dependencies = [
+    "typer (>=0.27.1,<0.28.0)",
+    "whurl (>=0.2.1,<0.3.0)",
+    "pyodbc (>=5.3.0,<6.0.0)",
+    "pywin32 (>=312,<313)"
+
+]
+
+[project.scripts]
+shadoof = "shadoof.cli:app"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.bumpversion]
+current_version = "0.1.0"
+commit = true
+tag = true
+tag_name = "{new_version}"
+
+[[tool.bumpversion.files]]
+filename = "pyproject.toml"
+search = "{current_version}"
+replace = "{new_version}"
+
+[[tool.bumpversion.files]]
+filename = "src/shadoof/__init__.py"
+search = "__version__ = \"{current_version}\""
+replace = "__version__ = \"{new_version}\""
+
+[[tool.bumpversion.files]]
+filename = "readme.rst"
+search = "{current_version}"
+replace = "{new_version}"

shadoof-0.1.0/src/shadoof/__init__.py

@@ -0,0 +1,4 @@
+"""Top-level package for GeoGlyph."""
+
+__author__ = """Sam Irvine"""
+__version__ = "0.1.0"

shadoof-0.1.0/src/shadoof/__main__.py

@@ -0,0 +1,3 @@
+if __name__ == "__main__":
+    from shadoof.cli import app
+    app()

shadoof-0.1.0/src/shadoof/auditor.py

@@ -0,0 +1,98 @@
+"""Adds an audit trail for a dsn copy consistent with copying the individual xml/hts files."""
+
+import datetime
+import getpass
+import re
+from pathlib import Path
+
+import pyodbc
+import whurl
+
+
+def get_audit_dict(whurl_measurement):
+    """Get audit details in desired dict format."""
+    return {
+            "site": whurl_measurement.site_name,
+            "data_source": whurl_measurement.data_source.name,
+            "ts_type": whurl_measurement.data_source.ts_type,
+            "start_date": whurl_measurement.data.timeseries.index[0],
+            "end_date": whurl_measurement.data.timeseries.index[-1],
+        }
+
+
+def get_whurl_audit(whurl_root):
+    """Get the details from a whurl object, returns details for audit."""
+    details = []
+    for meas in whurl_root.measurement:
+        details.append(get_audit_dict(meas))
+    return details
+
+
+def get_audit_destination(hts_destination):
+    """Gets the audit destination from an hts file."""
+    audit_destination = str(Path(hts_destination).with_suffix(".accdb"))
+    if not Path(audit_destination).exists():
+        raise FileNotFoundError(f"The audit file {audit_destination} does not exist.")
+    return audit_destination
+
+
+def write_to_access_for_single_copy(source, destination):
+    """Adds an audit trail for copying an individual xml/hts file."""
+    root = whurl.schemas.responses.GetDataResponse.from_xml(source)
+    for transaction in get_whurl_audit(root):
+        write_access_row(
+            get_audit_destination(destination),
+            transaction["site"],
+            transaction["data_source"],
+            transaction["ts_type"],
+            transaction["start_date"],
+            transaction["end_date"],
+            source,
+        )
+
+
+def write_access_row(
+    access_file, site, data_source, ts_type, start_date, end_date, source
+):
+    """Write a single access audit entry."""
+    connection_string = (
+        "DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=" + access_file + ";"
+    )
+    cnxn = pyodbc.connect(connection_string)
+
+    insert_query = """
+        INSERT INTO Audit ("Date","UserName","Process","Site","DataSource","TsType","StartDate","EndDate","Comment")
+        VALUES (?,?,?,?,?,?,?,?,?);
+    """
+    values = (
+        datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+        getpass.getuser(),
+        "Copy from Site",
+        site,
+        data_source,
+        ts_type,
+        start_date,
+        end_date,
+        source,
+    )
+
+    cursor = cnxn.cursor()
+    print(values)  # Actually good to have some feedback - not just a debug print
+    cursor.execute(insert_query, values)
+    cursor.commit()
+
+    cursor.close()
+    cnxn.close()
+
+
+def write_to_access_for_dsn(source_dsn, destination):
+    """Parse dsn or other file to turn it into auditing instructions."""
+    with open(source_dsn) as file:
+        dsn_text = file.read()
+    regex = re.compile(r'File\d*="(.*)"')
+    source_file = regex.findall(dsn_text)
+    for path in source_file:
+        if Path(path).suffix == ".dsn":
+            write_to_access_for_dsn(path, destination)
+        else:
+            write_to_access_for_single_copy(path, destination)

shadoof-0.1.0/src/shadoof/cli.py

@@ -0,0 +1,13 @@
+"""Command line client."""
+
+import typer
+
+from shadoof.uploader import write_to_hilltop
+
+
+app = typer.Typer()
+app.command()(write_to_hilltop)
+
+
+if __name__ == "__main__":
+    app()

shadoof-0.1.0/src/shadoof/uploader.py

@@ -0,0 +1,194 @@
+"""Write data to a Hilltop file."""
+from datetime import UTC, datetime
+
+import pandas as pd
+import pythoncom
+import pywintypes
+import win32com.client
+from win32com.client import VARIANT
+
+import whurl
+import re
+from pathlib import Path
+import os
+
+import shadoof.auditor as audit
+
+
+def _data_date_converter(df_row, info):
+    if info == "D":
+        return pd.to_datetime(df_row).to_pydatetime().replace(tzinfo=UTC)
+    else:
+        if pd.isna(df_row):
+            return ""
+        else:
+            return df_row
+
+
+def tstype_converter(ts_type: str) -> int:
+    """
+    Convert tstype strings into ints for COM interpretation.
+
+    standard = 1
+    quality = 2
+    check = 3
+    """
+    match ts_type:
+        case "StdSeries":
+            return 1
+        case "StdQualSeries":
+            return 2
+        case "CheckSeries":
+            return 3
+        case _:
+            raise ValueError(f"Unrecognised ts_type: {ts_type}")
+
+
+class ToHilltop:
+    """
+    Hilltop COM writer.
+
+    Writes directly into a Hilltop file.
+    """
+
+    def __init__(self, output_hts):
+        self.dput = win32com.client.Dispatch("Hilltop.DataInput")
+        if not self.dput.open(output_hts):
+            raise RuntimeError(f"No data file.  Error is: {self.dput.errormsg}")
+
+        # Variants for receiving the date and values  They have to be initialised, not just declared.
+        # We give a dummy date to get started
+        self.vtTime = VARIANT(pythoncom.VT_DATE, pywintypes.Time(datetime(2024, 1, 1)))
+        self.vtValue = VARIANT(pythoncom.VT_R4, 0.0)
+        self.vtValues = VARIANT(
+            pythoncom.VT_BYREF | pythoncom.VT_ARRAY | pythoncom.VT_VARIANT, []
+        )
+
+    def close(self):
+        """
+        Close the file - maybe.
+
+        It appears to just call a bool, which shouldn't have an effect. However, there isn't enough documentation to
+        say that this doesn't have a side effect that prevents a memory leak - so leaving this be for now. COM object
+        bad. Returning it so that the linter doesn't complain.
+        """
+        return self.dput.Close
+
+    def putData(self, data_source_name, ts_type, item_info, site_name, data):
+        data.attrs["timeSeriesType"] = tstype_converter(ts_type)  # time series type
+        data.attrs["featureOfInterest"] = site_name  # Site
+        data.attrs["procedure"] = data_source_name  # Datasource
+        if tstype_converter(ts_type) == 2:
+            # Quality doesn't have item_info normally
+            item_info = ["I"]
+
+        self._put(data, item_info)
+
+    # Write a Pandas Frame
+    def put_std(self, df):
+        """Insert Standard data into Hilltop COM."""
+        df.attrs["timeSeriesType"] = 1
+        self._put(df, ["F"])
+
+    def put_qual(self, df):
+        """Insert Quality data into hilltop COM."""
+        df.attrs["timeSeriesType"] = 2
+        self._put(df, ["I"])
+
+    def put_check(self, df, data_type_list):
+        """Insert check data into hilltop COM."""
+
+        df.attrs["timeSeriesType"] = 3
+        self._put(df, data_type_list)
+
+    def _put(self, df, data_type_list):
+        """Put data of arbitrary type to Hilltop via COM."""
+
+        """Insert check data into hilltop COM."""
+        if not self.dput.PutNew2(
+            df.attrs["featureOfInterest"],
+            df.attrs["procedure"],
+            df.attrs["timeSeriesType"],
+        ):
+            raise RuntimeError(f"{self.dput.ErrorMsg}")
+
+        if df.attrs["timeSeriesType"] not in [1, 2, 3]:
+            raise ValueError(f"Invalid time series value {df.attrs['timeSeriesType']}")
+
+        for data_type in data_type_list:
+            known_data_types = ["F", "D", "S", "I"]  # float, date, string, integer
+            if data_type not in known_data_types:
+                raise ValueError(
+                    f"Invalid data_type: {data_type}. Valid data types are {known_data_types}."
+                )
+        if len(data_type_list) != len(df.columns):
+            raise ValueError(
+                f"Received different number of data types and data columns. There are "
+                f"{len(data_type_list)} data types given and {len(df.columns)} values in the data frame."
+            )
+
+        for row in df.itertuples():
+            tm = row.Index.to_pydatetime()
+            tm = tm.replace(tzinfo=UTC)  # Remove the time zone win32com hates it
+            self.vtTime = tm
+            self.vtValues = [
+                _data_date_converter(data, info)
+                for (data, info) in zip(row[1:], data_type_list, strict=True)
+            ]
+            self.dput.PutArray(self.vtTime, self.vtValues)
+
+
+def read_hilltop_xml(xml):
+    with open(xml, "r") as file:
+        xml_content = file.read()
+    root = whurl.schemas.responses.GetDataResponse.from_xml(xml_content)
+    outputs = []
+    for meas in root.measurements:
+        meas_dict = {
+            "data_source_name": meas.data_source.name,
+            "ts_type": meas.data_source.ts_type,
+            "item_info": [m.item_format for m in meas.data_source.item_info],
+            "site_name": meas.site_name,
+            "data": meas.data.timeseries,
+        }
+        audit_dict = audit.get_audit_dict(meas)
+        outputs.append((meas_dict, audit_dict))
+    return outputs
+
+
+def read_hilltop_dsn(dsn):
+    """Parse dsn to turn it into write instructions."""
+    with open(dsn) as file:
+        dsn_text = file.read()
+    regex = re.compile(r'File\d*="(.*)"')
+    source_file = regex.findall(dsn_text)
+    outputs = []
+    for path in source_file:
+        print(path)
+        if Path(path).suffix == ".dsn":
+            outputs += read_hilltop_dsn(path)
+        else:
+            outputs.append(read_hilltop_xml(path))
+    return outputs
+
+
+def write_to_hilltop(input_file, destination):
+    """Write a data file to a given Hilltop file."""
+
+    if os.path.splitext(destination)[1] != ".hts":
+        raise ValueError(f"Destination must be a .hts file, path is {destination}")
+    outputs = None
+    match os.path.splitext(input_file)[1]:
+        case ".xml":
+            outputs = read_hilltop_xml(input_file)
+        case ".dsn":
+            outputs = read_hilltop_dsn(input_file)
+        case _:
+            raise ValueError(f"Unrecognised input file type '{os.path.splitext(input_file)[1]}'. Full path is {input_file}.")
+
+    hts_com = ToHilltop(destination)
+    for o in outputs:
+        print(o[1])
+        audit.write_access_row(audit.get_audit_destination(destination), source=input_file, **o[1])
+        hts_com.putData(**o[0])
+    hts_com.close()