pthelma 0.99.3.dev0__cp312-cp312-win_amd64.whl

enhydris_api_client/__init__.py

@@ -0,0 +1,252 @@
+from copy import copy
+from io import StringIO
+from urllib.parse import urljoin
+
+try:
+    from zoneinfo import ZoneInfo
+except ImportError:
+    from backports.zoneinfo import ZoneInfo
+
+import iso8601
+import requests
+
+from htimeseries import HTimeseries
+
+
+class EnhydrisApiClient:
+    def __init__(self, base_url, token=None):
+        self.base_url = base_url
+        self.token = token
+        self.session = requests.Session()
+        if token is not None:
+            self.session.headers.update({"Authorization": f"token {self.token}"})
+
+        # My understanding from requests' documentation is that when I make a post
+        # request, it shouldn't be necessary to specify Content-Type:
+        # application/x-www-form-urlencoded, and that requests adds the header
+        # automatically. However, when running in Python 3, apparently requests does not
+        # add the header (although it does convert the post data to
+        # x-www-form-urlencoded format). This is why I'm specifying it explicitly.
+        self.session.headers.update(
+            {"Content-Type": "application/x-www-form-urlencoded"}
+        )
+
+    def __enter__(self):
+        self.session.__enter__()
+        return self
+
+    def __exit__(self, *args):
+        self.session.__exit__(*args)
+
+    def check_response(self, expected_status_code=None):
+        try:
+            self._raise_HTTPError_on_error(expected_status_code=expected_status_code)
+        except requests.HTTPError as e:
+            if self.response.text:
+                raise requests.HTTPError(
+                    f"{str(e)}. Server response: {self.response.text}"
+                )
+
+    def _raise_HTTPError_on_error(self, expected_status_code):
+        self._check_status_code_is_nonerror()
+        self._check_status_code_is_the_one_expected(expected_status_code)
+
+    def _check_status_code_is_nonerror(self):
+        self.response.raise_for_status()
+
+    def _check_status_code_is_the_one_expected(self, expected_status_code):
+        if expected_status_code and self.response.status_code != expected_status_code:
+            raise requests.HTTPError(
+                f"Expected status code {expected_status_code}; "
+                f"got {self.response.status_code} instead"
+            )
+
+    def get_token(self, username, password):
+        if not username:
+            return
+
+        # Get a csrftoken
+        login_url = urljoin(self.base_url, "api/auth/login/")
+        data = f"username={username}&password={password}"
+        self.response = self.session.post(login_url, data=data, allow_redirects=False)
+        self.check_response()
+        key = self.response.json()["key"]
+        self.session.headers.update({"Authorization": f"token {key}"})
+        return key
+
+    def get_station(self, station_id):
+        url = urljoin(self.base_url, f"api/stations/{station_id}/")
+        self.response = self.session.get(url)
+        self.check_response()
+        return self.response.json()
+
+    def post_station(self, data):
+        self.response = self.session.post(
+            urljoin(self.base_url, "api/stations/"), data=data
+        )
+        self.check_response()
+        return self.response.json()["id"]
+
+    def put_station(self, station_id, data):
+        self.response = self.session.put(
+            urljoin(self.base_url, f"api/stations/{station_id}/"), data=data
+        )
+        self.check_response()
+
+    def patch_station(self, station_id, data):
+        self.response = self.session.patch(
+            urljoin(self.base_url, f"api/stations/{station_id}/"), data=data
+        )
+        self.check_response()
+
+    def delete_station(self, station_id):
+        url = urljoin(self.base_url, f"api/stations/{station_id}/")
+        self.response = self.session.delete(url)
+        self.check_response(expected_status_code=204)
+
+    def get_timeseries_group(self, station_id, timeseries_group_id):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/",
+        )
+        self.response = self.session.get(url)
+        self.check_response()
+        return self.response.json()
+
+    def post_timeseries_group(self, station_id, data):
+        url = urljoin(self.base_url, f"api/stations/{station_id}/timeseriesgroups/")
+        self.response = self.session.post(url, data=data)
+        self.check_response()
+        return self.response.json()["id"]
+
+    def put_timeseries_group(self, station_id, timeseries_group_id, data):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/",
+        )
+        self.response = self.session.put(url, data=data)
+        self.check_response()
+        return self.response.json()["id"]
+
+    def patch_timeseries_group(self, station_id, timeseries_group_id, data):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/",
+        )
+        self.response = self.session.patch(url, data=data)
+        self.check_response()
+
+    def delete_timeseries_group(self, station_id, timeseries_group_id):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/",
+        )
+        self.response = self.session.delete(url)
+        self.check_response(expected_status_code=204)
+
+    def list_timeseries(self, station_id, timeseries_group_id):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/"
+            "timeseries/",
+        )
+        self.response = self.session.get(url)
+        self.check_response()
+        return self.response.json()["results"]
+
+    def get_timeseries(self, station_id, timeseries_group_id, timeseries_id):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/"
+            f"timeseries/{timeseries_id}/",
+        )
+        self.response = self.session.get(url)
+        self.check_response()
+        return self.response.json()
+
+    def post_timeseries(self, station_id, timeseries_group_id, data):
+        self.response = self.session.post(
+            urljoin(
+                self.base_url,
+                f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/"
+                "timeseries/",
+            ),
+            data=data,
+        )
+        self.check_response()
+        return self.response.json()["id"]
+
+    def delete_timeseries(self, station_id, timeseries_group_id, timeseries_id):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/"
+            f"timeseries/{timeseries_id}/",
+        )
+        self.response = self.session.delete(url)
+        self.check_response(expected_status_code=204)
+
+    def read_tsdata(
+        self,
+        station_id,
+        timeseries_group_id,
+        timeseries_id,
+        start_date=None,
+        end_date=None,
+        timezone=None,
+    ):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/"
+            f"timeseries/{timeseries_id}/data/",
+        )
+        params = {"fmt": "hts"}
+        tzinfo = ZoneInfo(timezone) if timezone else None
+        dates_are_aware = (start_date is None or start_date.tzinfo is not None) and (
+            end_date is None or end_date.tzinfo is not None
+        )
+        if not dates_are_aware:
+            raise ValueError("start_date and end_date must be aware")
+        params["start_date"] = start_date and start_date.isoformat()
+        params["end_date"] = end_date and end_date.isoformat()
+        params["timezone"] = timezone
+        self.response = self.session.get(url, params=params)
+        self.check_response()
+        if self.response.text:
+            return HTimeseries(StringIO(self.response.text), default_tzinfo=tzinfo)
+        else:
+            return HTimeseries()
+
+    def post_tsdata(self, station_id, timeseries_group_id, timeseries_id, ts):
+        f = StringIO()
+        data = copy(ts.data)
+        try:
+            data.index = data.index.tz_convert("UTC")
+        except AttributeError:
+            assert data.empty
+        data.to_csv(f, header=False)
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/"
+            f"timeseries/{timeseries_id}/data/",
+        )
+        self.response = self.session.post(
+            url, data={"timeseries_records": f.getvalue(), "timezone": "UTC"}
+        )
+        self.check_response()
+        return self.response.text
+
+    def get_ts_end_date(
+        self, station_id, timeseries_group_id, timeseries_id, timezone=None
+    ):
+        url = urljoin(
+            self.base_url,
+            f"api/stations/{station_id}/timeseriesgroups/{timeseries_group_id}/"
+            f"timeseries/{timeseries_id}/bottom/",
+        )
+        self.response = self.session.get(url, params={"timezone": timezone})
+        self.check_response()
+        try:
+            datestring = self.response.text.strip().split(",")[0]
+            return iso8601.parse_date(datestring, default_timezone=None)
+        except (IndexError, iso8601.ParseError):
+            return None

enhydris_cache/__init__.py

@@ -0,0 +1,5 @@
+from .enhydris_cache import *  # NOQA
+
+__author__ = """Antonis Christofides"""
+__email__ = "antonis@antonischristofides.com"
+__version__ = "0.1.0.dev0"

enhydris_cache/cli.py

@@ -0,0 +1,150 @@
+import configparser
+import datetime as dt
+import logging
+import os
+import sys
+import traceback
+
+import click
+
+from enhydris_cache import TimeseriesCache
+from pthelma._version import __version__
+
+
+class WrongValueError(configparser.Error):
+    pass
+
+
+class App:
+    def __init__(self, configfilename):
+        self.configfilename = configfilename
+
+    def run(self):
+        self.config = AppConfig(self.configfilename)
+        self.config.read()
+        self._setup_logger()
+        self._execute_with_error_handling()
+
+    def _execute_with_error_handling(self):
+        self.logger.info("Starting enhydris-cache, " + dt.datetime.today().isoformat())
+        try:
+            self._execute()
+        except Exception as e:
+            self.logger.error(str(e))
+            self.logger.debug(traceback.format_exc())
+            self.logger.info(
+                "enhydris-cache terminated with error, "
+                + dt.datetime.today().isoformat()
+            )
+            raise click.ClickException(str(e))
+        else:
+            self.logger.info(
+                "Finished enhydris-cache, " + dt.datetime.today().isoformat()
+            )
+
+    def _setup_logger(self):
+        self.logger = logging.getLogger("spatialize")
+        self._set_logger_handler()
+        self.logger.setLevel(self.config.loglevel.upper())
+
+    def _set_logger_handler(self):
+        if getattr(self.config, "logfile", None):
+            self.logger.addHandler(logging.FileHandler(self.config.logfile))
+        else:
+            self.logger.addHandler(logging.StreamHandler())
+
+    def _execute(self):
+        os.chdir(self.config.cache_dir)
+        self.cache = TimeseriesCache(self.config.timeseries_group)
+        self.cache.update()
+
+
+class AppConfig:
+    config_file_general_options = {
+        "logfile": {"fallback": ""},
+        "loglevel": {"fallback": "warning"},
+        "cache_dir": {"fallback": os.getcwd()},
+    }
+    config_file_timeseries_options = {
+        "base_url": {},
+        "station_id": {},
+        "timeseries_id": {},
+        "timeseries_group_id": {},
+        "auth_token": {"fallback": None},
+        "file": {},
+    }
+
+    def __init__(self, configfilename):
+        self.configfilename = configfilename
+
+    def read(self):
+        try:
+            self._parse_config()
+        except (OSError, configparser.Error) as e:
+            sys.stderr.write(str(e))
+            raise click.ClickException(str(e))
+
+    def _parse_config(self):
+        self._read_config_file()
+        self._parse_general_section()
+        self._parse_timeseries_sections()
+
+    def _read_config_file(self):
+        self.config = configparser.ConfigParser(interpolation=None)
+        with open(self.configfilename) as f:
+            self.config.read_file(f)
+
+    def _parse_general_section(self):
+        options = {
+            opt: self.config.get("General", opt, **kwargs)
+            for opt, kwargs in self.config_file_general_options.items()
+        }
+        for key, value in options.items():
+            setattr(self, key, value)
+        self._parse_log_level()
+
+    def _parse_log_level(self):
+        log_levels = ("ERROR", "WARNING", "INFO", "DEBUG")
+        self.loglevel = self.loglevel.upper()
+        if self.loglevel not in log_levels:
+            raise WrongValueError("loglevel must be one of " + ", ".join(log_levels))
+
+    def _parse_timeseries_sections(self):
+        self.timeseries_group = []
+        for section in self.config:
+            if section in ("General", "DEFAULT"):
+                continue
+            item = self._read_section(section)
+            self.timeseries_group.append(item)
+
+    def _read_section(self, section):
+        options = {
+            opt: self.config.get(section, opt, **kwargs)
+            for opt, kwargs in self.config_file_timeseries_options.items()
+        }
+        options["name"] = section
+        try:
+            options["station_id"] = int(options["station_id"])
+            options["timeseries_group_id"] = int(options["timeseries_group_id"])
+            options["timeseries_id"] = int(options["timeseries_id"])
+        except ValueError:
+            raise WrongValueError(
+                '"{}" or "{}" or "{}" is not a valid integer'.format(
+                    options["station_id"],
+                    options["timeseries_group_id"],
+                    options["timeseries_id"],
+                )
+            )
+        return options
+
+
+@click.command()
+@click.argument("configfile")
+@click.version_option(
+    version=__version__, message="%(prog)s from pthelma v.%(version)s"
+)
+def main(configfile):
+    """Spatial integration"""
+
+    app = App(configfile)
+    app.run()

enhydris_cache/enhydris_cache.py

@@ -0,0 +1,69 @@
+import datetime as dt
+
+import pandas as pd
+
+from enhydris_api_client import EnhydrisApiClient
+from htimeseries import HTimeseries
+
+
+class TimeseriesCache(object):
+    def __init__(self, timeseries_group):
+        self.timeseries_group = timeseries_group
+
+    def update(self):
+        for item in self.timeseries_group:
+            self.base_url = item["base_url"]
+            if self.base_url[-1] != "/":
+                self.base_url += "/"
+            self.station_id = item["station_id"]
+            self.timeseries_group_id = item["timeseries_group_id"]
+            self.timeseries_id = item["timeseries_id"]
+            self.auth_token = item["auth_token"]
+            self.filename = item["file"]
+            self._update_for_one_timeseries()
+
+    def _update_for_one_timeseries(self):
+        cached_ts = self._read_timeseries_from_cache_file()
+        end_date = self._get_timeseries_end_date(cached_ts)
+        start_date = end_date + dt.timedelta(minutes=1)
+        new_ts = self._append_newer_timeseries(start_date, cached_ts)
+        with open(self.filename, "w", encoding="utf-8") as f:
+            new_ts.write(f, format=HTimeseries.FILE)
+
+    def _read_timeseries_from_cache_file(self):
+        try:
+            with open(self.filename, newline="\n") as f:
+                return HTimeseries(f)
+        except (FileNotFoundError, ValueError):
+            # If file is corrupted or nonexistent, continue with empty time series
+            return HTimeseries()
+
+    def _get_timeseries_end_date(self, timeseries):
+        try:
+            end_date = timeseries.data.index[-1]
+        except IndexError:
+            # Timeseries is totally empty; no start and end date
+            end_date = dt.datetime(1, 1, 1, 0, 0, tzinfo=dt.timezone.utc)
+        return end_date
+
+    def _append_newer_timeseries(self, start_date, old_ts):
+        with EnhydrisApiClient(self.base_url, token=self.auth_token) as api_client:
+            ts = api_client.read_tsdata(
+                self.station_id,
+                self.timeseries_group_id,
+                self.timeseries_id,
+                start_date=start_date,
+            )
+            new_data = ts.data
+
+            # For appending to work properly, both time series need to have the same
+            # tz.
+            if len(old_ts.data):
+                new_data.index = new_data.index.tz_convert(old_ts.data.index.tz)
+            else:
+                old_ts.data.index = old_ts.data.index.tz_convert(new_data.index.tz)
+
+            ts.data = pd.concat(
+                [old_ts.data, new_data], verify_integrity=True, sort=False
+            )
+        return ts

evaporation/__init__.py

@@ -0,0 +1,4 @@
+from .evaporation import *  # NOQA
+
+__author__ = """Antonis Christofides"""
+__email__ = "antonis@antonischristofides.com"