tonik 0.0.1__tar.gz → 0.0.2__tar.gz

tonik-0.0.2/.devcontainer/devcontainer.json

@@ -0,0 +1,22 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Python 3",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:1-3.10-bullseye"
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

{tonik-0.0.1 → tonik-0.0.2}/.pytest_cache/v/cache/lastfailed

@@ -5,5 +5,8 @@
   "lib64/python3.10/site-packages/h5py/tests": true,
   "lib64/python3.10/site-packages/numpy": true,
   "lib64/python3.10/site-packages/pandas": true,
-  "lib64/python3.10/site-packages/xarray/tests": true
+  "lib64/python3.10/site-packages/xarray/tests": true,
+  "tests/test_xarray2hdf5.py::test_xarray2hdf5": true,
+  "tests/test_xarray2hdf5.py::test_xarray2hdf5_with_gaps": true,
+  "tests/test_xarray2hdf5.py::test_xarray2hdf5_multi_access": true
 }

{tonik-0.0.1 → tonik-0.0.2}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.1
 Name: tonik
-Version: 0.0.1
+Version: 0.0.2
 Summary: A collection of tools to integrate with GNS Science's time series classification platform.
-Project-URL: Homepage, https://github.com/tsc-tools/tonik
+Project-URL: Homepage, https://tsc-tools.github.io/tonik.github.io
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues
 Author-email: Yannik Behr <y.behr@gns.cri.nz>, Christof Mueller <c.mueller@gns.cri.nz>
 License-File: LICENSE
@@ -15,6 +15,7 @@ Requires-Dist: h5py
 Requires-Dist: netcdf4
 Requires-Dist: pandas
 Requires-Dist: pytest
+Requires-Dist: python-json-logger
 Requires-Dist: xarray
 Description-Content-Type: text/markdown
 

tonik-0.0.2/docs/user_guide.md

@@ -0,0 +1,53 @@
+## Examples on how to store and retrieve data using lockerroom
+
+```python
+from datetime import datetime
+import os
+import numpy as np
+import pandas as pd
+import xarray as xr
+from tonik import Locker, LockerRoom
+```
+
+A locker room consists of many lockers. Each locker stores data for a single sensor and the locker room groups sensors by, for example, experiment or geographic location. We will start by generating fake spectrogram data.
+
+```python
+dates = pd.date_range("2024-01-02", freq='10min', periods=288)
+data = np.abs(np.cumsum(np.random.normal(0, 8., len(dates))))
+data = np.tile(data, (10, 1))
+freqs = np.arange(10)
+xrd = xr.Dataset({'spectrogram': xr.DataArray(data, coords=[freqs, dates],
+                                              dims=['frequency', 'datetime'])})
+xrd['spectrogram'].plot()
+```
+
+![1st spectrogram](img/spectrogram1.png)
+
+Now we will store the data under two different sites of the same experiment.
+
+```python
+lr = LockerRoom('test_experiment')
+l1 = lr.get_locker(site='MDR1', location='00', channel='HHZ')
+l2 = lr.get_locker(site='MDR2', location='00', channel='HHZ')
+l1.save(xrd)
+l2.save(xrd)
+lr
+```
+
+![lockerroom output](img/lockerroom_output.png)
+
+```
+!tree /tmp/test_experiment
+```
+![directory listing](img/directory_listing.png)
+
+Next we want to retrieve the data we just saved. Before retrieving data we have to set the timespan over which we want to retrieve data.
+
+```python
+lr.starttime = datetime(2024, 1, 2, 18, 0, 0)
+lr.endtime = datetime(2024, 1, 3, 6, 0, 0)
+l = lr.get_locker('MDR2', '00', 'HHZ')
+l('spectrogram').plot()
+```
+
+![2nd spectrogram](img/spectrogram2.png)

tonik-0.0.2/examples/tonik_example.ipynb

@@ -0,0 +1,117 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Examples on how to store and retrieve data using lockerroom"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datetime import datetime\n",
+    "import os\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import xarray as xr\n",
+    "from tonik import Locker, LockerRoom"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "A locker room consists of many lockers. Each locker stores data for a single sensor and the locker room groups sensors by, for example, experiment or geographic location. We will start by generating fake spectrogram data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dates = pd.date_range(\"2024-01-02\", freq='10min', periods=288)\n",
+    "data = np.abs(np.cumsum(np.random.normal(0, 8., len(dates))))\n",
+    "data = np.tile(data, (10, 1))\n",
+    "freqs = np.arange(10)\n",
+    "xrd = xr.Dataset({'spectrogram': xr.DataArray(data, coords=[freqs, dates],\n",
+    "                                              dims=['frequency', 'datetime'])})\n",
+    "fig = xrd['spectrogram'].plot()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now we will store the data under two different sites of the same experiment."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lr = LockerRoom('test_experiment')\n",
+    "l1 = lr.get_locker(site='MDR1', location='00', channel='HHZ')\n",
+    "l2 = lr.get_locker(site='MDR2', location='00', channel='HHZ')\n",
+    "l1.save(xrd)\n",
+    "l2.save(xrd)\n",
+    "lr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!tree /tmp/test_experiment"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Next we want to retrieve the data we just saved. Before retrieving data we have to set the timespan over which we want to retrieve data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lr.starttime = datetime(2024, 1, 2, 18, 0, 0)\n",
+    "lr.endtime = datetime(2024, 1, 3, 6, 0, 0)\n",
+    "l = lr.get_locker('MDR2', '00', 'HHZ')\n",
+    "l('spectrogram').plot()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tonik-0.0.2/mkdocs.yml

@@ -0,0 +1,5 @@
+site_name: Tonik 
+nav:
+  - Home: index.md
+  - User Guide: user_guide.md
+theme: readthedocs

{tonik-0.0.1 → tonik-0.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "tonik"
-version = "0.0.1"
+version = "0.0.2"
 authors = [
   { name="Yannik Behr", email="y.behr@gns.cri.nz" },
   { name="Christof Mueller", email="c.mueller@gns.cri.nz" }
@@ -24,9 +24,10 @@ dependencies = [
     "pandas",
     "netcdf4",
     "h5netcdf",
-    "pytest"
+    "pytest",
+    "python-json-logger"
 ]
 
 [project.urls]
-Homepage = "https://github.com/tsc-tools/tonik"
+Homepage = "https://tsc-tools.github.io/tonik.github.io"
 Issues = "https://github.com/tsc-tools/tonik/issues"

tonik-0.0.2/src/tonik/__init__.py

@@ -0,0 +1 @@
+from .lockerroom import Locker, LockerRoom

tonik-0.0.2/src/tonik/lockerroom.py

@@ -0,0 +1,290 @@
+from datetime import datetime, timedelta
+import logging
+import logging.config
+import os
+import re
+import tempfile
+
+import pandas as pd
+import xarray as xr
+
+from .xarray2hdf5 import xarray2hdf5
+
+
+ERROR_LOG_FILENAME = "tonik.log"
+
+LOGGING_CONFIG = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {      
+        "default": {  # The formatter name, it can be anything that I wish
+            "format": "%(asctime)s:%(name)s:%(process)d:%(lineno)d " "%(levelname)s %(message)s",  #  What to add in the message
+            "datefmt": "%Y-%m-%d %H:%M:%S",  # How to display dates
+        },
+        "json": {  # The formatter name
+         "()": "pythonjsonlogger.jsonlogger.JsonFormatter",  # The class to instantiate!
+            # Json is more complex, but easier to read, display all attributes!
+            "format": """
+                    asctime: %(asctime)s
+                    created: %(created)f
+                    filename: %(filename)s
+                    funcName: %(funcName)s
+                    levelname: %(levelname)s
+                    levelno: %(levelno)s
+                    lineno: %(lineno)d
+                    message: %(message)s
+                    module: %(module)s
+                    msec: %(msecs)d
+                    name: %(name)s
+                    pathname: %(pathname)s
+                    process: %(process)d
+                    processName: %(processName)s
+                    relativeCreated: %(relativeCreated)d
+                    thread: %(thread)d
+                    threadName: %(threadName)s
+                    exc_info: %(exc_info)s
+                """,
+            "datefmt": "%Y-%m-%d %H:%M:%S",  # How to display dates
+        },
+    }, 
+    "handlers": {
+        "logfile": {  # The handler name
+            "formatter": "json",  # Refer to the formatter defined above
+            "level": "ERROR",  # FILTER: Only ERROR and CRITICAL logs
+            "class": "logging.handlers.RotatingFileHandler",  # OUTPUT: Which class to use
+            "filename": ERROR_LOG_FILENAME,  # Param for class above. Defines filename to use, load it from constant
+            "backupCount": 2,  # Param for class above. Defines how many log files to keep as it grows
+        }, 
+        "simple": {  # The handler name
+            "formatter": "default",  # Refer to the formatter defined above
+            "class": "logging.StreamHandler",  # OUTPUT: Same as above, stream to console
+            "stream": "ext://sys.stdout",
+        },
+    },
+    "loggers": { 
+        "zizou": {  # The name of the logger, this SHOULD match your module!
+            "level": "DEBUG",  # FILTER: only INFO logs onwards from "tryceratops" logger
+            "handlers": [
+                "simple",  # Refer the handler defined above
+            ],
+        },
+    },
+    "root": {
+        "level": "ERROR",  # FILTER: only INFO logs onwards
+        "handlers": [
+            "logfile",  # Refer the handler defined above
+        ]
+    },
+}
+
+logging.config.dictConfig(LOGGING_CONFIG)
+logger = logging.getLogger("__name__")
+
+
+class LockerRoom:
+    """
+    Query computed features
+
+    :param rootdir: Path to parent directory.
+    :type rootdir: str
+    :param starttime: Begin of request
+    :type starttime: :class:`datetime.datetime`
+    :param endtime: Begin of request
+    :type endtime: :class:`datetime.datetime`
+
+    >>> import datetime
+    >>> fq = FeatureRequest()
+    >>> start = datetime.datetime(2012,1,1,0,0,0)
+    >>> end = datetime.datetime(2012,1,2,23,59,59)
+    >>> group = 'Whakaari'
+    >>> site = 'WIZ'
+    >>> chan = 'HHZ'
+    >>> fq.group = group
+    >>> fq.starttime = start
+    >>> fq.endtime = end
+    >>> fq.site = site
+    >>> fq.channel = chan
+    >>> rsam = fq("rsam")
+    """
+    def __init__(self, group, rootdir=tempfile.gettempdir(),
+                 starttime=None, endtime=None):
+        self.groupdir = os.path.join(rootdir, group)
+        self.lockers = {} 
+
+    def get_locker(self, site, location, channel):
+        key = (site, location, channel)
+        if key not in self.lockers:
+            self.lockers[key] = Locker(site, location, channel, rootdir=self.groupdir)
+        return self.lockers[key]
+
+    def __repr__(self):
+        rstr = f"LockerRoom: {self.groupdir}\n"
+        for site, location, channel in self.lockers.keys():
+            rstr += f"Site: {site}, Location: {location}, Channel: {channel}\n" 
+        return rstr
+
+    def get_starttime(self):
+        return self.__starttime
+
+    def set_starttime(self, time):
+        if time is None:
+            self.__starttime = None
+            self.__sdate = None
+            return
+        self.__starttime = time
+        self.__sdate = '{}{:02d}{:02d}'.format(time.year,
+                                               time.month,
+                                               time.day)
+        for key, locker in self.lockers.items():
+            locker.starttime = time
+
+    def get_endtime(self):
+        return self.__endtime
+
+    def set_endtime(self, time):
+        if time is None:
+            self.__endtime = None
+            self.__edate = None
+            return
+        self.__endtime = time
+        self.__edate = '{}{:02d}{:02d}'.format(time.year,
+                                               time.month,
+                                               time.day)
+        for key, locker in self.lockers.items():
+            locker.endtime = time
+
+    starttime = property(get_starttime, set_starttime)
+    endtime = property(get_endtime, set_endtime)
+
+
+class Locker:
+    def __init__(self, site=None, location=None, channel=None,
+                 rootdir=None, starttime=None, endtime=None,
+                 interval='10min'):
+
+        self.site = site
+        self.location = location
+        self.channel = channel
+        self.starttime = starttime
+        self.endtime = endtime
+        self.rootdir = rootdir
+        self.interval = interval
+
+    def __call__(self, feature, stack_length=None):
+        """
+        Request a particular feature
+
+        :param feature: Feature name
+        :type feature: str
+        :param stack_length: length of moving average in time
+        :type stack_length: str
+
+        """
+        if self.endtime <= self.starttime:
+            raise ValueError('Startime has to be smaller than endtime.')
+
+        feature = feature.lower()
+        filename = os.path.join(self.sitedir, '%s.nc' % feature)
+        if not os.path.isfile(filename):
+            raise ValueError('Feature {} does not exist.'.format(feature))
+
+        logger.debug(f"Reading feature {feature} between {self.starttime} and {self.endtime}")
+        num_periods = None
+        if stack_length is not None:
+            valid_stack_units = ['W', 'D', 'H', 'T', 'min', 'S']
+            if not re.match(r'\d*\s*(\w*)', stack_length).group(1)\
+                   in valid_stack_units:
+                raise ValueError(
+                    'Stack length should be one of: {}'.
+                        format(', '.join(valid_stack_units))
+                )
+
+            if pd.to_timedelta(stack_length) < pd.to_timedelta(self.interval):
+                raise ValueError('Stack length {} is less than interval {}'.
+                                 format(stack_length, self.interval))
+
+            # Rewind starttime to account for stack length
+            self.starttime -= pd.to_timedelta(stack_length)
+
+            num_periods = (pd.to_timedelta(stack_length)/
+                           pd.to_timedelta(self.interval))
+            if not num_periods.is_integer():
+                raise ValueError(
+                    'Stack length {} / interval {} = {}, but it needs'
+                    ' to be a whole number'.
+                        format(stack_length, self.interval, num_periods))
+
+        xd_index = dict(datetime=slice(self.starttime,
+                                       (self.endtime-
+                                        pd.to_timedelta(self.interval))))
+        with xr.open_dataset(filename, group='original', engine='h5netcdf') as ds:
+            ds.sortby("datetime")
+            rq = ds.loc[xd_index].load()
+
+        # Stack features
+        if stack_length is not None:
+            logger.debug("Stacking feature...")
+            try:
+                xdf = rq[feature].rolling(datetime=int(num_periods),
+                                        center=False,
+                                        min_periods=1).mean()
+                # Return requested timeframe to that defined in initialisation
+                self.starttime += pd.to_timedelta(stack_length)
+                xdf_new = xdf.loc[
+                        self.starttime:
+                        self.endtime-pd.to_timedelta(self.interval)]
+                xdf_new = xdf_new.rename(feature)
+            except ValueError as e:
+                logger.error(e)
+                logger.error('Stack length {} is not valid for feature {}'.
+                             format(stack_length, feature))
+            else:
+                return xdf_new
+
+        return rq[feature]
+
+    def get_site(self):
+        return self.__site
+
+    def set_site(self, value):
+        self.__site = value
+
+    def get_location(self):
+        return self.__location
+
+    def set_location(self, value):
+        self.__location = value
+
+    def get_channel(self):
+        return self.__channel
+
+    def set_channel(self, value):
+        self.__channel = value
+
+    @property
+    def sitedir(self):
+        try:
+            __sdir =  os.path.join(self.rootdir,
+                                   self.site,
+                                   self.location,
+                                   self.channel)
+            os.makedirs(__sdir, exist_ok=True)
+            return __sdir
+        except TypeError:
+            return None
+
+    site = property(get_site, set_site)
+    location = property(get_location, set_location)
+    channel = property(get_channel, set_channel)
+
+    def load(self, *args, **kwargs):
+        """
+        Load a feature from disk
+        """
+        self.__call__(*args, **kwargs)
+
+    def save(self, data):
+        """
+        Save a feature to disk
+        """
+        xarray2hdf5(data, self.sitedir)

{tonik-0.0.1 → tonik-0.0.2}/src/tonik/xarray2hdf5.py

@@ -3,10 +3,9 @@ import logging
 import os
 from warnings import filterwarnings
 
-from cftime import num2date, date2num, date2index
+from cftime import num2date, date2num
 import h5netcdf
 import numpy as np
-import xarray as xr
 
 
 def xarray2hdf5(xArray, fdir, rootGroupName="original", timedim="datetime"):

{tonik-0.0.1 → tonik-0.0.2}/tests/test_xarray2hdf5.py

@@ -6,7 +6,7 @@ import pandas as pd
 import pytest
 import xarray as xr
 
-from tsctools import xarray2hdf5
+from tonik import xarray2hdf5
 
 
 def generate_test_data(dim=1, nhours=30, nfreqs=10,

tonik-0.0.1/src/tonik/__init__.py

@@ -1 +0,0 @@
-from .xarray2hdf5 import xarray2hdf5