tonik 0.1.10__py3-none-any.whl → 0.1.11__py3-none-any.whl

tonik/__init__.py

@@ -3,7 +3,7 @@ from os import PathLike
 from typing import Optional
 
 from .storage import Storage, Path
-from .utils import generate_test_data
+from .utils import generate_test_data, get_labels
 
 
 def get_data(filename: Optional[PathLike] = None) -> str:

tonik/api.py

@@ -10,7 +10,7 @@ import datashader as dsh
 import numpy as np
 import pandas as pd
 import uvicorn
-from cftime import date2num, num2date
+from cftime import date2num, num2pydate
 from fastapi import FastAPI, HTTPException, Query
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import HTMLResponse, StreamingResponse
@@ -37,6 +37,7 @@ class TonikAPI:
         self.app.get("/", response_class=HTMLResponse)(self.root)
         self.app.get("/feature")(self.feature)
         self.app.get("/inventory")(self.inventory)
+        self.app.get("/labels")(self.labels)
 
     def root(self):
         with open(get_data("package_data/index.html"), "r", encoding="utf-8") as file:
@@ -101,11 +102,12 @@ class TonikAPI:
             dates = np.tile(dates, freq.size)
             df = pd.DataFrame(
                 {'dates': dates, 'freqs': freqs, 'feature': vals})
+            df.dates = pd.to_datetime(df.dates.values).tz_localize('UTC')
             output = df.to_csv(index=False,
                                columns=['dates', 'freqs', 'feature'])
         else:
             df = pd.DataFrame(data=feat.to_pandas(), columns=[feat.name])
-            df['dates'] = df.index
+            df['dates'] = df.index.tz_localize('UTC')
             if resolution != 'full':
                 try:
                     current_resolution = pd.Timedelta(
@@ -138,7 +140,7 @@ class TonikAPI:
         agg = cvs.raster(source=feat)
         freq_dim = feat.dims[0]
         freq, d, spec = agg.coords[freq_dim].values, agg.coords['datetime'].values, agg.data
-        dates = num2date(
+        dates = num2pydate(
             d, units='hours since 1970-01-01 00:00:00.0', calendar='gregorian')
         return freq, dates, spec
 
@@ -156,8 +158,25 @@ class TonikAPI:
             return sg.to_dict()
         else:
             dir_contents = os.listdir(c.path)
+            if 'labels.json' in dir_contents:
+                dir_contents.remove('labels.json')
             return [fn.replace('.nc', '').replace('.zarr', '') for fn in dir_contents]
 
+    def labels(self, group: str, subdir: SubdirType = None, starttime: Optional[str] = None, endtime: Optional[str] = None):
+        _st = self.preprocess_datetime(starttime)
+        _et = self.preprocess_datetime(endtime)
+        sg = Storage(group, rootdir=self.rootdir,
+                     starttime=_st, endtime=_et, create=False)
+        try:
+            c = sg.get_substore(*subdir)
+        except TypeError:
+            c = sg
+        except FileNotFoundError:
+            msg = "Directory {} not found.".format(
+                '/'.join([sg.path] + subdir))
+            raise HTTPException(status_code=404, detail=msg)
+        return c.get_labels()
+
 
 def main(argv=None):
     parser = ArgumentParser()

tonik/storage.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import logging.config
 import os
@@ -157,6 +158,30 @@ class Path(object):
         with xr.open_dataset(filename, group='original', engine=self.engine) as ds:
             return ds[feature].sizes
 
+    def save_labels(self, labels):
+        """
+        Save all labels. Labels are stored in a list of dictionaries with the following keys:
+        {
+            'time': 'time, or the beginning of the time window',
+            'timeEnd': 'end of the time window [optional]',
+            'title': 'title of the label',
+            'text': 'A more detailed description of the label',
+            'tags': 'tags to sort labels',
+            'id': 'unique id of the label'
+        }
+        """
+        filename = os.path.join(self.path, 'labels.json')
+        with open(filename, 'w') as f:
+            json.dump(labels, f)
+
+    def get_labels(self):
+        """
+        Load all labels.
+        """
+        filename = os.path.join(self.path, 'labels.json')
+        with open(filename) as f:
+            return json.load(f)
+
 
 class Storage(Path):
     """
@@ -238,7 +263,10 @@ class Storage(Path):
         if name.endswith('.zarr'):
             return name.replace('.zarr', '')
         elif os.path.isdir(path):
-            return {name: [Storage.directory_tree_to_dict(os.path.join(path, child)) for child in sorted(os.listdir(path))]}
+            dir_contents = os.listdir(path)
+            if 'labels.json' in dir_contents:
+                dir_contents.remove('labels.json')
+            return {name: [Storage.directory_tree_to_dict(os.path.join(path, child)) for child in sorted(dir_contents)]}
         else:
             if name.endswith('.nc'):
                 return name.replace('.nc', '')

tonik/utils.py

@@ -1,4 +1,5 @@
-from datetime import datetime
+from typing import List
+from datetime import datetime, timezone, timedelta
 
 import numpy as np
 import pandas as pd
@@ -89,3 +90,100 @@ def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray,
             freq=f'{resolution}h')
         xda_new = xda_new.reindex(datetime=new_dates)
     return xda_new
+
+
+def extract_consecutive_integers(nums: List[int]) -> List[List[int]]:
+    """
+    Extract consecutive integers from a list of integers.
+    """
+    if not len(nums) > 0:
+        return []
+
+    nums.sort()  # Sort the array
+    result = []
+    temp = [nums[0]]  # Initialize the first group with the first number
+
+    for i in range(1, len(nums)):
+        if nums[i] == nums[i - 1] + 1:  # Check if consecutive
+            temp.append(nums[i])
+        else:
+            result.append(temp)  # Add the current group to the result
+            temp = [nums[i]]  # Start a new group
+
+    result.append(temp)  # Add the last group
+    return result
+
+
+def get_labels(xda: xr.DataArray, threshold: float) -> dict:
+    """
+    Generate labels for time windows where the values are exceeding the threshold.
+    """
+    # add labels to time windows where the values are exceeding the 85th percentile
+    labels = {xda.name: []}
+    idx = np.where(xda > threshold)[0]
+    window_list = extract_consecutive_integers(idx)
+    ids = 0
+    for _w in window_list:
+        timeEnd = None
+        # Convert to unix time milliseconds
+        timeStart = xda.datetime.isel(dict(datetime=_w[0])).values.astype(
+            'datetime64[ms]').astype(int)
+        timeStart = int(timeStart)
+        if len(_w) > 1:
+            timeEnd = xda.datetime.isel(
+                dict(datetime=_w[-1])).values.astype('datetime64[ms]').astype(int)
+            timeEnd = int(timeEnd)
+        label = dict(time=timeStart,
+                     timeEnd=timeEnd,
+                     title='Greater 85th percentile',
+                     description='Values exceed 85th percentile',
+                     tags=['anomaly'],
+                     id=ids)
+        ids += 1
+        labels[xda.name].append(label)
+    return labels
+
+
+def main():
+    from tonik import Storage
+    import logging
+
+    logger = logging.getLogger(__name__)
+    logger.info("Generating test data")
+    rootdir = '/tmp'
+    g = Storage('volcanoes', rootdir=rootdir)
+    st1 = g.get_substore('Mt Doom', 'MDR', '00', 'BHZ')
+    st2 = g.get_substore('Misty Mountain', 'MMS', '10', 'HHZ')
+
+    # Get start time of the current 10 minute window
+    tstart = datetime.now(timezone.utc).timestamp()
+    tstart -= tstart % 600
+    tstart = datetime.fromtimestamp(tstart, timezone.utc)
+    tstart -= timedelta(days=30)
+    tstart = tstart.replace(tzinfo=None)
+    logger.info(f"Start time: {tstart}")
+
+    # Generate test data
+    xdf_1D = generate_test_data(dim=1, tstart=tstart, add_nans=False, seed=42)
+    xdf_1D_1 = generate_test_data(
+        dim=1, tstart=tstart, add_nans=False, seed=24)
+    xdf_2D = generate_test_data(
+        dim=2, tstart=tstart, add_nans=False, seed=1234)
+    xdf_2D_1 = generate_test_data(
+        dim=2, tstart=tstart, add_nans=False, seed=4321)
+    logger.info("Saving test data to " + st1.path)
+    st1.save(xdf_1D)
+    st1.save(xdf_2D)
+    logger.info("Saving test data to " + st2.path)
+    st2.save(xdf_1D_1)
+    st2.save(xdf_2D_1)
+
+    # add labels to RSAM time windows where the values are exceeding the 85th percentile
+    logger.info("Adding labels to DSAR data")
+    labels = get_labels(xdf_1D.dsar, float(xdf_1D.dsar.quantile(0.85)))
+    logger.info("Saving labels to " + st1.path + '/labels.json')
+    st1.save_labels(labels)
+
+
+if __name__ == '__main__':
+    main()

tonik/xarray2netcdf.py

@@ -103,7 +103,7 @@ def _create_h5_Structure(defaultGroupName, featureName, h5f, xArray, starttime,
     coordinates = rootGrp.create_variable(timedim, (timedim,), float)
     coordinates.attrs['units'] = 'hours since 1970-01-01 00:00:00.0'
     coordinates.attrs['calendar'] = 'gregorian'
-    rootGrp.attrs['starttime'] = str(starttime)
+    rootGrp.attrs['archive_starttime'] = str(starttime)
     for label, size in xArray.sizes.items():
         if not np.issubdtype(xArray[label].dtype, np.datetime64):
             rootGrp.dimensions[label] = size

{tonik-0.1.10.dist-info → tonik-0.1.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: tonik
-Version: 0.1.10
+Version: 0.1.11
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues
@@ -10,6 +10,7 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.7
+Requires-Dist: dask<=2024.10.0
 Requires-Dist: datashader>=0.14
 Requires-Dist: fastapi>=0.112
 Requires-Dist: h5netcdf>=1.1
@@ -30,8 +31,8 @@ Description-Content-Type: text/markdown
 
 # Tonik
 
-Tonik provides you with a solution to store and retrieve scientific data as well as serving it through an API.
-For visualisations, the API can serve large requests very quickly by downsampling the data to the requested resolution on demand.
+Tonik provides you with a solution to store and retrieve scientific time-series data as well as serving it through an API.
+For visualisations, the API can serve large requests very quickly by downsampling the data to the requested resolution on demand. The API was optimised to visualise time-series and data labels with [Grafana](https://grafana.com/oss/grafana/).
 
 ## Requirements
 * h5py
@@ -54,10 +55,6 @@ pip install -U tonik
 
 Learn more about tonik in its official [documentation](https://tsc-tools.github.io/tonik)
 
-## Contributing
-
-You can find information about contributing to tonik at our [Contributing page]
-
 ## Get in touch
 
-Report bugs, suggest features, view the source code, and ask questions [on GitHub](https://github.com/tsc-tools/tonik).
+Report bugs, suggest features, view the source code, and ask questions [on GitHub](https://github.com/tsc-tools/tonik/issues).

tonik-0.1.11.dist-info/RECORD

@@ -0,0 +1,12 @@
+tonik/__init__.py,sha256=dov-nMeGFBzLspmj4rWKjC4r736vmaPDgMEkHSUfP98,523
+tonik/api.py,sha256=xUH-fr-xUwc_a21QYz11Dk3YjB2nRuCclmALvU64UJM,7592
+tonik/storage.py,sha256=IklM_atZD4rebUsnXsUj5JldSHU2LqmuqME03PHp_UI,10441
+tonik/utils.py,sha256=9eSVKIbs8TIZlJCz_-B7FrvOUQCQHO3K52v4Heus-uE,6135
+tonik/xarray2netcdf.py,sha256=gDNT6nxnRbXPeRqZ3URW5oXY3Nfh3TCrfueE-eUrIoY,5181
+tonik/xarray2zarr.py,sha256=xJjKcFZF0oz6gw47apuCiXFtW5HgWqnZgiIuEVQHhBI,2363
+tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
+tonik-0.1.11.dist-info/METADATA,sha256=xEhlRUeS79ZRxwL_-ksmf5do2OrOapKBealQRPUSFpE,2005
+tonik-0.1.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+tonik-0.1.11.dist-info/entry_points.txt,sha256=mT3B4eBE8SHlAeMhFnZGor9-YkVtoWM1NVHVuypJ-uY,74
+tonik-0.1.11.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tonik-0.1.11.dist-info/RECORD,,

{tonik-0.1.10.dist-info → tonik-0.1.11.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.25.0
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any

{tonik-0.1.10.dist-info → tonik-0.1.11.dist-info}/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+test_data = tonik.utils:main
 tonik_api = tonik.api:main

tonik-0.1.10.dist-info/RECORD

@@ -1,12 +0,0 @@
-tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
-tonik/api.py,sha256=JaKHzavqYDjZo2DypTHgAaM01Y4EOkn6UF3ZVm50F5k,6694
-tonik/storage.py,sha256=GNJ6w9VHOeTR_ZJMZ-Ipqe3nFK2I91fkHYwg1k9bEuo,9470
-tonik/utils.py,sha256=3nSRU_GnV6arP4e63YHn4oEV8XbqzVAW8FCvQVIwGdg,2757
-tonik/xarray2netcdf.py,sha256=RUsPwPDEOnUIMGt7_9F1VDyg83zu8x-Qs3jmZs3Dq0o,5173
-tonik/xarray2zarr.py,sha256=xJjKcFZF0oz6gw47apuCiXFtW5HgWqnZgiIuEVQHhBI,2363
-tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
-tonik-0.1.10.dist-info/METADATA,sha256=gDvofXsP2pnRFDcwYzCd67dZ6BC0GGBDtdC_sn71k7o,1940
-tonik-0.1.10.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-tonik-0.1.10.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
-tonik-0.1.10.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-tonik-0.1.10.dist-info/RECORD,,