tonik 0.1.15__tar.gz → 0.1.17__tar.gz

{tonik-0.1.15 → tonik-0.1.17}/.devcontainer/devcontainer.json

@@ -1,9 +1,9 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/python
 {
-	"name": "Python 3",
+	"name": "Tonik",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/python:1-3.9-bullseye",
+	"image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
 
 
 	// Features to add to the dev container. More info: https://containers.dev/features.

{tonik-0.1.15 → tonik-0.1.17}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: tonik
-Version: 0.1.15
+Version: 0.1.17
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues
@@ -18,10 +18,11 @@ Requires-Dist: matplotlib
 Requires-Dist: netcdf4>=1.6
 Requires-Dist: pandas>=2.0
 Requires-Dist: python-json-logger>=2.0
+Requires-Dist: s3fs
 Requires-Dist: uvicorn[standard]>=0.22
 Requires-Dist: xarray[accel,io,parallel]
-Requires-Dist: zarr<3; python_version < '3.11'
-Requires-Dist: zarr>=3.0.3; python_version >= '3.11'
+Requires-Dist: zarr[remote-tests]<3; python_version < '3.11'
+Requires-Dist: zarr[remote-tests]>=3.0.3; python_version >= '3.11'
 Provides-Extra: dev
 Requires-Dist: httpx; extra == 'dev'
 Requires-Dist: ipykernel; extra == 'dev'

{tonik-0.1.15 → tonik-0.1.17}/grafana_example/docker-compose.yml

@@ -10,7 +10,7 @@ services:
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=tonikdemo
     ports:
-      - "11223:3000"
+      - "11224:3000"
     networks:
       - tonik_network
 
@@ -21,7 +21,7 @@ services:
     container_name: tonik_example_api
     image: tonik_example_api
     ports:
-      - "11222:8003"
+      - "11225:8003"
     networks:
       - tonik_network
 

tonik-0.1.17/pyproject.toml

@@ -0,0 +1,76 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+  "/.github",
+  "/site",
+  "/docs",
+  "/sandbox.ipynb"
+]
+
+[project]
+name = "tonik"
+version = "0.1.17"
+authors = [
+  { name="Yannik Behr", email="y.behr@gns.cri.nz" },
+  { name="Christof Mueller", email="c.mueller@gns.cri.nz" }
+]
+
+description = "Store time series data as HDF5 files and access them through an API."
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "h5py>=3.8",
+    "datashader>=0.14",
+    "xarray[io,accel,parallel]",
+    "pandas>=2.0",
+    "netcdf4>=1.6",
+    "h5netcdf>=1.1",
+    "python-json-logger>=2.0",
+    "uvicorn[standard]>=0.22",
+    "fastapi>=0.112",
+    "matplotlib",
+    "zarr[remote_tests]>=3.0.3; python_version >= '3.11'",
+    "zarr[remote_tests]<3; python_version < '3.11'",
+    "s3fs"
+]
+
+[project.optional-dependencies]
+dev = ["pytest",
+       "httpx",
+       "ipykernel",
+       "mkdocs",
+       "mkdocstrings[python]",
+       "mkdocs-jupyter"]
+
+[project.urls]
+Homepage = "https://tsc-tools.github.io/tonik"
+Issues = "https://github.com/tsc-tools/tonik/issues"
+
+[project.scripts]
+tonik_api = "tonik.api:main"
+test_data = "tonik.utils:main"
+grafana_annotations = "tonik.grafana_annotations:main"
+
+[tool.pytest.ini_options]
+log_cli = true
+
+[tool.hatch.envs.test]
+dependencies = [
+  "coverage[toml]",
+  "pytest",
+  "httpx"
+]
+
+[[tool.hatch.envs.test.matrix]]
+python = ["3.11", "3.9"]
+
+[tool.hatch.envs.test.scripts]
+run-pytest = "pytest tests"

tonik-0.1.15/pyproject.toml → tonik-0.1.17/pyproject.toml~

@@ -12,7 +12,7 @@ exclude = [
 
 [project]
 name = "tonik"
-version = "0.1.15"
+version = "0.1.16"
 authors = [
   { name="Yannik Behr", email="y.behr@gns.cri.nz" },
   { name="Christof Mueller", email="c.mueller@gns.cri.nz" }
@@ -37,8 +37,9 @@ dependencies = [
     "uvicorn[standard]>=0.22",
     "fastapi>=0.112",
     "matplotlib",
-    "zarr>=3.0.3; python_version >= '3.11'",
-    "zarr<3; python_version < '3.11'",
+    "zarr[remote_tests]>=3.0.3; python_version >= '3.11'",
+    "zarr[remote_tests]<3; python_version < '3.11'",
+    "s3fs"
 ]
 
 [project.optional-dependencies]
@@ -56,6 +57,7 @@ Issues = "https://github.com/tsc-tools/tonik/issues"
 [project.scripts]
 tonik_api = "tonik.api:main"
 test_data = "tonik.utils:main"
+grafana_annotations = "tonik.grafana_annotations:main"
 
 [tool.pytest.ini_options]
 log_cli = true

{tonik-0.1.15 → tonik-0.1.17}/src/tonik/api.py

@@ -26,8 +26,9 @@ InventoryReturnType = Union[list, dict]
 
 class TonikAPI:
 
-    def __init__(self, rootdir) -> None:
+    def __init__(self, rootdir, backend='netcdf') -> None:
         self.rootdir = rootdir
+        self.backend = backend
         self.app = FastAPI()
 
         # -- allow any origin to query API
@@ -72,7 +73,7 @@ class TonikAPI:
         _et = self.preprocess_datetime(endtime)
         g = Storage(group, rootdir=self.rootdir,
                     starttime=_st, endtime=_et,
-                    create=False)
+                    create=False, backend=self.backend)
         c = g
         if subdir:
             c = g.get_substore(*subdir)
@@ -147,7 +148,8 @@ class TonikAPI:
         return freq, dates, spec
 
     async def inventory(self, group: str, subdir: SubdirType = None, tree: bool = True) -> InventoryReturnType:
-        sg = Storage(group, rootdir=self.rootdir, create=False)
+        sg = Storage(group, rootdir=self.rootdir,
+                     create=False, backend=self.backend)
         try:
             c = sg.get_substore(*subdir)
         except TypeError:
@@ -168,7 +170,8 @@ class TonikAPI:
         _st = self.preprocess_datetime(starttime)
         _et = self.preprocess_datetime(endtime)
         sg = Storage(group, rootdir=self.rootdir,
-                     starttime=_st, endtime=_et, create=False)
+                     starttime=_st, endtime=_et, create=False,
+                     backend=self.backend)
         try:
             c = sg.get_substore(*subdir)
         except TypeError:
@@ -183,10 +186,11 @@ class TonikAPI:
 def main(argv=None):
     parser = ArgumentParser()
     parser.add_argument("--rootdir", default='/tmp')
+    parser.add_argument("--backend", default='netcdf')
     parser.add_argument("-p", "--port", default=8003, type=int)
     parser.add_argument("--host", default='0.0.0.0')
     args = parser.parse_args(argv)
-    ta = TonikAPI(args.rootdir)
+    ta = TonikAPI(args.rootdir, backend=args.backend)
     uvicorn.run(ta.app, host=args.host, port=args.port)
 
 

tonik-0.1.17/src/tonik/grafana_annotations.py

@@ -0,0 +1,152 @@
+"""
+Post labels as Grafana annotations and retrieve them via the Grafana API.
+Labels are assumed to be in the following format:
+
+[
+    {"time": "2023-01-01T00:00:00Z",
+     "title": "Label 1",
+     "description": "Some description",
+     "tags": ["tag1", "tag2"],
+     "id": 12345
+     },
+    {"time": "2023-02-01T00:00:00Z",
+     "timeEnd": "2023-02-01T01:00:00Z",
+     "title": "Label 2", description:
+     "description": "Another description",
+     "tags": ["tag1", "tag2"],
+     "id": 12346
+     },
+     ...
+]
+
+Note: The `timeEnd` field is optional. If it is provided, the grafana annotation will 
+be a range annotation, otherwise it will be a point annotation. The label id is used to
+check if an annotation was already added to Grafana. If the label id is not present in the
+annotations, a new annotation will be created. If the label id is present, the annotation
+will be skipped to avoid duplicates.
+"""
+
+import numpy as np
+import requests
+import uuid
+from typing import List, Dict, Any, Optional
+
+
+def get_annotations(baseurl: str, tags: Optional[List[str]] = None) -> Dict[str, Dict[str, Any]]:
+    """
+    Get existing annotations from the Grafana API.
+
+    Parameters
+    ----------
+    baseurl : str
+        The base URL of the Grafana API. This is typically in the format
+        "http://<grafana-user>:<user-pwd>@<grafana-host>:<port>".
+    tags : list of str, optional
+        A list of tags to filter the annotations. If None, all annotations are retrieved.
+    Returns
+    -------
+    annotations : dict
+        A dictionary containing the annotations, where the keys are the label IDs
+        and the values are dictionaries with the following keys:
+            - 'text': The text of the annotation.
+            - 'time': The start time of the annotation in milliseconds since epoch.
+            - 'timeEnd': (optional) The end time of the annotation in milliseconds since epoch.
+            - 'tags': A list of tags associated with the annotation.
+            - 'id': The ID of the annotation.
+    """
+    url = baseurl + "/api/annotations"
+    params = {}
+    if tags is not None:
+        params = {"tags": tags}
+    rval = requests.get(url, params=params)
+    if rval.status_code != 200:
+        raise RuntimeError(
+            f"Failed to retrieve annotations: {rval.status_code} {rval.text}")
+    annotations = {}
+    for atn in rval.json():
+        # Extract label ID from text using regex
+        try:
+            label_id = atn["text"].split("Id: ")[-1].strip()
+        except IndexError:
+            label_id = str(uuid.uuid4())
+        annotations[label_id] = {"text": atn["text"],
+                                 "time": atn["time"],
+                                 "timeEnd": atn.get("timeEnd", None),
+                                 "tags": atn.get("tags", []),
+                                 "id": atn["id"]}
+    return annotations
+
+
+def post_annotations(baseurl: str, labels: List[Dict[str, Any]]) -> None:
+    """
+    Post annotations from a list of labels if they do not already exist.
+
+    Parameters
+    ----------
+    baseurl : str
+        The base URL of the Grafana API. This is typically in the format
+        "http://<grafana-user>:<user-pwd>@<grafana-host>:<port>".
+    labels : list of dict
+        A list of dictionaries containing label information. Each dictionary
+        should have the following keys:
+            - 'time': The start time of the annotation in ISO format.
+            - 'timeEnd': (optional) The end time of the annotation in ISO format.
+            - 'title': The title of the annotation.
+            - 'description': The description of the annotation.
+            - 'tags': A list of tags associated with the annotation.
+            - 'id': A unique identifier for the label, used to check for duplicates.
+    """
+    url = baseurl + "/api/annotations"
+    header = {"Content-type": "application/json"}
+    for label in labels:
+        existing_labels = get_annotations(baseurl, tags=label['tags'])
+        if str(label['id']) in existing_labels:
+            print(
+                f"Label with ID {label['id']} already exists. Skipping post.")
+            continue
+        starttime = np.datetime64(label['time']).astype(
+            'datetime64[ms]').astype(int)
+        try:
+            endtime = np.datetime64(label['timeEnd']).astype(
+                'datetime64[ms]').astype(int)
+            endtime = int(endtime)
+        except KeyError:
+            endtime = None
+        text = f"{label['title']}\n{label['description']}\nId: {label['id']}"
+        new_annotation = {
+            "time": int(starttime),
+            "timeEnd": endtime,
+            "text": text,
+            "tags": label['tags']
+        }
+        rval = requests.post(url, headers=header, json=new_annotation)
+        if rval.status_code != 200:
+            raise RuntimeError(
+                f"Failed to post annotation: {rval.status_code} {rval.text}")
+        else:
+            print("Annotation posted successfully.")
+
+
+def main(argv=None):
+    import argparse
+    import json
+    parser = argparse.ArgumentParser(
+        description=__doc__)
+    parser.add_argument('url', type=str,
+                        help='Grafana API URL for annotations.')
+    parser.add_argument('--labels', type=str, default=None,
+                        help='Path to JSON file containing labels to post.')
+    parser.add_argument('--get-annotations', action='store_true',
+                        help='Get existing annotations from Grafana API.')
+    args = parser.parse_args(argv)
+    if args.labels is not None:
+        with open(args.labels, 'r') as f:
+            labels = json.load(f)
+        post_annotations(args.url, labels)
+    elif args.get_annotations:
+        annotations = get_annotations(args.url)
+        print(json.dumps(annotations, indent=2))
+
+
+if __name__ == "__main__":
+    main()

tonik-0.1.17/src/tonik/package_data/whakaari_labels.json

@@ -0,0 +1,162 @@
+[
+    {
+        "time": "2012-08-04T00:00:00",
+        "title": "Eruption",
+        "description": "Explosive eruption",
+        "tags": ["volcano", "eruption"],
+        "id": 1
+
+    },
+    {
+        "time": "2012-11-22T00:00:00",
+        "timeEnd": "2012-12-10T00:00:00",
+        "title": "Dome extrusion",
+        "description": "Dome extrusion",
+        "tags": ["volcano", "eruption"],
+        "id": 2
+
+    },
+    {
+        "time": "2013-01-15T00:00:00",
+        "timeEnd": "2013-04-10T00:00:00",
+        "title": "Geysering",
+        "description": "Geysering",
+        "tags": ["volcano", "eruption"],
+        "id": 3
+
+    },
+    {
+        "time": "2013-08-19T00:00:00",
+        "title": "Steam and mud eruption",
+        "description": "Minor steam and mud eruption",
+        "tags": ["volcano", "eruption"],
+        "id": 4
+    },
+    {
+        "time": "2013-10-04T00:00:00",
+        "title": "Steam and mud eruption",
+        "description": "Minor steam and mud eruption",
+        "tags": ["volcano", "eruption"],
+        "id": 5
+    },
+    
+    {
+        "time": "2013-10-08T00:00:00",
+        "title": "Steam and mud eruption",
+        "description": "Minor steam and mud eruption",
+        "tags": ["volcano", "eruption"],
+        "id": 6
+    },
+    {
+        "time": "2013-10-11T00:00:00",
+        "title": "Eruption",
+        "description": "Explosive eruption",
+        "tags": ["volcano", "eruption"],
+        "id": 7
+
+    },
+    {
+        "time": "2015-10-13T00:00:00",
+        "timeEnd": "2015-10-20T00:00:00",
+        "title": "Banded tremor",
+        "description": "Banded tremor",
+        "tags": ["volcano", "tremor"],
+        "id": 8
+
+    },
+    {
+        "time": "2016-04-27T00:00:00",
+        "title": "Eruption",
+        "description": "Explosive eruption",
+        "tags": ["volcano", "eruption"],
+        "id": 9
+
+    },
+    {
+        "time": "2016-09-13T00:00:00",
+        "timeEnd": "2016-09-18T00:00:00",
+        "title": "Ashing",
+        "description": "Non-explosive ash venting",
+        "tags": ["volcano", "ash"],
+        "id": 10 
+
+    },
+    {
+        "time": "2019-04-23T00:00:00",
+        "timeEnd": "2019-07-01T00:00:00",
+        "title": "Earthquake swarm",
+        "description": "Earthquake swarm",
+        "tags": ["volcano", "swarm"],
+        "id": 11 
+
+    },
+    {
+        "time": "2019-12-09T00:00:00",
+        "title": "Eruption",
+        "description": "Explosive eruption",
+        "tags": ["volcano", "eruption"],
+        "id": 12 
+
+    },
+    {
+        "time": "2019-12-23T00:00:00",
+        "timeEnd": "2019-12-29T00:00:00",
+        "title": "Ashing",
+        "description": "Minor ash emissions",
+        "tags": ["volcano", "ash"],
+        "id": 13 
+
+    },
+    {
+        "time": "2020-01-10T00:00:00",
+        "timeEnd": "2020-01-20T00:00:00",
+        "title": "Dome extrusion",
+        "description": "Dome extrusion",
+        "tags": ["volcano", "eruption"],
+        "id": 14 
+
+    },
+    {
+        "time": "2020-11-13T00:00:00",
+        "timeEnd": "2020-12-01T00:00:00",
+        "title": "Ashing",
+        "description": "Minor ash emissions",
+        "tags": ["volcano", "ash"],
+        "id": 15 
+
+    },
+    {
+        "time": "2020-12-29T00:00:00",
+        "title": "Eruption",
+        "description": "Small steam explosions",
+        "tags": ["volcano", "eruption"],
+        "id": 16 
+
+    },
+    {
+        "time": "2022-09-18T00:00:00",
+        "timeEnd": "2022-09-24T00:00:00",
+        "title": "Ashing",
+        "description": "Minor ash emissions",
+        "tags": ["volcano", "ash"],
+        "id": 17 
+
+    },
+    {
+        "time": "2024-05-24T00:00:00",
+        "title": "Eruption",
+        "description": "Small steam explosions",
+        "tags": ["volcano", "eruption"],
+        "id": 18 
+
+    },
+    {
+        "time": "2024-07-24T00:00:00",
+        "timeEnd": "2024-09-10T00:00:00",
+        "title": "Ashing",
+        "description": "Minor ash emissions",
+        "tags": ["volcano", "ash"],
+        "id": 19 
+
+    }
+]

{tonik-0.1.15 → tonik-0.1.17}/src/tonik/utils.py

@@ -59,8 +59,55 @@ def generate_test_data(dim=1, ndays=30, nfreqs=10,
     return xds
 
 
+def get_dt(times):
+    """
+    Infer the sampling of the time dimension.
+    """
+    pd_times = pd.to_datetime(times)
+    dt = pd.infer_freq(pd_times)
+    if dt is None:
+        dt = pd_times.diff().median()
+    try:
+        dt = pd.Timedelta(dt)
+    except ValueError:
+        dt = pd.Timedelta(f"1{dt}")
+    return dt
+
+
+def fill_time_gaps(xds: xr.Dataset, timedim: str = 'datetime') -> xr.Dataset:
+    """
+    Fill gaps in time series with NaN values by reindexing to a complete datetime range.
+
+    Parameters
+    ----------
+    xds : xr.Dataset
+        Input dataset with potential time gaps
+    freq : str, optional
+        Frequency string (e.g., 'H', 'D', '15min'). If None, will try to infer.
+    timedim : str
+        Name of the time dimension, by default 'datetime'
+
+    Returns
+    -------
+    xr.Dataset
+        Dataset with gaps filled with NaN
+    """
+    if timedim not in xds.coords:
+        raise ValueError(
+            f"{timedim} coordinate not found in dataset coordinates.")
+
+    # Infer sample interval
+    dt = get_dt(xds.coords[timedim])
+    start_time = xds[timedim].values[0]
+    end_time = xds[timedim].values[-1]
+    complete_time = pd.date_range(start=start_time, end=end_time, freq=dt)
+
+    # Reindex to fill gaps with NaN
+    return xds.reindex({timedim: complete_time})
+
+
 def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray,
-                 resolution: float = None) -> xr.DataArray:
+                 timedim: str = 'datetime', resolution: float = None) -> xr.DataArray:
     """
     Merge two xarray datasets with the same datetime index.
 
@@ -79,16 +126,17 @@ def merge_arrays(xds_old: xr.DataArray, xds_new: xr.DataArray,
         Merged array.
     """
     xda_old = xds_old.drop_duplicates(
-        'datetime', keep='last')
+        timedim, keep='last')
     xda_new = xds_new.drop_duplicates(
-        'datetime', keep='last')
+        timedim, keep='last')
     xda_new = xda_new.combine_first(xda_old)
     if resolution is not None:
         new_dates = pd.date_range(
-            xda_new.datetime.values[0],
-            xda_new.datetime.values[-1],
+            xda_new[timedim].values[0],
+            xda_new[timedim].values[-1],
             freq=f'{resolution}h')
-        xda_new = xda_new.reindex(datetime=new_dates)
+        xda_new = xda_new.reindex(dict(timedim=new_dates))
+    xda_new = fill_time_gaps(xda_new, timedim=timedim)
     return xda_new