tonik 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

tonik/api.py

@@ -1,22 +1,21 @@
-from argparse import ArgumentParser
-from datetime import timedelta, datetime, timezone
 import logging
 import os
+from argparse import ArgumentParser
+from datetime import datetime
+from typing import Annotated
 from urllib.parse import unquote
 
-from cftime import num2date, date2num
 import datashader as dsh
 import numpy as np
 import pandas as pd
 import uvicorn
+from cftime import date2num, num2date
 from fastapi import FastAPI, HTTPException, Query
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import HTMLResponse, StreamingResponse
-from pydantic import BaseModel
-from typing import Annotated
 
-from .storage import Storage
 from . import get_data
+from .storage import Storage
 
 logger = logging.getLogger(__name__)
 
@@ -67,7 +66,8 @@ class TonikAPI:
         _st = self.preprocess_datetime(starttime)
         _et = self.preprocess_datetime(endtime)
         g = Storage(group, rootdir=self.rootdir,
-                    starttime=_st, endtime=_et)
+                    starttime=_st, endtime=_et,
+                    create=False)
         if subdir is None:
             c = g
         else:
@@ -103,13 +103,15 @@ class TonikAPI:
         else:
             df = pd.DataFrame(data=feat.to_pandas(), columns=[feat.name])
             df['dates'] = df.index
-            try:
-                current_resolution = pd.Timedelta(df['dates'].diff().mean())
-                if current_resolution < pd.Timedelta(resolution):
-                    df = df.resample(pd.Timedelta(resolution)).mean()
-            except ValueError as e:
-                logger.warning(
-                    f"Cannot resample {feat.name} to {resolution}: e")
+            if resolution != 'full':
+                try:
+                    current_resolution = pd.Timedelta(
+                        df['dates'].diff().mean())
+                    if current_resolution < pd.Timedelta(resolution):
+                        df = df.resample(pd.Timedelta(resolution)).mean()
+                except ValueError:
+                    logger.warning(
+                        f"Cannot resample {feat.name} to {resolution}: e")
             df.rename(columns={feat.name: 'feature'}, inplace=True)
             output = df.to_csv(index=False, columns=['dates', 'feature'])
         return StreamingResponse(iter([output]),
@@ -143,7 +145,7 @@ class TonikAPI:
             c = sg.get_substore(*subdir)
         except TypeError:
             c = sg
-        except FileNotFoundError as e:
+        except FileNotFoundError:
             msg = "Directory {} not found.".format(
                 '/'.join([sg.path] + subdir))
             raise HTTPException(status_code=404, detail=msg)

tonik/utils.py

@@ -8,8 +8,8 @@ import xarray as xr
 def generate_test_data(dim=1, ndays=30, nfreqs=10,
                        tstart=datetime.now(),
                        freq='10min', intervals=None,
-                       feature_name=None, seed=42,
-                       freq_name=None, add_nans=True):
+                       feature_names=None, seed=42,
+                       freq_names=None, add_nans=True):
     """
     Generate a 1D or 2D feature for testing.
     """
@@ -28,25 +28,31 @@ def generate_test_data(dim=1, ndays=30, nfreqs=10,
         data = np.tile(data, (nfreqs, 1))
     # Add 10% NaNs
     idx_nan = rs.integers(0, nints-1, int(0.1*nints))
+
+    xds_dict = {}
     if dim == 1:
         if add_nans:
             data[idx_nan] = np.nan
-        if feature_name is None:
-            feature_name = 'rsam'
-        xrd = xr.Dataset({feature_name: xr.DataArray(
-            data, coords=[dates], dims=['datetime'])})
+        if feature_names is None:
+            feature_names = ['rsam', 'dsar']
+        for feature in feature_names:
+            xds_dict[feature] = xr.DataArray(
+                data, coords=[dates], dims=['datetime'])
     if dim == 2:
         if add_nans:
             data[:, idx_nan] = np.nan
         freqs = np.arange(nfreqs)
-        if feature_name is None:
-            feature_name = 'ssam'
-        if freq_name is None:
-            freq_name = 'frequency'
-        xrd = xr.Dataset({feature_name: xr.DataArray(
-            data, coords=[freqs, dates], dims=[freq_name, 'datetime'])})
-    xrd.attrs['starttime'] = dates[0].isoformat()
-    xrd.attrs['endtime'] = dates[-1].isoformat()
-    xrd.attrs['station'] = 'MDR'
-    xrd.attrs['interval'] = '10min'
-    return xrd
+        if feature_names is None:
+            feature_names = ['ssam', 'filterbank']
+        if freq_names is None:
+            freq_names = ['frequency', 'fbfrequency']
+
+        for feature_name, freq_name in zip(feature_names, freq_names):
+            xds_dict[feature_name] = xr.DataArray(
+                data, coords=[freqs, dates], dims=[freq_name, 'datetime'])
+    xds = xr.Dataset(xds_dict)
+    xds.attrs['starttime'] = dates[0].isoformat()
+    xds.attrs['endtime'] = dates[-1].isoformat()
+    xds.attrs['station'] = 'MDR'
+    xds.attrs['interval'] = '10min'
+    return xds

tonik/xarray2zarr.py

@@ -1,7 +1,10 @@
+import logging
 import os
 
 import xarray as xr
 
+logger = logging.getLogger(__name__)
+
 
 def xarray2zarr(xds, path, mode='a'):
     for feature in xds.data_vars.keys():
@@ -11,13 +14,24 @@ def xarray2zarr(xds, path, mode='a'):
                 fout, group='original', mode='w')
         else:
             xds_existing = xr.open_zarr(fout, group='original')
-            overlap = xds_existing.datetime.where(
-                xds_existing.datetime == xds.datetime)
-            if overlap.size > 0:
-                xds.loc[dict(datetime=overlap)].to_zarr(
-                    fout, group='original', mode='r+', region='auto')
-                xds.drop_sel(datetime=overlap).to_zarr(
-                    fout, group='original', mode='a', append_dim="datetime")
-            else:
-                xds[feature].to_zarr(
-                    fout, group='original', append_dim='datetime')
+            try:
+                overlap = xds_existing.datetime.where(
+                    xds_existing.datetime == xds.datetime)
+                if overlap.size > 0:
+                    xds[feature].loc[dict(datetime=overlap)].to_zarr(
+                        fout, group='original', mode='r+', region='auto')
+                    xds[feature].drop_sel(datetime=overlap).to_zarr(
+                        fout, group='original', mode='a', append_dim="datetime")
+                else:
+                    xds[feature].to_zarr(
+                        fout, group='original', append_dim='datetime')
+            except Exception as e:
+                msg = f"Appending {feature} to {fout} failed: {e}\n"
+                msg += "Attempting to merge the two datasets."
+                logger.error(msg)
+                # remove duplicate datetime entries
+                xda_existing = xds_existing[feature].drop_duplicates(
+                    'datetime', keep='last')
+                xda_new = xds[feature].drop_duplicates('datetime', keep='last')
+                xda_new = xda_new.combine_first(xda_existing)
+                xda_new.to_zarr(fout, group='original', mode='w')

{tonik-0.0.8.dist-info → tonik-0.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: tonik
-Version: 0.0.8
+Version: 0.0.10
 Summary: Store time series data as HDF5 files and access them through an API.
 Project-URL: Homepage, https://tsc-tools.github.io/tonik
 Project-URL: Issues, https://github.com/tsc-tools/tonik/issues

tonik-0.0.10.dist-info/RECORD

@@ -0,0 +1,12 @@
+tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
+tonik/api.py,sha256=8YS0WCMlm5xs2N6V8n9BiP87ywJC_BUzYsPvWR9KWJk,6499
+tonik/storage.py,sha256=sScIFA4KXURNPwTnV-rvDh6cWCy9sRrErr9BshZpw2I,11303
+tonik/utils.py,sha256=_TxXf9o9fOvtuOvGO6-ww9F5m0QelHyfQzQw8RGjTV4,1868
+tonik/xarray2hdf5.py,sha256=cekO9vo9ZRlr0VndswJjPC27CEVD3TpRVKLAJ-aAO0g,4465
+tonik/xarray2zarr.py,sha256=9gvm7vFnoe-O39-ow2pqK4xoFSg9Tg865qdqKYI-Zy8,1605
+tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
+tonik-0.0.10.dist-info/METADATA,sha256=IN-k1gZOtxQrsVlYDtpiJuBW4gIJCXRsqFO4pNCT_uo,1939
+tonik-0.0.10.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+tonik-0.0.10.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
+tonik-0.0.10.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+tonik-0.0.10.dist-info/RECORD,,

{tonik-0.0.8.dist-info → tonik-0.0.10.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.17.1
+Generator: hatchling 1.25.0
 Root-Is-Purelib: true
 Tag: py3-none-any

tonik-0.0.8.dist-info/RECORD

@@ -1,12 +0,0 @@
-tonik/__init__.py,sha256=ZBVGh4dm_l9xwiBGb33O5QV9MfZeNiEd3DBDAm6DiHk,511
-tonik/api.py,sha256=vdsWHNGGWo4sbqlDyZQj2tX5oe6hAWCzyL6ffsfpCB4,6437
-tonik/storage.py,sha256=sScIFA4KXURNPwTnV-rvDh6cWCy9sRrErr9BshZpw2I,11303
-tonik/utils.py,sha256=YD2zZx5nKGfTJKTYTsEZVV78uNRCSakvU_6X6Mgwx-s,1664
-tonik/xarray2hdf5.py,sha256=cekO9vo9ZRlr0VndswJjPC27CEVD3TpRVKLAJ-aAO0g,4465
-tonik/xarray2zarr.py,sha256=d7FAOe7DESbKC9CZS41r62DjlNy0S8ik01lMGXBvJKw,901
-tonik/package_data/index.html,sha256=GKDClUhIam_fAYbNfzAolORhSCG3ae1wW3VjWCg4PMk,2732
-tonik-0.0.8.dist-info/METADATA,sha256=uqrHvBl01n05hqWN8lnNETuoYIGZ8vNq9QWnRmajcEY,1938
-tonik-0.0.8.dist-info/WHEEL,sha256=KGYbc1zXlYddvwxnNty23BeaKzh7YuoSIvIMO4jEhvw,87
-tonik-0.0.8.dist-info/entry_points.txt,sha256=VnGfC5qAzpntEHAb5pooUEpYABSgOfQoNhCEtLDJyf8,45
-tonik-0.0.8.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-tonik-0.0.8.dist-info/RECORD,,