nuthatch 0.1.0__py3-none-any.whl

nuthatch/processors/timeseries.py

@@ -0,0 +1,157 @@
+from nuthatch.processor import NuthatchProcessor
+import dateparser
+import datetime
+import pandas as pd
+import dask.dataframe as dd
+import xarray as xr
+
+class TimeseriesProcessor(NuthatchProcessor):
+    """
+    Processor for timeseries data.
+
+    This processor is used to slice a timeseries dataset based on the start and end times.
+
+    It also validates the timeseries data to ensure it has data within the start and end times.
+
+    It supports xarray datasets and pandas/dask dataframes.
+    """
+    def __init__(self, func, timeseries, validate_timeseries):
+        """
+        Initialize the timeseries processor.
+
+        Args:
+            func: The function to wrap.
+            timeseries: The name of the timeseries dimension.
+            validate_timeseries: Whether to validate the timeseries data.
+        """
+        self.func = func
+        self.validate_timeseries = validate_timeseries
+        self.timeseries = timeseries
+
+    def post_process(self, ds):
+        start_time = self.start_time
+        end_time = self.end_time
+
+        if isinstance(ds, xr.Dataset):
+            match_time = [t for t in self.timeseries if t in ds.dims]
+            if len(match_time) == 0:
+                raise RuntimeError(
+                    f"Timeseries must have a dimension named {self.timeseries} for slicing."
+                )
+
+            time_col = match_time[0]
+            ds = ds.sel({time_col: slice(start_time, end_time)})
+        elif isinstance(ds, pd.DataFrame) or isinstance(ds, dd.DataFrame):
+            match_time = [t for t in self.timeseries if t in ds.columns]
+
+            if len(match_time) == 0:
+                raise RuntimeError(
+                    f"Timeseries must have a dimension named {self.timeseries} for slicing."
+                )
+
+            time_col = match_time[0]
+
+            try:
+                if start_time is not None:
+                    ds = ds[ds[time_col] >= start_time]
+                if end_time is not None:
+                    ds = ds[ds[time_col] <= end_time]
+
+            except TypeError as e:
+                if "Invalid comparison" not in str(e):
+                    raise e
+
+                time_col_tz = ds[time_col].dt.tz.compute()
+
+                if start_time is not None:
+                    start_time = pd.Timestamp(start_time)
+                    if start_time.tz is None:
+                        start_time = start_time.tz_localize(time_col_tz)
+                    else:
+                        start_time = start_time.tz_convert(time_col_tz)
+                    ds = ds[ds[time_col] >= start_time]
+
+                if end_time is not None:
+                    end_time = pd.Timestamp(end_time)
+                    if end_time.tz is None:
+                        end_time = end_time.tz_localize(time_col_tz)
+                    else:
+                        end_time = end_time.tz_convert(time_col_tz)
+                    ds = ds[ds[time_col] <= end_time]
+        else:
+            raise RuntimeError(f"Cannot filter timeseries for data type {type(ds)}")
+
+        return ds
+
+    def process_arguments(self, params, args, kwargs):
+        if 'validate_timeseries' in kwargs:
+            passed_validate_timeseries = kwargs['validate_timeseries']
+            if passed_validate_timeseries:
+                self.validate_timeseries = passed_validate_timeseries
+            del kwargs['validate_timeseries']
+
+        # Validate time series params
+        self.start_time = None
+        self.end_time = None
+
+        # Convert to a list if not
+        self.timeseries = self.timeseries if isinstance(self.timeseries, list) else [self.timeseries]
+
+        if 'start_time' not in params or 'end_time' not in params:
+            raise ValueError(
+                "Time series functions must have the parameters 'start_time' and 'end_time'")
+        else:
+            keys = [item for item in params]
+            try:
+                self.start_time = args[keys.index('start_time')]
+                self.end_time = args[keys.index('end_time')]
+            except IndexError:
+                raise ValueError("'start_time' and 'end_time' must be passed as positional arguments, not "
+                                     "keyword arguments")
+
+        return args, kwargs
+
+
+    def validate_data(self, ds):
+        start_time = self.start_time
+        end_time = self.end_time
+        if self.validate_timeseries:
+            if isinstance(ds, xr.Dataset):
+                # Check to see if the dataset extends roughly the full time series set
+                match_time = [t for t in self.timeseries if t in ds.dims]
+                if len(match_time) == 0:
+                    raise RuntimeError("Timeseries array functions must return "
+                                       "a time dimension for slicing. "
+                                       "This could be an invalid cache. "
+                                       "Try running with recompute=True to reset the cache.")
+                else:
+                    time_col = match_time[0]
+
+                    # Assign start and end times if None are passed
+                    st = dateparser.parse(start_time) if start_time is not None \
+                        else pd.Timestamp(ds[time_col].min().values)
+                    et = dateparser.parse(end_time) if end_time is not None \
+                        else pd.Timestamp(ds[time_col].max().values)
+
+                    # Check if within 1 year at least
+                    if (pd.Timestamp(ds[time_col].min().values) <
+                        st + datetime.timedelta(days=365) and
+                            pd.Timestamp(ds[time_col].max().values) >
+                            et - datetime.timedelta(days=365)):
+                        return True
+                    else:
+                        print("""WARNING: The cached array does not have data within
+                              1 year of your start or end time. Triggering recompute.
+                              If you do not want to recompute the result set
+                              `validate_cache_timeseries=False`""")
+                        return False
+            else:
+                raise RuntimeError(f"Cannot validate timeseries for data type {type(ds)}")
+
+        return True
+
+def timeseries(timeseries='time', validate_timeseries=False):
+    def decorator(func):
+         return TimeseriesProcessor(func, timeseries, validate_timeseries)
+    return decorator
+

nuthatch-0.1.0.dist-info/METADATA

@@ -0,0 +1,38 @@
+Metadata-Version: 2.3
+Name: nuthatch
+Version: 0.1.0
+Summary: Cacheable big data pipelines
+Author-email: Joshua Adkins <josh@rhizaresearch.org>, Genevieve Flaspohler <geneviee@rhizaresearch.org>
+Requires-Python: >=3.12
+Requires-Dist: click
+Requires-Dist: dask-deltatable
+Requires-Dist: dask[dataframe]
+Requires-Dist: deltalake==1.1.2
+Requires-Dist: fsspec
+Requires-Dist: gitpython
+Requires-Dist: pandas
+Requires-Dist: psycopg2
+Requires-Dist: pyarrow
+Requires-Dist: sqlalchemy
+Requires-Dist: terracotta
+Requires-Dist: xarray
+Requires-Dist: zarr
+Description-Content-Type: text/markdown
+
+# Nuthatch
+
+Nuthatch is a tool for building pure-python big data pipelines. At its core it
+enables the transparent multi-level caching and recall of results in formats that 
+are efficient for each data type. It supports a variety of 
+common storage backends, data processing frameworks, and their associated
+data types for caching. 
+
+It also provides a framework for re-using and sharing data-type specific
+post-processing, and for these data type
+processors to pass hints to storage backends for more efficient storager and recall.
+
+Nuthatch was created to alleviate the comon pattern of data processing pipelines manually
+specifying their output storage locations, and the requirements of pipeline builders to
+use external data orchestration tools to specify the execution of their pipeliness. With Nuthatch
+simply tag your functions and anyone who has access to your storage backend - you, your
+team, or the public - can acess and build off of your most up-to-date data.

nuthatch-0.1.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+nuthatch/__init__.py,sha256=h4jzi33pztmMY3QPHSp6WcOeO3QMzEWjVUDAWICHgjE,384
+nuthatch/backend.py,sha256=1BijlFcQrkyvr7V8ai4PnKoizDsOl3-u1KpxOyDGARg,11018
+nuthatch/cache.py,sha256=QVhwi2x5nRopPJdJBWuudCwUHs3PcTvTmpmwuLqmPNI,22193
+nuthatch/cli.py,sha256=9AU1N_aMsTC_Nm81XgcH0v4CPaeULo1pW74nEV6CZJk,7016
+nuthatch/config.py,sha256=Rwqkjn70IHM4N_M3cCg5hZEqOroPC95rs_e2--z0Hjc,3756
+nuthatch/memoizer.py,sha256=MkEdzodZjq3gZnTGNWhR0cJRFDsRo2EKRJaDrLpWPsM,1798
+nuthatch/nuthatch.py,sha256=mSi2uAhuY06uOqnnXxsavaeKV5RrLKQcNkUVpSkDHnw,20859
+nuthatch/processor.py,sha256=PTEg6jTURJfDeDCZP0TOQim1Q44Vi3S0Z6uIeKdxkrI,2834
+nuthatch/backends/__init__.py,sha256=7vT9xP1QNTSX2Y3YE3Flt5h8mgQi45ROlcD9Wl9IkUk,312
+nuthatch/backends/basic.py,sha256=KPQUTDbDRkFCvkfOp8zmaRxd9IkrxRQBPZx-VfW0e9Y,856
+nuthatch/backends/delta.py,sha256=1HQILMAtJ4oIrvbLFtwIr4JbhlqBZ_X6qsNDYyIVyQs,1846
+nuthatch/backends/parquet.py,sha256=q1ysnpdfkedT64xNW28StvqiWRWd7AXdMGvMOlCqXe0,5319
+nuthatch/backends/sql.py,sha256=BHdxReQAr5GO3Nu99aSIHUZHFrvk-xqDHogxAR-bSCw,6746
+nuthatch/backends/terracotta.py,sha256=My3nPYEYNuuZ2d93Ogsh5PBmJWm9JBQT9h0HbX0Mcss,7049
+nuthatch/backends/zarr.py,sha256=hJ8Cy28cBwDlBNTgzhWEFi5E747fP2PuwIC-jGgmMs0,8356
+nuthatch/processors/__init__.py,sha256=6b46sDY3IoxnK0V8x9vPomCfGXoHWoserGcUtxWKyQg,111
+nuthatch/processors/timeseries.py,sha256=Jumjn0qqsel2Oh-AJWCTUHW4xodi5_oMoaXbDRGoo9Y,6427
+nuthatch-0.1.0.dist-info/METADATA,sha256=o1BJhQxY0KYRRLfhXUccGj8QsLZePbxftwksRrYRoVA,1558
+nuthatch-0.1.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+nuthatch-0.1.0.dist-info/entry_points.txt,sha256=Sn3SYzKxaDtQyiM1dfAz13g_NiZRzIOagOawnCmfrR4,47
+nuthatch-0.1.0.dist-info/RECORD,,

nuthatch-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.26.3
+Root-Is-Purelib: true
+Tag: py3-none-any

nuthatch-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nuthatch = nuthatch.cli:main