ocf-data-sampler 0.2.1__tar.gz → 0.2.3__tar.gz

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf-data-sampler
-Version: 0.2.1
+Version: 0.2.3
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/numpy_sample/collate.py

@@ -2,8 +2,10 @@
 
 import numpy as np
 
+from ocf_data_sampler.numpy_sample.common_types import NumpyBatch
 
-def stack_np_samples_into_batch(dict_list: list[dict]) -> dict:
+
+def stack_np_samples_into_batch(dict_list: list[dict]) -> NumpyBatch:
     """Stacks list of dict samples into a dict where all samples are joined along a new axis.
 
     Args:

ocf_data_sampler-0.2.3/ocf_data_sampler/numpy_sample/common_types.py

@@ -0,0 +1,11 @@
+"""This module defines type aliases for numpy and torch data structures used in the project."""
+
+
+from typing import TypeAlias
+
+import numpy as np
+import torch
+
+NumpySample: TypeAlias = dict[str, np.ndarray | dict[str, np.ndarray]]
+NumpyBatch: TypeAlias = dict[str, np.ndarray | dict[str, np.ndarray]]
+TensorBatch: TypeAlias = dict[str, torch.Tensor | dict[str, torch.Tensor]]

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/numpy_sample/datetime_features.py

@@ -3,6 +3,8 @@
 import numpy as np
 import pandas as pd
 
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+
 
 def _get_date_time_in_pi(dt: pd.DatetimeIndex) -> tuple[np.ndarray, np.ndarray]:
     """Create positional embeddings for the datetimes in radians.
@@ -22,7 +24,7 @@ def _get_date_time_in_pi(dt: pd.DatetimeIndex) -> tuple[np.ndarray, np.ndarray]:
     return date_in_pi, time_in_pi
 
 
-def make_datetime_numpy_dict(datetimes: pd.DatetimeIndex, key_prefix: str = "wind") -> dict:
+def make_datetime_numpy_dict(datetimes: pd.DatetimeIndex, key_prefix: str = "wind") -> NumpySample:
     """Creates dictionary of cyclical datetime features - encoded."""
     date_in_pi, time_in_pi = _get_date_time_in_pi(datetimes)
 

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/numpy_sample/gsp.py

@@ -2,6 +2,8 @@
 
 import xarray as xr
 
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+
 
 class GSPSampleKey:
     """Keys for the GSP sample dictionary."""
@@ -16,7 +18,7 @@ class GSPSampleKey:
     y_osgb = "gsp_y_osgb"
 
 
-def convert_gsp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
+def convert_gsp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> NumpySample:
     """Convert from Xarray to NumpySample.
 
     Args:

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/numpy_sample/nwp.py

@@ -3,6 +3,8 @@
 import pandas as pd
 import xarray as xr
 
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+
 
 class NWPSampleKey:
     """Keys for NWP NumpySample."""
@@ -15,7 +17,7 @@ class NWPSampleKey:
     t0_idx = "nwp_t0_idx"
 
 
-def convert_nwp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
+def convert_nwp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> NumpySample:
     """Convert from Xarray to NWP NumpySample.
 
     Args:

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/numpy_sample/satellite.py

@@ -2,6 +2,8 @@
 
 import xarray as xr
 
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+
 
 class SatelliteSampleKey:
     """Keys for the SatelliteSample dictionary."""
@@ -13,7 +15,7 @@ class SatelliteSampleKey:
     t0_idx = "satellite_t0_idx"
 
 
-def convert_satellite_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
+def convert_satellite_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> NumpySample:
     """Convert from Xarray to NumpySample.
 
     Args:

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/numpy_sample/site.py

@@ -2,6 +2,8 @@
 
 import xarray as xr
 
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+
 
 class SiteSampleKey:
     """Keys for the site sample dictionary."""
@@ -17,7 +19,7 @@ class SiteSampleKey:
     time_cos = "site_time_cos"
 
 
-def convert_site_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
+def convert_site_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> NumpySample:
     """Convert from Xarray to NumpySample.
 
     Args:

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/numpy_sample/sun_position.py

@@ -4,6 +4,8 @@ import numpy as np
 import pandas as pd
 import pvlib
 
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+
 
 def calculate_azimuth_and_elevation(
     datetimes: pd.DatetimeIndex,
@@ -35,7 +37,7 @@ def make_sun_position_numpy_sample(
     datetimes: pd.DatetimeIndex,
     lon: float,
     lat: float,
-) -> dict:
+) -> NumpySample:
     """Creates NumpySample with standardized solar coordinates.
 
     Args:

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/sample/base.py

@@ -1,14 +1,11 @@
 """Base class for handling flat/nested data structures with NWP consideration."""
 
 from abc import ABC, abstractmethod
-from typing import TypeAlias
 
 import numpy as np
 import torch
 
-NumpySample: TypeAlias = dict[str, np.ndarray | dict[str, np.ndarray]]
-NumpyBatch: TypeAlias = dict[str, np.ndarray | dict[str, np.ndarray]]
-TensorBatch: TypeAlias = dict[str, torch.Tensor | dict[str, torch.Tensor]]
+from ocf_data_sampler.numpy_sample.common_types import NumpyBatch, NumpySample, TensorBatch
 
 
 class SampleBase(ABC):

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/sample/site.py

@@ -3,7 +3,8 @@
 import xarray as xr
 from typing_extensions import override
 
-from ocf_data_sampler.sample.base import NumpySample, SampleBase
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+from ocf_data_sampler.sample.base import SampleBase
 from ocf_data_sampler.torch_datasets.datasets.site import convert_netcdf_to_numpy_sample
 
 

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/sample/uk_regional.py

@@ -8,7 +8,8 @@ from ocf_data_sampler.numpy_sample import (
     NWPSampleKey,
     SatelliteSampleKey,
 )
-from ocf_data_sampler.sample.base import NumpySample, SampleBase
+from ocf_data_sampler.numpy_sample.common_types import NumpySample
+from ocf_data_sampler.sample.base import SampleBase
 
 
 class UKRegionalSample(SampleBase):

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/select/select_time_slice.py

@@ -55,7 +55,10 @@ def select_time_slice_nwp(
     if accum_channels is None:
         accum_channels = []
 
-    if dropout_timedeltas is not None:
+    if dropout_timedeltas is None:
+        dropout_timedeltas = []
+
+    if len(dropout_timedeltas)>0:
         if not all(t < pd.Timedelta(0) for t in dropout_timedeltas):
             raise ValueError("dropout timedeltas must be negative")
         if len(dropout_timedeltas) < 1:
@@ -64,7 +67,7 @@ def select_time_slice_nwp(
     if not (0 <= dropout_frac <= 1):
         raise ValueError("dropout_frac must be between 0 and 1")
 
-    consider_dropout = (dropout_timedeltas is not None) and dropout_frac > 0
+    consider_dropout = len(dropout_timedeltas) > 0 and dropout_frac > 0
 
     # The accumatated and non-accumulated channels
     accum_channels = np.intersect1d(da.channel.values, accum_channels)

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler/torch_datasets/utils/valid_time_periods.py

@@ -28,7 +28,7 @@ def find_valid_time_periods(datasets_dict: dict, config: Configuration) -> pd.Da
         for nwp_key, nwp_config in config.input_data.nwp.items():
             da = datasets_dict["nwp"][nwp_key]
 
-            if nwp_config.dropout_timedeltas_minutes is None:
+            if nwp_config.dropout_timedeltas_minutes==[]:
                 max_dropout = minutes(0)
             else:
                 max_dropout = minutes(np.max(np.abs(nwp_config.dropout_timedeltas_minutes)))

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: ocf-data-sampler
-Version: 0.2.1
+Version: 0.2.3
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.2.1 → ocf_data_sampler-0.2.3}/ocf_data_sampler.egg-info/SOURCES.txt

@@ -29,6 +29,7 @@ ocf_data_sampler/load/nwp/providers/ukv.py
 ocf_data_sampler/load/nwp/providers/utils.py
 ocf_data_sampler/numpy_sample/__init__.py
 ocf_data_sampler/numpy_sample/collate.py
+ocf_data_sampler/numpy_sample/common_types.py
 ocf_data_sampler/numpy_sample/datetime_features.py
 ocf_data_sampler/numpy_sample/gsp.py
 ocf_data_sampler/numpy_sample/nwp.py