ocf-data-sampler 0.1.9__py3-none-any.whl → 0.1.11__py3-none-any.whl

ocf_data_sampler/config/model.py

@@ -49,31 +49,34 @@ class TimeWindowMixin(Base):
         ...,
         description="Data interval ends at `t0 + interval_end_minutes`",
     )
-    
+
     @model_validator(mode='after')
-    def check_interval_range(cls, values):
-        if values.interval_start_minutes > values.interval_end_minutes:
-            raise ValueError('interval_start_minutes must be <= interval_end_minutes')
+    def validate_intervals(cls, values):
+        start = values.interval_start_minutes
+        end = values.interval_end_minutes
+        resolution = values.time_resolution_minutes
+        if start > end:
+            raise ValueError(
+                f"interval_start_minutes ({start}) must be <= interval_end_minutes ({end})"
+            )
+        if (start % resolution != 0):
+            raise ValueError(
+                f"interval_start_minutes ({start}) must be divisible "
+                f"by time_resolution_minutes ({resolution})"
+            )
+        if (end % resolution != 0):
+            raise ValueError(
+                f"interval_end_minutes ({end}) must be divisible "
+                f"by time_resolution_minutes ({resolution})"
+            )
         return values
 
-    @field_validator("interval_start_minutes")
-    def interval_start_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
-        if v % info.data["time_resolution_minutes"] != 0:
-            raise ValueError("interval_start_minutes must be divisible by time_resolution_minutes")
-        return v
-
-    @field_validator("interval_end_minutes")
-    def interval_end_minutes_divide_by_time_resolution(cls, v: int, info: ValidationInfo) -> int:
-        if v % info.data["time_resolution_minutes"] != 0:
-            raise ValueError("interval_end_minutes must be divisible by time_resolution_minutes")
-        return v
-
 
 class DropoutMixin(Base):
     """Mixin class, to add dropout minutes"""
 
-    dropout_timedeltas_minutes: Optional[List[int]] = Field(
-        default=None,
+    dropout_timedeltas_minutes: List[int] = Field(
+        default=[],
         description="List of possible minutes before t0 where data availability may start. Must be "
         "negative or zero.",
     )
@@ -88,18 +91,17 @@ class DropoutMixin(Base):
     @field_validator("dropout_timedeltas_minutes")
     def dropout_timedeltas_minutes_negative(cls, v: List[int]) -> List[int]:
         """Validate 'dropout_timedeltas_minutes'"""
-        if v is not None:
-            for m in v:
-                assert m <= 0, "Dropout timedeltas must be negative"
+        for m in v:
+            assert m <= 0, "Dropout timedeltas must be negative"
         return v
 
     @model_validator(mode="after")
     def dropout_instructions_consistent(self) -> Self:
         if self.dropout_fraction == 0:
-            if self.dropout_timedeltas_minutes is not None:
+            if self.dropout_timedeltas_minutes != []:
                 raise ValueError("To use dropout timedeltas dropout fraction should be > 0")
         else:
-            if self.dropout_timedeltas_minutes is None:
+            if self.dropout_timedeltas_minutes == []:
                 raise ValueError("To dropout fraction > 0 requires a list of dropout timedeltas")
         return self
 

ocf_data_sampler/load/satellite.py

@@ -1,7 +1,5 @@
 """Satellite loader"""
 
-import subprocess
-
 import xarray as xr
 from ocf_data_sampler.load.utils import (
     check_time_unique_increasing,
@@ -10,63 +8,59 @@ from ocf_data_sampler.load.utils import (
 )
 
 
-def _get_single_sat_data(zarr_path: str) -> xr.Dataset:
-    """Helper function to open a Zarr from either a local or GCP path.
+def get_single_sat_data(zarr_path: str) -> xr.Dataset:
+    """Helper function to open a zarr from either a local or GCP path
 
     Args:
-        zarr_path: Path to a Zarr file. Wildcards (*) are supported **only** for local paths. 
-                   GCS paths (gs://) **do not support** wildcards.
+        zarr_path: path to a zarr file. Wildcards (*) are supported only for local paths
+                   GCS paths (gs://) do not support wildcards
 
     Returns:
-        An xarray Dataset containing satellite data.
+        An xarray Dataset containing satellite data
 
     Raises:
-        ValueError: If a wildcard (*) is used in a GCS (gs://) path.
+        ValueError: If a wildcard (*) is used in a GCS (gs://) path
     """
 
-    # These kwargs are used if the path contains "*"
-    openmf_kwargs = dict(
-        engine="zarr",
-        concat_dim="time",
-        combine="nested",
-        chunks="auto",
-        join="override",
-    )
-
     # Raise an error if a wildcard is used in a GCP path
     if "gs://" in str(zarr_path) and "*" in str(zarr_path):
-        raise ValueError("Wildcard (*) paths are not supported for GCP (gs://) URLs.")
+        raise ValueError("Wildcard (*) paths are not supported for GCP (gs://) URLs")
 
     # Handle multi-file dataset for local paths
     if "*" in str(zarr_path):
-        ds = xr.open_mfdataset(zarr_path, **openmf_kwargs)
+        ds = xr.open_mfdataset(
+            zarr_path,
+            engine="zarr",
+            concat_dim="time",
+            combine="nested",
+            chunks="auto",
+            join="override",
+        )
+        check_time_unique_increasing(ds.time)
     else:
         ds = xr.open_dataset(zarr_path, engine="zarr", chunks="auto")
 
-    # Ensure time is unique and sorted
-    ds = ds.drop_duplicates("time").sortby("time")
-
     return ds
 
 
 def open_sat_data(zarr_path: str | list[str]) -> xr.DataArray:
-    """Lazily opens the Zarr store.
+    """Lazily opens the zarr store
 
     Args:
-      zarr_path: Cloud URL or local path pattern, or list of these. If GCS URL, it must start with
-          'gs://'.
+      zarr_path: Cloud URL or local path pattern, or list of these. If GCS URL, 
+                 it must start with 'gs://'
     """
 
     # Open the data
     if isinstance(zarr_path, (list, tuple)):
         ds = xr.combine_nested(
-            [_get_single_sat_data(path) for path in zarr_path],
+            [get_single_sat_data(path) for path in zarr_path],
             concat_dim="time",
             combine_attrs="override",
             join="override",
         )
     else:
-        ds = _get_single_sat_data(zarr_path)
+        ds = get_single_sat_data(zarr_path)
 
     ds = ds.rename(
         {
@@ -76,9 +70,7 @@ def open_sat_data(zarr_path: str | list[str]) -> xr.DataArray:
     )
 
     check_time_unique_increasing(ds.time_utc)
-
     ds = make_spatial_coords_increasing(ds, x_coord="x_geostationary", y_coord="y_geostationary")
-    
     ds = ds.transpose("time_utc", "channel", "x_geostationary", "y_geostationary")
     
     # TODO: should we control the dtype of the DataArray?

ocf_data_sampler/load/site.py

@@ -20,7 +20,7 @@ def open_site(generation_file_path: str, metadata_file_path: str) -> xr.DataArra
 
     assert metadata_df.index.is_unique
 
-    # Ensure metadata aligns with the site_id dimension in data_ds
+    # Ensure metadata aligns with the site_id dimension in generation_ds
     metadata_df = metadata_df.reindex(generation_ds.site_id.values)
 
     # Assign coordinates to the Dataset using the aligned metadata

ocf_data_sampler/numpy_sample/gsp.py

@@ -18,9 +18,13 @@ class GSPSampleKey:
 
 
 def convert_gsp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
-    """Convert from Xarray to NumpySample"""
+    """Convert from Xarray to NumpySample
+    
+    Args:
+        da: Xarray DataArray containing GSP data
+        t0_idx: Index of the t0 timestamp in the time dimension of the GSP data
+    """
 
-   # Extract values from the DataArray
     sample = {
         GSPSampleKey.gsp: da.values,
         GSPSampleKey.nominal_capacity_mwp: da.isel(time_utc=0)["nominal_capacity_mwp"].values,

ocf_data_sampler/numpy_sample/nwp.py

@@ -12,30 +12,24 @@ class NWPSampleKey:
     step = 'nwp_step'
     target_time_utc = 'nwp_target_time_utc'
     t0_idx = 'nwp_t0_idx'
-    y_osgb = 'nwp_y_osgb'
-    x_osgb = 'nwp_x_osgb'
-
 
 
 def convert_nwp_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
-    """Convert from Xarray to NWP NumpySample"""
+    """Convert from Xarray to NWP NumpySample
+    
+    Args:
+        da: Xarray DataArray containing NWP data
+        t0_idx: Index of the t0 timestamp in the time dimension of the NWP
+    """
     
-    # Create example and add t if available
     sample = {
         NWPSampleKey.nwp: da.values,
         NWPSampleKey.channel_names: da.channel.values,
         NWPSampleKey.init_time_utc: da.init_time_utc.values.astype(float),
         NWPSampleKey.step: (da.step.values / pd.Timedelta("1h")).astype(int),
+        NWPSampleKey.target_time_utc: da.target_time_utc.values.astype(float),
     }
 
-    if "target_time_utc" in da.coords:
-        sample[NWPSampleKey.target_time_utc] = da.target_time_utc.values.astype(float)
-
-    # TODO: Do we need this at all? Especially since it is only present in UKV data
-    for sample_key, dataset_key in ((NWPSampleKey.y_osgb, "y_osgb"),(NWPSampleKey.x_osgb, "x_osgb"),):
-        if dataset_key in da.coords:
-            sample[sample_key] = da[dataset_key].values
-
     if t0_idx is not None:
         sample[NWPSampleKey.t0_idx] = t0_idx
         

ocf_data_sampler/numpy_sample/satellite.py

@@ -1,4 +1,5 @@
 """Convert Satellite to NumpySample"""
+
 import xarray as xr
 
 
@@ -12,19 +13,21 @@ class SatelliteSampleKey:
 
 
 def convert_satellite_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
-    """Convert from Xarray to NumpySample"""
+    """Convert from Xarray to NumpySample
+    
+    Args:
+        da: xarray DataArray containing satellite data
+        t0_idx: Index of the t0 timestamp in the time dimension of the satellite data
+    """
     sample = {
         SatelliteSampleKey.satellite_actual: da.values,
         SatelliteSampleKey.time_utc: da.time_utc.values.astype(float),
+        SatelliteSampleKey.x_geostationary: da.x_geostationary.values,
+        SatelliteSampleKey.y_geostationary: da.y_geostationary.values,
     }
 
-    for sample_key, dataset_key in (
-         (SatelliteSampleKey.x_geostationary, "x_geostationary"),
-        (SatelliteSampleKey.y_geostationary, "y_geostationary"),
-    ):
-        sample[sample_key] = da[dataset_key].values
-
     if t0_idx is not None:
         sample[SatelliteSampleKey.t0_idx] = t0_idx
 
-    return sample
+    return sample
+    

ocf_data_sampler/numpy_sample/site.py

@@ -18,9 +18,13 @@ class SiteSampleKey:
     time_cos = "site_time_cos"
 
 def convert_site_to_numpy_sample(da: xr.DataArray, t0_idx: int | None = None) -> dict:
-    """Convert from Xarray to NumpySample"""
+    """Convert from Xarray to NumpySample
+    
+    Args:
+        da: xarray DataArray containing site data
+        t0_idx: Index of the t0 timestamp in the time dimension of the site data
+    """
 
-    # Extract values from the DataArray
     sample = {
         SiteSampleKey.generation: da.values,
         SiteSampleKey.capacity_kwp: da.isel(time_utc=0)["capacity_kwp"].values,

ocf_data_sampler/numpy_sample/sun_position.py

@@ -27,16 +27,15 @@ def calculate_azimuth_and_elevation(
         latitude=lat,
         method='nrel_numpy'
     )
-    azimuth = solpos["azimuth"].values
-    elevation = solpos["elevation"].values
-    return azimuth, elevation
+
+    return solpos["azimuth"].values, solpos["elevation"].values
 
 
 def make_sun_position_numpy_sample(
-        datetimes: pd.DatetimeIndex, 
-        lon: float, 
-        lat: float, 
-        key_prefix: str = "gsp"
+    datetimes: pd.DatetimeIndex, 
+    lon: float, 
+    lat: float, 
+    key_prefix: str = "gsp"
 ) -> dict:
     """Creates NumpySample with standardized solar coordinates
 
@@ -44,22 +43,22 @@ def make_sun_position_numpy_sample(
         datetimes: The datetimes to calculate solar angles for
         lon: The longitude
         lat: The latitude
+        key_prefix: The prefix to add to the keys in the NumpySample
     """
     
     azimuth, elevation = calculate_azimuth_and_elevation(datetimes, lon, lat)
 
     # Normalise
-
     # Azimuth is in range [0, 360] degrees
     azimuth = azimuth / 360
 
     # Elevation is in range [-90, 90] degrees
     elevation = elevation / 180 + 0.5
-    
+
     # Make NumpySample
     sun_numpy_sample = {
         key_prefix + "_solar_azimuth": azimuth,
         key_prefix + "_solar_elevation": elevation,
     }
 
-    return sun_numpy_sample
+    return sun_numpy_sample

ocf_data_sampler/sample/__init__.py

@@ -1,10 +1,3 @@
 from ocf_data_sampler.sample.base import SampleBase
 from ocf_data_sampler.sample.uk_regional import UKRegionalSample
 from ocf_data_sampler.sample.site import SiteSample
-
-
-__all__ = [
-    'SampleBase',
-    'UKRegionalSample',
-    'SiteSample'
-    ]

ocf_data_sampler/sample/base.py

@@ -1,23 +1,15 @@
-"""
-Base class definition - abstract
-Handling of both flat and nested structures - consideration for NWP
-"""
+""" Base class for handling flat/nested data structures with NWP consideration """
 
-import logging
 import numpy as np
 import torch
-import xarray as xr
 
-from pathlib import Path
-from typing import Any, Dict, Optional, Union, TypeAlias
+from typing import TypeAlias
 from abc import ABC, abstractmethod
 
 
-logger = logging.getLogger(__name__)
-
-NumpySample: TypeAlias = Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
-NumpyBatch: TypeAlias = Dict[str, Union[np.ndarray, Dict[str, np.ndarray]]]
-TensorBatch: TypeAlias = Dict[str, Union[torch.Tensor, Dict[str, torch.Tensor]]]
+NumpySample: TypeAlias = dict[str, np.ndarray | dict[str, np.ndarray]]
+NumpyBatch: TypeAlias = dict[str, np.ndarray | dict[str, np.ndarray]]
+TensorBatch: TypeAlias = dict[str, torch.Tensor | dict[str, torch.Tensor]]
 
 
 class SampleBase(ABC):
@@ -26,43 +18,33 @@ class SampleBase(ABC):
     Provides core data storage functionality
     """
 
-    def __init__(self, data: Optional[Union[NumpySample, xr.Dataset]] = None):
-        """ Initialise data container """
-        logger.debug("Initialising SampleBase instance")
-        self._data = data
-
     @abstractmethod
     def to_numpy(self) -> NumpySample:
-        """ Convert data to a numpy array representation """
+        """Convert sample data to numpy format"""
         raise NotImplementedError
 
     @abstractmethod
-    def plot(self, **kwargs) -> None:
-        """ Abstract method for plotting """
+    def plot(self) -> None:
         raise NotImplementedError
 
     @abstractmethod
-    def save(self, path: Union[str, Path]) -> None:
-        """ Abstract method for saving sample data """
+    def save(self, path: str) -> None:
         raise NotImplementedError
 
     @classmethod
     @abstractmethod
-    def load(cls, path: Union[str, Path]) -> 'SampleBase':
-        """ Abstract class method for loading sample data """
+    def load(cls, path: str) -> 'SampleBase':
         raise NotImplementedError
 
 
 def batch_to_tensor(batch: NumpyBatch) -> TensorBatch:
     """
-    Moves ndarrays in a nested dict to torch tensors
+    Recursively converts numpy arrays in nested dict to torch tensors
     Args:
         batch: NumpyBatch with data in numpy arrays
     Returns:
         TensorBatch with data in torch tensors
     """
-    if not batch:
-        raise ValueError("Cannot convert empty batch to tensors")
 
     for k, v in batch.items():
         if isinstance(v, dict):
@@ -75,16 +57,15 @@ def batch_to_tensor(batch: NumpyBatch) -> TensorBatch:
     return batch
 
 
-def copy_batch_to_device(batch: dict, device: torch.device) -> dict:
-    """
-    Moves tensor leaves in a nested dict to a new device.
+def copy_batch_to_device(batch: TensorBatch, device: torch.device) -> TensorBatch:
+    """Recursively copies tensors in nested dict to specified device
 
     Args:
-        batch: Nested dict with tensors to move.
-        device: Device to move tensors to.
-
+        batch: Nested dict with tensors to move
+        device: Device to move tensors to
+    
     Returns:
-        A dict with tensors moved to the new device.
+        A dict with tensors moved to the new device
     """
     batch_copy = {}
 

ocf_data_sampler/sample/site.py

@@ -1,81 +1,44 @@
-""" 
-PVNet - Site sample / dataset implementation
-"""
+"""PVNet Site sample implementation for netCDF data handling and conversion"""
 
-import logging
 import xarray as xr
-import numpy as np
 
-from pathlib import Path
-from typing import Dict, Any, Union
+from typing_extensions import override
 
-from ocf_data_sampler.sample.base import SampleBase
+from ocf_data_sampler.sample.base import SampleBase, NumpySample
 from ocf_data_sampler.torch_datasets.datasets.site import convert_netcdf_to_numpy_sample
 
 
-logger = logging.getLogger(__name__)
-
-
 class SiteSample(SampleBase):
-    """ Sample class specific to Site PVNet """
+    """Handles PVNet site specific netCDF operations"""
 
-    def __init__(self):
-        logger.debug("Initialise SiteSample instance")
-        super().__init__()
-        self._data = {}
-
-    def to_numpy(self) -> Dict[str, Any]:
-        """ Convert sample numpy arrays - netCDF conversion """
-        logger.debug("Converting site sample to numpy format")
+    def __init__(self, data: xr.Dataset):
+        
+        if not isinstance(data, xr.Dataset):
+            raise TypeError(f"Data must be xarray Dataset - Found type {type(data)}")
         
-        try:
-            if not isinstance(self._data, xr.Dataset):
-                raise TypeError("Data must be xarray Dataset")
-            
-            numpy_data = convert_netcdf_to_numpy_sample(self._data)
+        self._data = data
 
-            logger.debug("Successfully converted to numpy format")
-            return numpy_data
-            
-        except Exception as e:
-            logger.error(f"Error converting to numpy: {str(e)}")
-            raise
+    @override
+    def to_numpy(self) -> NumpySample:                    
+        return convert_netcdf_to_numpy_sample(self._data)
 
-    def save(self, path: Union[str, Path]) -> None:
-        """ Save site sample as netCDF - h5netcdf engine """
-        logger.debug(f"Saving SiteSample to {path}")
-        path = Path(path)
+    def save(self, path: str) -> None:
+        """Save site sample data as netCDF
         
-        if path.suffix != '.nc':
-            logger.error(f"Invalid file format - {path.suffix}")
-            raise ValueError("Only .nc format is supported")
-        
-        if not isinstance(self._data, xr.Dataset):
-            raise TypeError("Data must be xarray Dataset for saving")
-                
-        self._data.to_netcdf(
-            path, 
-            mode="w", 
-            engine="h5netcdf"
-        )
-        logger.debug(f"Successfully saved SiteSample - {path}")
+        Args:
+            path: Path to save the netCDF file
+        """
+        self._data.to_netcdf(path, mode="w", engine="h5netcdf")
 
     @classmethod
-    def load(cls, path: str) -> None:
-        """ Load site sample from netCDF """
-        logger.debug(f"Loading SiteSample from {path}")
-        path = Path(path)
-        
-        if path.suffix != '.nc':
-            logger.error(f"Invalid file format - {path.suffix}")
-            raise ValueError("Only .nc format is supported")
+    def load(cls, path: str) -> 'SiteSample':
+        """Load site sample data from netCDF
         
-        instance = cls()
-        instance._data = xr.open_dataset(path)
-        logger.debug(f"Loaded SiteSample from {path}")
-        return instance
-
-    # TO DO - placeholder for now
-    def plot(self, **kwargs) -> None:
-        """ Plot sample data - placeholder """
-        pass
+        Args:
+            path: Path to load the netCDF file from
+        """
+        return cls(xr.open_dataset(path))
+
+    # TODO - placeholder for now
+    def plot(self) -> None:
+        raise NotImplementedError("Plotting not yet implemented for SiteSample")