ocf-data-sampler 0.5.41__py3-none-any.whl → 0.6.1__py3-none-any.whl

ocf_data_sampler/load/site.py

@@ -25,11 +25,20 @@ def open_site(generation_file_path: str, metadata_file_path: str) -> xr.DataArra
     metadata_df = metadata_df.reindex(generation_ds.site_id.values)
 
     # Assign coordinates to the Dataset using the aligned metadata
-    generation_ds = generation_ds.assign_coords(
-        latitude=("site_id", metadata_df["latitude"].values),
-        longitude=("site_id", metadata_df["longitude"].values),
-        capacity_kwp=("site_id", metadata_df["capacity_kwp"].values),
-    )
+    # Check if variable capacity was passed with the generation data
+    # If not assign static capacity from metadata
+    if hasattr(generation_ds,"capacity_kwp"):
+        generation_ds = generation_ds.assign_coords(
+            latitude=(metadata_df.latitude.to_xarray()),
+            longitude=(metadata_df.longitude.to_xarray()),
+            capacity_kwp=generation_ds.capacity_kwp,
+        )
+    else:
+        generation_ds = generation_ds.assign_coords(
+            latitude=(metadata_df.latitude.to_xarray()),
+            longitude=(metadata_df.longitude.to_xarray()),
+            capacity_kwp=(metadata_df.capacity_kwp.to_xarray()),
+        )
 
     # Sanity checks, to prevent inf or negative values
     # Note NaNs are allowed in generation_kw as can have non overlapping time periods for sites

ocf_data_sampler/torch_datasets/{sample/base.py → utils/torch_batch_utils.py}

@@ -1,36 +1,9 @@
-"""Base class for handling flat/nested data structures with NWP consideration."""
-
-from abc import ABC, abstractmethod
+"""Functions to convert batches to tensors and move them to a given device."""
 
 import numpy as np
 import torch
 
-from ocf_data_sampler.numpy_sample.common_types import NumpyBatch, NumpySample, TensorBatch
-
-
-class SampleBase(ABC):
-    """Abstract base class for all sample types."""
-
-    @abstractmethod
-    def to_numpy(self) -> NumpySample:
-        """Convert sample data to numpy format."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def plot(self) -> None:
-        """Create a visualisation of the data."""
-        raise NotImplementedError
-
-    @abstractmethod
-    def save(self, path: str) -> None:
-        """Saves the sample to disk in the implementations' required format."""
-        raise NotImplementedError
-
-    @classmethod
-    @abstractmethod
-    def load(cls, path: str) -> "SampleBase":
-        """Load a sample from disk from the implementations' format."""
-        raise NotImplementedError
+from ocf_data_sampler.numpy_sample.common_types import NumpyBatch, TensorBatch
 
 
 def batch_to_tensor(batch: NumpyBatch) -> TensorBatch:

{ocf_data_sampler-0.5.41.dist-info → ocf_data_sampler-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocf-data-sampler
-Version: 0.5.41
+Version: 0.6.1
 Author: James Fulton, Peter Dudfield
 Author-email: Open Climate Fix team <info@openclimatefix.org>
 License: MIT License

{ocf_data_sampler-0.5.41.dist-info → ocf_data_sampler-0.6.1.dist-info}/RECORD

@@ -11,7 +11,7 @@ ocf_data_sampler/load/gsp.py,sha256=zsQ39dZBS45qd86lGfCZUjheLRTtMzIUozj-j8c87UQ,
 ocf_data_sampler/load/load_dataset.py,sha256=K8rWykjII-3g127If7WRRFivzHNx3SshCvZj4uQlf28,2089
 ocf_data_sampler/load/open_xarray_tensorstore.py,sha256=YglCBeKa4mSjUU5qlcMOLZXUtFrPFWVKDeKHLjs_YbA,6353
 ocf_data_sampler/load/satellite.py,sha256=5o5SfcplQfZFlm3JJq73j8_m_cWKpFtKk0tTKGjjCuE,1856
-ocf_data_sampler/load/site.py,sha256=bpFABjpvlstn6yJ6OPVlPZms-CjJdxNwCkQafpnj0Ik,2539
+ocf_data_sampler/load/site.py,sha256=ZH85pgLTXTxa8zPHdJ-Y9Yy2ORDc3vRX2uD1I6uDI6g,2939
 ocf_data_sampler/load/utils.py,sha256=AGL0aOOQPrgqNBTjlBtR7Qg1PyQov3DFJo-y198u8pY,2044
 ocf_data_sampler/load/nwp/__init__.py,sha256=SmcrnbygO5xtCKmGR4wtHrj-HI7nOAvnAtfuvRufBGQ,25
 ocf_data_sampler/load/nwp/nwp.py,sha256=3nAsbw2yjmaF0YEHbCDciPLuIRBGxGuvixlMgKNc3zU,3570
@@ -44,10 +44,6 @@ ocf_data_sampler/torch_datasets/datasets/__init__.py,sha256=o0SsEXXZ6k9iL__5_RN1
 ocf_data_sampler/torch_datasets/datasets/picklecache.py,sha256=b8T5lgKfiPXLwuVQuFpCQBlU-HNBrA-Z-eSwYICKvsQ,1350
 ocf_data_sampler/torch_datasets/datasets/pvnet_uk.py,sha256=TDSraKUYE2wA5YF9SYO2RmAIbDJtcvfTtxr7WmYDszg,12385
 ocf_data_sampler/torch_datasets/datasets/site.py,sha256=IFMVy8c887mfT2NmcKX8ocm1OvlBhvzJNye_scfBMFQ,15265
-ocf_data_sampler/torch_datasets/sample/__init__.py,sha256=GL84vdZl_SjHDGVyh9Uekx2XhPYuZ0dnO3l6f6KXnHI,100
-ocf_data_sampler/torch_datasets/sample/base.py,sha256=cQ1oIyhdmlotejZK8B3Cw6MNvpdnBPD8G_o2h7Ye4Vc,2206
-ocf_data_sampler/torch_datasets/sample/site.py,sha256=40NwNTqjL1WVhPdwe02zDHHfDLG2u_bvCfRCtGAtFc0,1466
-ocf_data_sampler/torch_datasets/sample/uk_regional.py,sha256=GtChjvbFB-BqkGftgZyh8jgTUX9IostYeyaSoVMCzp0,10481
 ocf_data_sampler/torch_datasets/utils/__init__.py,sha256=4l1VcEmxHInU9G66zrimNMa8WcyKUASQST_iF9QfxUw,457
 ocf_data_sampler/torch_datasets/utils/add_alterate_coordinate_projections.py,sha256=w6Q4TyxNyl7PKAbhqiXvqOpnqIjwmOUcGREIvPNGYlQ,2666
 ocf_data_sampler/torch_datasets/utils/config_normalization_values_to_dicts.py,sha256=SGt1H2nXcaj44ND14-gHzvA7dkLfgjTacCq7rOkRGwg,1991
@@ -55,11 +51,12 @@ ocf_data_sampler/torch_datasets/utils/diff_nwp_data.py,sha256=o7NpKWxKHhwMbol3xB
 ocf_data_sampler/torch_datasets/utils/merge_and_fill_utils.py,sha256=VkQv4wJVihObh_OiSuwKqV_w2lEOweaYgJPkm075CZc,2132
 ocf_data_sampler/torch_datasets/utils/spatial_slice_for_dataset.py,sha256=Hvz0wHSWMYYamf2oHNiGlzJcM4cAH6pL_7ZEvIBL2dE,1882
 ocf_data_sampler/torch_datasets/utils/time_slice_for_dataset.py,sha256=1r1J2KNSo1_imN9gpVf5AupJaZ7VSnSevS1o_wck440,3925
+ocf_data_sampler/torch_datasets/utils/torch_batch_utils.py,sha256=yL_24FOKGs9rYg9V3ZTc3LXd8_9N6EFz2yxnxPt_bi8,1434
 ocf_data_sampler/torch_datasets/utils/valid_time_periods.py,sha256=xcy75cVxl0WrglnX5YUAFjXXlO2GwEBHWyqo8TDuiOA,4714
 ocf_data_sampler/torch_datasets/utils/validation_utils.py,sha256=YqmT-lExWlI8_ul3l0EP73Ik002fStr_bhsZh9mQqEU,4735
 scripts/download_gsp_location_data.py,sha256=rRDXMoqX-RYY4jPdxhdlxJGhWdl6r245F5UARgKV6P4,3121
 scripts/refactor_site.py,sha256=skzvsPP0Cn9yTKndzkilyNcGz4DZ88ctvCJ0XrBdc2A,3135
-ocf_data_sampler-0.5.41.dist-info/METADATA,sha256=-wCdU7dBURk_1dkDIuNOPZjfVkanwkbtHhfkSxhkOVg,13541
-ocf_data_sampler-0.5.41.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ocf_data_sampler-0.5.41.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
-ocf_data_sampler-0.5.41.dist-info/RECORD,,
+ocf_data_sampler-0.6.1.dist-info/METADATA,sha256=w0MKJCWR-a9VPWtGxWVP3_h2t5-V4ny1PiofevouGHY,13540
+ocf_data_sampler-0.6.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ocf_data_sampler-0.6.1.dist-info/top_level.txt,sha256=deUxqmsONNAGZDNbsntbXH7BRA1MqWaUeAJrCo6q_xA,25
+ocf_data_sampler-0.6.1.dist-info/RECORD,,

ocf_data_sampler/torch_datasets/sample/__init__.py

@@ -1,3 +0,0 @@
-from .base import SampleBase
-from .uk_regional import UKRegionalSample
-from .site import SiteSample

ocf_data_sampler/torch_datasets/sample/site.py

@@ -1,48 +0,0 @@
-"""PVNet Site sample implementation for netCDF data handling and conversion."""
-
-import torch
-from typing_extensions import override
-
-from ocf_data_sampler.numpy_sample.common_types import NumpySample
-
-from .base import SampleBase
-
-
-# TODO this is now similar to the UKRegionalSample
-# We should consider just having one Sample class for all datasets
-class SiteSample(SampleBase):
-    """Handles SiteSample specific operations."""
-
-    def __init__(self, data: NumpySample) -> None:
-        """Initializes the SiteSample object with the given NumpySample."""
-        self._data = data
-
-    @override
-    def to_numpy(self) -> NumpySample:
-        return self._data
-
-    @override
-    def save(self, path: str) -> None:
-        """Saves sample to the specified path in pickle format."""
-        # Saves to pickle format
-        torch.save(self._data, path)
-
-    @classmethod
-    @override
-    def load(cls, path: str) -> "SiteSample":
-        """Loads sample from the specified path.
-
-        Args:
-            path: Path to the saved sample file.
-
-        Returns:
-            A SiteSample instance with the loaded data.
-        """
-        # Loads from .pt format
-        # TODO: We should move away from using torch.load(..., weights_only=False)
-        return cls(torch.load(path, weights_only=False))
-
-    @override
-    def plot(self) -> None:
-        # TODO - placeholder for now
-        raise NotImplementedError("Plotting not yet implemented for SiteSample")

ocf_data_sampler/torch_datasets/sample/uk_regional.py

@@ -1,262 +0,0 @@
-"""PVNet UK Regional sample implementation for dataset handling and visualisation."""
-
-import logging
-
-import torch
-from typing_extensions import override
-
-from ocf_data_sampler.config import Configuration
-from ocf_data_sampler.numpy_sample import (
-    GSPSampleKey,
-    NWPSampleKey,
-    SatelliteSampleKey,
-)
-from ocf_data_sampler.numpy_sample.common_types import NumpySample
-from ocf_data_sampler.torch_datasets.sample.base import SampleBase
-from ocf_data_sampler.torch_datasets.utils.validation_utils import (
-    calculate_expected_shapes,
-    check_dimensions,
-    validation_warning,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class UKRegionalSample(SampleBase):
-    """Handles UK Regional PVNet data operations."""
-
-    def __init__(self, data: NumpySample) -> None:
-        """Initialises UK Regional sample with data."""
-        self._data = data
-
-    @override
-    def to_numpy(self) -> NumpySample:
-        """Returns the data as a NumPy sample."""
-        return self._data
-
-    @override
-    def save(self, path: str) -> None:
-        """Saves sample to the specified path in pickle format."""
-        # Saves to pickle format
-        torch.save(self._data, path)
-
-    @classmethod
-    @override
-    def load(cls, path: str) -> "UKRegionalSample":
-        """Loads sample from the specified path.
-
-        Args:
-            path: Path to the saved sample file.
-
-        Returns:
-            A UKRegionalSample instance with the loaded data.
-        """
-        # Loads from .pt format
-        # TODO: We should move away from using torch.load(..., weights_only=False)
-        return cls(torch.load(path, weights_only=False))
-
-    def validate_sample(self, config: Configuration) -> dict:
-        """Validates the sample, logging warnings and raising errors.
-
-        Checks that the sample has the expected structure and data shapes based
-        on the provided configuration. Critical issues (missing required data,
-        shape mismatches) will raise a ValueError. Non-critical issues (e.g.,
-        unexpected data components found) will be logged as warnings using
-        the standard Python logging module.
-
-        Args:
-            config: Configuration object defining expected shapes and required fields.
-
-        Returns:
-            dict: A dictionary indicating success: `{"valid": True}`.
-                  If validation fails due to a critical issue, an exception is raised
-                  instead of returning. Warnings encountered are logged.
-
-        Raises:
-            TypeError: If `config` is not a Configuration object.
-            ValueError: For critical validation failures like missing expected data,
-                        incorrect data shapes, or missing required NWP providers.
-        """
-        if not isinstance(config, Configuration):
-            raise TypeError("config must be Configuration object")
-
-        # Calculate expected shapes from configuration
-        expected_shapes = calculate_expected_shapes(config)
-
-        # Check GSP shape if specified
-        gsp_key = GSPSampleKey.gsp
-        if gsp_key in expected_shapes and gsp_key not in self._data:
-            raise ValueError(f"Configuration expects GSP data ('{gsp_key}') but is missing.")
-
-        if gsp_key in self._data:
-            if gsp_key in expected_shapes:
-                gsp_data = self._data[gsp_key]
-                check_dimensions(
-                    actual_shape=gsp_data.shape,
-                    expected_shape=expected_shapes[gsp_key],
-                    name="GSP",
-                )
-            else:
-                validation_warning(
-                    message=f"GSP data ('{gsp_key}') is present but not expected in configuration.",
-                    warning_type="unexpected_component",
-                    component=str(gsp_key),
-                )
-
-        # Checks for NWP data
-        nwp_key = NWPSampleKey.nwp
-        if nwp_key in expected_shapes and nwp_key not in self._data:
-            raise ValueError(f"Configuration expects NWP data ('{nwp_key}') but is missing.")
-
-        if nwp_key in self._data:
-            nwp_data_all_providers = self._data[nwp_key]
-            if not isinstance(nwp_data_all_providers, dict):
-                raise ValueError(f"NWP data ('{nwp_key}') should be a dictionary.")
-
-            if nwp_key in expected_shapes:
-                expected_providers = set(expected_shapes[nwp_key].keys())
-                actual_providers = set(nwp_data_all_providers.keys())
-
-                unexpected_providers = actual_providers - expected_providers
-                if unexpected_providers:
-                    validation_warning(
-                        message=f"Unexpected NWP providers found: {list(unexpected_providers)}",
-                        warning_type="unexpected_provider",
-                        providers=list(unexpected_providers),
-                    )
-
-                missing_expected_providers = expected_providers - actual_providers
-                if missing_expected_providers:
-                    raise ValueError(
-                        f"Expected NWP providers are missing from the data: "
-                        f"{list(missing_expected_providers)}",
-                    )
-
-                for provider in expected_shapes[nwp_key]:
-                    provider_data = nwp_data_all_providers[provider]
-
-                    if "nwp" not in provider_data:
-                        error_msg = (
-                            f"Missing array key 'nwp' in NWP data for provider '{provider}'."
-                        )
-                        raise ValueError(error_msg)
-
-                    nwp_array = provider_data["nwp"]
-                    check_dimensions(
-                        actual_shape=nwp_array.shape,
-                        expected_shape=expected_shapes[nwp_key][provider],
-                        name=f"NWP data ({provider})",
-                    )
-            else:
-                validation_warning(
-                    message=(
-                        f"NWP data ('{nwp_key}') is present but not expected "
-                        "in configuration."
-                    ),
-                    warning_type="unexpected_component",
-                    component=str(nwp_key),
-                )
-
-        # Validate satellite data
-        sat_key = SatelliteSampleKey.satellite_actual
-        if sat_key in expected_shapes and sat_key not in self._data:
-            raise ValueError(f"Configuration expects Satellite data ('{sat_key}') but is missing.")
-
-        if sat_key in self._data:
-            if sat_key in expected_shapes:
-                sat_data = self._data[sat_key]
-                check_dimensions(
-                    actual_shape=sat_data.shape,
-                    expected_shape=expected_shapes[sat_key],
-                    name="Satellite data",
-                )
-            else:
-                validation_warning(
-                    message=(
-                        f"Satellite data ('{sat_key}') is present but not expected "
-                        "in configuration."
-                    ),
-                    warning_type="unexpected_component",
-                    component=str(sat_key),
-                )
-
-        # Validate solar coordinates data
-        solar_keys = ["solar_azimuth", "solar_elevation"]
-        for solar_key in solar_keys:
-            solar_name = solar_key.replace("_", " ").title()
-            if solar_key in expected_shapes and solar_key not in self._data:
-                raise ValueError(f"Configuration expects {solar_key} data but is missing.")
-
-            if solar_key in self._data:
-                if solar_key in expected_shapes:
-                    solar_data = self._data[solar_key]
-                    check_dimensions(
-                        actual_shape=solar_data.shape,
-                        expected_shape=expected_shapes[solar_key],
-                        name=f"{solar_name} data",
-                    )
-                else:
-                    validation_warning(
-                        message=(
-                            f"{solar_name} data is present but not expected "
-                            "in configuration."
-                        ),
-                        warning_type="unexpected_component",
-                        component=solar_key,
-                    )
-
-        # Check for potentially unexpected components
-        checked_keys = {gsp_key, nwp_key, sat_key} | set(solar_keys)
-        all_present_keys = set(self._data.keys())
-        unexpected_present_keys = all_present_keys - set(expected_shapes.keys())
-
-        for key in unexpected_present_keys:
-            if key not in checked_keys:
-                validation_warning(
-                    message=(
-                        f"Unexpected component '{key}' is present in data but not defined "
-                        "in configuration's expected shapes."
-                    ),
-                    warning_type="unexpected_component",
-                    component=str(key),
-                )
-
-        return {
-            "valid": True,
-        }
-
-
-    @override
-    def plot(self) -> None:
-        """Plots the sample data for visualization."""
-        from matplotlib import pyplot as plt
-
-        _, axes = plt.subplots(2, 2, figsize=(12, 8))
-
-        if NWPSampleKey.nwp in self._data:
-            first_nwp = next(iter(self._data[NWPSampleKey.nwp].values()))
-            if "nwp" in first_nwp:
-                axes[0, 1].imshow(first_nwp["nwp"][0])
-                title = "NWP (First Channel)"
-                if NWPSampleKey.channel_names in first_nwp:
-                    channel_names = first_nwp[NWPSampleKey.channel_names]
-                    if channel_names:
-                        title = f"NWP: {channel_names[0]}"
-                axes[0, 1].set_title(title)
-
-        if GSPSampleKey.gsp in self._data:
-            axes[0, 0].plot(self._data[GSPSampleKey.gsp])
-            axes[0, 0].set_title("GSP Generation")
-
-        if "solar_azimuth" in self._data and "solar_elevation" in self._data:
-            axes[1, 1].plot(self._data["solar_azimuth"], label="Azimuth")
-            axes[1, 1].plot(self._data["solar_elevation"], label="Elevation")
-            axes[1, 1].set_title("Solar Position")
-            axes[1, 1].legend()
-
-        if SatelliteSampleKey.satellite_actual in self._data:
-            axes[1, 0].imshow(self._data[SatelliteSampleKey.satellite_actual])
-            axes[1, 0].set_title("Satellite Data")
-
-        plt.tight_layout()
-        plt.show()