PVNet 5.0.26__tar.gz → 5.1.1__tar.gz

{pvnet-5.0.26 → pvnet-5.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.0.26
+Version: 5.1.1
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: >=3.11
@@ -142,12 +142,6 @@ pip install -e <PATH-TO-ocf-data-sampler-REPO>
 If you install the local version of `ocf-data-sampler` that is more recent than the version 
 specified in `PVNet` it is not guarenteed to function properly with this library.
 
-## Streaming samples (no pre-save)
-
-PVNet now trains and validates directly from **streamed_samples** (i.e. no pre-saving to disk).
-
-Make sure you have copied example configs (as already stated above):
-cp -r configs.example configs
 
 ### Set up and config example for streaming
 

{pvnet-5.0.26 → pvnet-5.1.1}/PVNet.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.0.26
+Version: 5.1.1
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: >=3.11
@@ -142,12 +142,6 @@ pip install -e <PATH-TO-ocf-data-sampler-REPO>
 If you install the local version of `ocf-data-sampler` that is more recent than the version 
 specified in `PVNet` it is not guarenteed to function properly with this library.
 
-## Streaming samples (no pre-save)
-
-PVNet now trains and validates directly from **streamed_samples** (i.e. no pre-saving to disk).
-
-Make sure you have copied example configs (as already stated above):
-cp -r configs.example configs
 
 ### Set up and config example for streaming
 

{pvnet-5.0.26 → pvnet-5.1.1}/PVNet.egg-info/SOURCES.txt

@@ -7,13 +7,10 @@ PVNet.egg-info/dependency_links.txt
 PVNet.egg-info/requires.txt
 PVNet.egg-info/top_level.txt
 pvnet/__init__.py
+pvnet/datamodule.py
 pvnet/load_model.py
 pvnet/optimizers.py
 pvnet/utils.py
-pvnet/data/__init__.py
-pvnet/data/base_datamodule.py
-pvnet/data/site_datamodule.py
-pvnet/data/uk_regional_datamodule.py
 pvnet/models/__init__.py
 pvnet/models/base_model.py
 pvnet/models/ensemble.py
@@ -33,4 +30,5 @@ pvnet/training/__init__.py
 pvnet/training/lightning_module.py
 pvnet/training/plots.py
 pvnet/training/train.py
+tests/test_datamodule.py
 tests/test_end2end.py

{pvnet-5.0.26 → pvnet-5.1.1}/README.md

@@ -113,12 +113,6 @@ pip install -e <PATH-TO-ocf-data-sampler-REPO>
 If you install the local version of `ocf-data-sampler` that is more recent than the version 
 specified in `PVNet` it is not guarenteed to function properly with this library.
 
-## Streaming samples (no pre-save)
-
-PVNet now trains and validates directly from **streamed_samples** (i.e. no pre-saving to disk).
-
-Make sure you have copied example configs (as already stated above):
-cp -r configs.example configs
 
 ### Set up and config example for streaming
 

pvnet-5.0.26/pvnet/data/base_datamodule.py → pvnet-5.1.1/pvnet/datamodule.py

@@ -1,13 +1,14 @@
 """ Data module for pytorch lightning """
 
 import os
-from glob import glob
 
 import numpy as np
 from lightning.pytorch import LightningDataModule
 from ocf_data_sampler.numpy_sample.collate import stack_np_samples_into_batch
 from ocf_data_sampler.numpy_sample.common_types import NumpySample, TensorBatch
-from ocf_data_sampler.torch_datasets.sample.base import SampleBase, batch_to_tensor
+from ocf_data_sampler.torch_datasets.datasets.pvnet_uk import PVNetUKRegionalDataset
+from ocf_data_sampler.torch_datasets.datasets.site import SitesDataset
+from ocf_data_sampler.torch_datasets.sample.base import batch_to_tensor
 from torch.utils.data import DataLoader, Dataset, Subset
 
 
@@ -16,87 +17,8 @@ def collate_fn(samples: list[NumpySample]) -> TensorBatch:
     return batch_to_tensor(stack_np_samples_into_batch(samples))
 
 
-class PresavedSamplesDataset(Dataset):
-    """Dataset of pre-saved samples
-
-    Args:
-        sample_dir: Path to the directory of pre-saved samples.
-        sample_class: sample class type to use for save/load/to_numpy
-    """
-
-    def __init__(self, sample_dir: str, sample_class: SampleBase):
-        """Initialise PresavedSamplesDataset"""
-        self.sample_paths = glob(f"{sample_dir}/*")
-        self.sample_class = sample_class
-
-    def __len__(self) -> int:
-        return len(self.sample_paths)
-
-    def __getitem__(self, idx) -> NumpySample:
-        sample = self.sample_class.load(self.sample_paths[idx])
-        return sample.to_numpy()
-
-
-class BasePresavedDataModule(LightningDataModule):
-    """Base Datamodule for loading pre-saved samples."""
-
-    def __init__(
-        self,
-        sample_dir: str,
-        batch_size: int = 16,
-        num_workers: int = 0,
-        prefetch_factor: int | None = None,
-        persistent_workers: bool = False,
-        pin_memory: bool = False,
-    ):
-        """Base Datamodule for loading pre-saved samples
-
-        Args:
-            sample_dir: Path to the directory of pre-saved samples.
-            batch_size: Batch size.
-            num_workers: Number of workers to use in multiprocess batch loading.
-            prefetch_factor: Number of batches loaded in advance by each worker.
-            persistent_workers: If True, the data loader will not shut down the worker processes 
-                after a dataset has been consumed once. This allows to maintain the workers Dataset 
-                instances alive.
-            pin_memory: If True, the data loader will copy Tensors into device/CUDA pinned memory 
-                before returning them.
-        """
-        super().__init__()
-
-        self.sample_dir = sample_dir
-
-        self._common_dataloader_kwargs = dict(
-            batch_size=batch_size,
-            sampler=None,
-            batch_sampler=None,
-            num_workers=num_workers,
-            collate_fn=collate_fn,
-            pin_memory=pin_memory,
-            drop_last=False,
-            timeout=0,
-            worker_init_fn=None,
-            prefetch_factor=prefetch_factor,
-            persistent_workers=persistent_workers,
-            multiprocessing_context="spawn" if num_workers>0 else None,
-        )
-
-    def _get_premade_samples_dataset(self, subdir: str) -> Dataset:
-        raise NotImplementedError
-
-    def train_dataloader(self) -> DataLoader:
-        """Construct train dataloader"""
-        dataset = self._get_premade_samples_dataset("train")
-        return DataLoader(dataset, shuffle=True, **self._common_dataloader_kwargs)
-
-    def val_dataloader(self) -> DataLoader:
-        """Construct val dataloader"""
-        dataset = self._get_premade_samples_dataset("val")
-        return DataLoader(dataset, shuffle=False, **self._common_dataloader_kwargs)
-
-
-class BaseStreamedDataModule(LightningDataModule):
-    """Base Datamodule which streams samples using a sampler for ocf-data-sampler."""
+class BaseDataModule(LightningDataModule):
+    """Base Datamodule which streams samples using a sampler from ocf-data-sampler."""
 
     def __init__(
         self,
@@ -159,10 +81,10 @@ class BaseStreamedDataModule(LightningDataModule):
         if stage == "fit":
 
             # Prepare the train dataset
-            self.train_dataset = self._get_streamed_samples_dataset(*self.train_period)
+            self.train_dataset = self._get_dataset(*self.train_period)
 
             # Prepare and pre-shuffle the val dataset and set seed for reproducibility
-            val_dataset = self._get_streamed_samples_dataset(*self.val_period)
+            val_dataset = self._get_dataset(*self.val_period)
 
             shuffled_indices = np.random.default_rng(seed=self.seed).permutation(len(val_dataset))
             self.val_dataset = Subset(val_dataset, shuffled_indices)
@@ -194,11 +116,7 @@ class BaseStreamedDataModule(LightningDataModule):
                 if os.path.exists(filepath):
                     os.remove(filepath)
 
-    def _get_streamed_samples_dataset(
-        self,
-        start_time: str | None,
-        end_time: str | None
-    ) -> Dataset:
+    def _get_dataset(self, start_time: str | None, end_time: str | None) -> Dataset:
         raise NotImplementedError
 
     def train_dataloader(self) -> DataLoader:
@@ -208,3 +126,17 @@ class BaseStreamedDataModule(LightningDataModule):
     def val_dataloader(self) -> DataLoader:
         """Construct val dataloader"""
         return DataLoader(self.val_dataset, shuffle=False, **self._common_dataloader_kwargs)
+
+
+class UKRegionalDataModule(BaseDataModule):
+    """Datamodule for streaming UK regional samples."""
+
+    def _get_dataset(self, start_time: str | None, end_time: str | None) -> PVNetUKRegionalDataset:
+        return PVNetUKRegionalDataset(self.configuration, start_time=start_time, end_time=end_time)
+
+
+class SitesDataModule(BaseDataModule):
+    """Datamodule for streaming site samples."""
+
+    def _get_dataset(self, start_time: str | None, end_time: str | None) -> SitesDataset:
+        return SitesDataset(self.configuration, start_time=start_time, end_time=end_time)

{pvnet-5.0.26 → pvnet-5.1.1}/pvnet/load_model.py

@@ -73,6 +73,8 @@ def get_model_from_checkpoints(
         else:
             raise FileNotFoundError(f"File {data_config} does not exist")
 
+        # TODO: This should be removed in a future release since no new models will be trained on 
+        # presaved samples
         # Check for datamodule config
         # This only exists if the model was trained with presaved samples
         datamodule_config = f"{path}/{DATAMODULE_CONFIG_NAME}"

{pvnet-5.0.26 → pvnet-5.1.1}/pvnet/training/lightning_module.py

@@ -11,7 +11,7 @@ import xarray as xr
 from ocf_data_sampler.numpy_sample.common_types import TensorBatch
 from ocf_data_sampler.torch_datasets.sample.base import copy_batch_to_device
 
-from pvnet.data.base_datamodule import collate_fn
+from pvnet.datamodule import collate_fn
 from pvnet.models.base_model import BaseModel
 from pvnet.optimizers import AbstractOptimizer
 from pvnet.training.plots import plot_sample_forecasts, wandb_line_plot

{pvnet-5.0.26 → pvnet-5.1.1}/pvnet/training/train.py

@@ -15,10 +15,8 @@ from lightning.pytorch.callbacks import ModelCheckpoint
 from lightning.pytorch.loggers import Logger, WandbLogger
 from omegaconf import DictConfig, OmegaConf
 
-from pvnet.data.base_datamodule import BasePresavedDataModule
 from pvnet.utils import (
     DATA_CONFIG_NAME,
-    DATAMODULE_CONFIG_NAME,
     FULL_CONFIG_NAME,
     MODEL_CONFIG_NAME,
 )
@@ -102,27 +100,8 @@ def train(config: DictConfig) -> None:
                 os.makedirs(save_dir, exist_ok=True)
                 OmegaConf.save(config.model, f"{save_dir}/{MODEL_CONFIG_NAME}")
 
-                # If using pre-saved samples we need to extract the data config from the directory
-                # those samples were saved to
-                if isinstance(datamodule, BasePresavedDataModule):
-                    data_config = f"{config.datamodule.sample_dir}/{DATA_CONFIG_NAME}"
-
-                    # We also save the datamodule config used to create the samples to the output 
-                    # directory and to wandb
-                    shutil.copyfile(
-                        f"{config.datamodule.sample_dir}/{DATAMODULE_CONFIG_NAME}", 
-                        f"{save_dir}/{DATAMODULE_CONFIG_NAME}"
-                    )
-                    wandb_logger.experiment.save(
-                        f"{save_dir}/{DATAMODULE_CONFIG_NAME}", 
-                        base_path=save_dir,
-                    )
-                else:
-                    # If we are streaming batches the data config is defined and we don't need to
-                    # save the datamodule config separately
-                    data_config = config.datamodule.configuration
-
                 # Save the data config to the output directory and to wandb
+                data_config = config.datamodule.configuration
                 shutil.copyfile(data_config, f"{save_dir}/{DATA_CONFIG_NAME}")
                 wandb_logger.experiment.save(f"{save_dir}/{DATA_CONFIG_NAME}", base_path=save_dir)
 

pvnet-5.1.1/tests/test_datamodule.py

@@ -0,0 +1,15 @@
+from pvnet.datamodule import SitesDataModule
+
+
+
+def test_sites_data_module(site_data_config_path):
+    """Test SitesDataModule initialization"""
+
+    _ = SitesDataModule(
+        configuration=site_data_config_path,
+        batch_size=2,
+        num_workers=0,
+        prefetch_factor=None,
+        train_period=[None, None],
+        val_period=[None, None],
+    )

{pvnet-5.0.26 → pvnet-5.1.1}/tests/test_end2end.py

@@ -1,6 +1,6 @@
 import lightning
 
-from pvnet.data import UKRegionalStreamedDataModule
+from pvnet.datamodule import UKRegionalDataModule
 from pvnet.optimizers import EmbAdamWReduceLROnPlateau
 from pvnet.training.lightning_module import PVNetLightningModule
 
@@ -8,7 +8,7 @@ from pvnet.training.lightning_module import PVNetLightningModule
 def test_model_trainer_fit(session_tmp_path, uk_data_config_path, late_fusion_model):
     """Test end-to-end training."""
 
-    datamodule = UKRegionalStreamedDataModule(
+    datamodule = UKRegionalDataModule(
         configuration=uk_data_config_path,
         batch_size=2,
         num_workers=2,

pvnet-5.0.26/pvnet/data/__init__.py

@@ -1,3 +0,0 @@
-"""Data parts"""
-from .site_datamodule import SitePresavedDataModule, SiteStreamedDataModule
-from .uk_regional_datamodule import UKRegionalPresavedDataModule, UKRegionalStreamedDataModule

pvnet-5.0.26/pvnet/data/site_datamodule.py

@@ -1,29 +0,0 @@
-""" Data module for pytorch lightning """
-
-from ocf_data_sampler.torch_datasets.datasets.site import SitesDataset
-from ocf_data_sampler.torch_datasets.sample.site import SiteSample
-from torch.utils.data import Dataset
-
-from pvnet.data.base_datamodule import (
-    BasePresavedDataModule,
-    BaseStreamedDataModule,
-    PresavedSamplesDataset,
-)
-
-
-class SitePresavedDataModule(BasePresavedDataModule):
-    """Datamodule for loading pre-saved samples."""
-
-    def _get_premade_samples_dataset(self, subdir: str) -> Dataset:
-        return PresavedSamplesDataset(f"{self.sample_dir}/{subdir}", SiteSample)
-
-
-class SiteStreamedDataModule(BaseStreamedDataModule):
-    """Datamodule which streams samples using sampler for ocf-data-sampler."""
-
-    def _get_streamed_samples_dataset(
-        self,
-        start_time: str | None,
-        end_time: str | None
-    ) -> Dataset:
-        return SitesDataset(self.configuration, start_time=start_time, end_time=end_time)

pvnet-5.0.26/pvnet/data/uk_regional_datamodule.py

@@ -1,29 +0,0 @@
-""" Data module for pytorch lightning """
-
-from ocf_data_sampler.torch_datasets.datasets.pvnet_uk import PVNetUKRegionalDataset
-from ocf_data_sampler.torch_datasets.sample.uk_regional import UKRegionalSample
-from torch.utils.data import Dataset
-
-from pvnet.data.base_datamodule import (
-    BasePresavedDataModule,
-    BaseStreamedDataModule,
-    PresavedSamplesDataset,
-)
-
-
-class UKRegionalPresavedDataModule(BasePresavedDataModule):
-    """Datamodule for loading pre-saved samples."""
-
-    def _get_premade_samples_dataset(self, subdir: str) -> Dataset:
-        return PresavedSamplesDataset(f"{self.sample_dir}/{subdir}", UKRegionalSample)
-
-
-class UKRegionalStreamedDataModule(BaseStreamedDataModule):
-    """Datamodule which streams samples using sampler for ocf-data-sampler."""
-
-    def _get_streamed_samples_dataset(
-        self,
-        start_time: str | None,
-        end_time: str | None
-    ) -> Dataset:
-        return PVNetUKRegionalDataset(self.configuration, start_time=start_time, end_time=end_time)