PVNet 5.0.26__py3-none-any.whl → 5.1.1__py3-none-any.whl

pvnet/{data/base_datamodule.py → datamodule.py}

@@ -1,13 +1,14 @@
 """ Data module for pytorch lightning """
 
 import os
-from glob import glob
 
 import numpy as np
 from lightning.pytorch import LightningDataModule
 from ocf_data_sampler.numpy_sample.collate import stack_np_samples_into_batch
 from ocf_data_sampler.numpy_sample.common_types import NumpySample, TensorBatch
-from ocf_data_sampler.torch_datasets.sample.base import SampleBase, batch_to_tensor
+from ocf_data_sampler.torch_datasets.datasets.pvnet_uk import PVNetUKRegionalDataset
+from ocf_data_sampler.torch_datasets.datasets.site import SitesDataset
+from ocf_data_sampler.torch_datasets.sample.base import batch_to_tensor
 from torch.utils.data import DataLoader, Dataset, Subset
 
 
@@ -16,87 +17,8 @@ def collate_fn(samples: list[NumpySample]) -> TensorBatch:
     return batch_to_tensor(stack_np_samples_into_batch(samples))
 
 
-class PresavedSamplesDataset(Dataset):
-    """Dataset of pre-saved samples
-
-    Args:
-        sample_dir: Path to the directory of pre-saved samples.
-        sample_class: sample class type to use for save/load/to_numpy
-    """
-
-    def __init__(self, sample_dir: str, sample_class: SampleBase):
-        """Initialise PresavedSamplesDataset"""
-        self.sample_paths = glob(f"{sample_dir}/*")
-        self.sample_class = sample_class
-
-    def __len__(self) -> int:
-        return len(self.sample_paths)
-
-    def __getitem__(self, idx) -> NumpySample:
-        sample = self.sample_class.load(self.sample_paths[idx])
-        return sample.to_numpy()
-
-
-class BasePresavedDataModule(LightningDataModule):
-    """Base Datamodule for loading pre-saved samples."""
-
-    def __init__(
-        self,
-        sample_dir: str,
-        batch_size: int = 16,
-        num_workers: int = 0,
-        prefetch_factor: int | None = None,
-        persistent_workers: bool = False,
-        pin_memory: bool = False,
-    ):
-        """Base Datamodule for loading pre-saved samples
-
-        Args:
-            sample_dir: Path to the directory of pre-saved samples.
-            batch_size: Batch size.
-            num_workers: Number of workers to use in multiprocess batch loading.
-            prefetch_factor: Number of batches loaded in advance by each worker.
-            persistent_workers: If True, the data loader will not shut down the worker processes 
-                after a dataset has been consumed once. This allows to maintain the workers Dataset 
-                instances alive.
-            pin_memory: If True, the data loader will copy Tensors into device/CUDA pinned memory 
-                before returning them.
-        """
-        super().__init__()
-
-        self.sample_dir = sample_dir
-
-        self._common_dataloader_kwargs = dict(
-            batch_size=batch_size,
-            sampler=None,
-            batch_sampler=None,
-            num_workers=num_workers,
-            collate_fn=collate_fn,
-            pin_memory=pin_memory,
-            drop_last=False,
-            timeout=0,
-            worker_init_fn=None,
-            prefetch_factor=prefetch_factor,
-            persistent_workers=persistent_workers,
-            multiprocessing_context="spawn" if num_workers>0 else None,
-        )
-
-    def _get_premade_samples_dataset(self, subdir: str) -> Dataset:
-        raise NotImplementedError
-
-    def train_dataloader(self) -> DataLoader:
-        """Construct train dataloader"""
-        dataset = self._get_premade_samples_dataset("train")
-        return DataLoader(dataset, shuffle=True, **self._common_dataloader_kwargs)
-
-    def val_dataloader(self) -> DataLoader:
-        """Construct val dataloader"""
-        dataset = self._get_premade_samples_dataset("val")
-        return DataLoader(dataset, shuffle=False, **self._common_dataloader_kwargs)
-
-
-class BaseStreamedDataModule(LightningDataModule):
-    """Base Datamodule which streams samples using a sampler for ocf-data-sampler."""
+class BaseDataModule(LightningDataModule):
+    """Base Datamodule which streams samples using a sampler from ocf-data-sampler."""
 
     def __init__(
         self,
@@ -159,10 +81,10 @@ class BaseStreamedDataModule(LightningDataModule):
         if stage == "fit":
 
             # Prepare the train dataset
-            self.train_dataset = self._get_streamed_samples_dataset(*self.train_period)
+            self.train_dataset = self._get_dataset(*self.train_period)
 
             # Prepare and pre-shuffle the val dataset and set seed for reproducibility
-            val_dataset = self._get_streamed_samples_dataset(*self.val_period)
+            val_dataset = self._get_dataset(*self.val_period)
 
             shuffled_indices = np.random.default_rng(seed=self.seed).permutation(len(val_dataset))
             self.val_dataset = Subset(val_dataset, shuffled_indices)
@@ -194,11 +116,7 @@ class BaseStreamedDataModule(LightningDataModule):
                 if os.path.exists(filepath):
                     os.remove(filepath)
 
-    def _get_streamed_samples_dataset(
-        self,
-        start_time: str | None,
-        end_time: str | None
-    ) -> Dataset:
+    def _get_dataset(self, start_time: str | None, end_time: str | None) -> Dataset:
         raise NotImplementedError
 
     def train_dataloader(self) -> DataLoader:
@@ -208,3 +126,17 @@ class BaseStreamedDataModule(LightningDataModule):
     def val_dataloader(self) -> DataLoader:
         """Construct val dataloader"""
         return DataLoader(self.val_dataset, shuffle=False, **self._common_dataloader_kwargs)
+
+
+class UKRegionalDataModule(BaseDataModule):
+    """Datamodule for streaming UK regional samples."""
+
+    def _get_dataset(self, start_time: str | None, end_time: str | None) -> PVNetUKRegionalDataset:
+        return PVNetUKRegionalDataset(self.configuration, start_time=start_time, end_time=end_time)
+
+
+class SitesDataModule(BaseDataModule):
+    """Datamodule for streaming site samples."""
+
+    def _get_dataset(self, start_time: str | None, end_time: str | None) -> SitesDataset:
+        return SitesDataset(self.configuration, start_time=start_time, end_time=end_time)

pvnet/load_model.py

@@ -73,6 +73,8 @@ def get_model_from_checkpoints(
         else:
             raise FileNotFoundError(f"File {data_config} does not exist")
 
+        # TODO: This should be removed in a future release since no new models will be trained on 
+        # presaved samples
         # Check for datamodule config
         # This only exists if the model was trained with presaved samples
         datamodule_config = f"{path}/{DATAMODULE_CONFIG_NAME}"

pvnet/training/lightning_module.py

@@ -11,7 +11,7 @@ import xarray as xr
 from ocf_data_sampler.numpy_sample.common_types import TensorBatch
 from ocf_data_sampler.torch_datasets.sample.base import copy_batch_to_device
 
-from pvnet.data.base_datamodule import collate_fn
+from pvnet.datamodule import collate_fn
 from pvnet.models.base_model import BaseModel
 from pvnet.optimizers import AbstractOptimizer
 from pvnet.training.plots import plot_sample_forecasts, wandb_line_plot

pvnet/training/train.py

@@ -15,10 +15,8 @@ from lightning.pytorch.callbacks import ModelCheckpoint
 from lightning.pytorch.loggers import Logger, WandbLogger
 from omegaconf import DictConfig, OmegaConf
 
-from pvnet.data.base_datamodule import BasePresavedDataModule
 from pvnet.utils import (
     DATA_CONFIG_NAME,
-    DATAMODULE_CONFIG_NAME,
     FULL_CONFIG_NAME,
     MODEL_CONFIG_NAME,
 )
@@ -102,27 +100,8 @@ def train(config: DictConfig) -> None:
                 os.makedirs(save_dir, exist_ok=True)
                 OmegaConf.save(config.model, f"{save_dir}/{MODEL_CONFIG_NAME}")
 
-                # If using pre-saved samples we need to extract the data config from the directory
-                # those samples were saved to
-                if isinstance(datamodule, BasePresavedDataModule):
-                    data_config = f"{config.datamodule.sample_dir}/{DATA_CONFIG_NAME}"
-
-                    # We also save the datamodule config used to create the samples to the output 
-                    # directory and to wandb
-                    shutil.copyfile(
-                        f"{config.datamodule.sample_dir}/{DATAMODULE_CONFIG_NAME}", 
-                        f"{save_dir}/{DATAMODULE_CONFIG_NAME}"
-                    )
-                    wandb_logger.experiment.save(
-                        f"{save_dir}/{DATAMODULE_CONFIG_NAME}", 
-                        base_path=save_dir,
-                    )
-                else:
-                    # If we are streaming batches the data config is defined and we don't need to
-                    # save the datamodule config separately
-                    data_config = config.datamodule.configuration
-
                 # Save the data config to the output directory and to wandb
+                data_config = config.datamodule.configuration
                 shutil.copyfile(data_config, f"{save_dir}/{DATA_CONFIG_NAME}")
                 wandb_logger.experiment.save(f"{save_dir}/{DATA_CONFIG_NAME}", base_path=save_dir)
 

{pvnet-5.0.26.dist-info → pvnet-5.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.0.26
+Version: 5.1.1
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: >=3.11
@@ -142,12 +142,6 @@ pip install -e <PATH-TO-ocf-data-sampler-REPO>
 If you install the local version of `ocf-data-sampler` that is more recent than the version 
 specified in `PVNet` it is not guarenteed to function properly with this library.
 
-## Streaming samples (no pre-save)
-
-PVNet now trains and validates directly from **streamed_samples** (i.e. no pre-saving to disk).
-
-Make sure you have copied example configs (as already stated above):
-cp -r configs.example configs
 
 ### Set up and config example for streaming
 

{pvnet-5.0.26.dist-info → pvnet-5.1.1.dist-info}/RECORD

@@ -1,11 +1,8 @@
 pvnet/__init__.py,sha256=TAZm88TJ5ieL1XjEyRg1LciIGuSScEucdAruQLfM92I,25
-pvnet/load_model.py,sha256=LzN06O3oXzqhj1Dh_VlschDTxOq_Eea0OWDxrrboSKw,3726
+pvnet/datamodule.py,sha256=1qWYoRaxIvewnQJZ3GYz3zveQ7BT92XAT9vHqpLci0I,6356
+pvnet/load_model.py,sha256=P1QODX_mJRnKZ_kIll9BlOjK_A1W4YM3QG-mZd-2Mcc,3852
 pvnet/optimizers.py,sha256=1N4b-Xd6QiIrcUU8cbU326bbFC0BvMNIV8VYWtGILJc,6548
 pvnet/utils.py,sha256=iHG0wlN_cITKXpR16w54fs2R68wkX7vy3a_6f4SuanY,5414
-pvnet/data/__init__.py,sha256=FFD2tkLwEw9YiAVDam3tmaXNWMKiKVMHcnIz7zXCtrg,191
-pvnet/data/base_datamodule.py,sha256=Ibz0RoSr15HT6tMCs6ftXTpMa-NOKAmEd5ky55MqEK0,8615
-pvnet/data/site_datamodule.py,sha256=-KGxirGCBXVwcCREsjFkF7JDfa6NICv8bBDV6EILF_Q,962
-pvnet/data/uk_regional_datamodule.py,sha256=KA2_7DYuSggmD5b-XiXshXq8xmu36BjtFmy_pS7e4QE,1017
 pvnet/models/__init__.py,sha256=owzZ9xkD0DRTT51mT2Dx_p96oJjwDz57xo_MaMIEosk,145
 pvnet/models/base_model.py,sha256=CnQaaf2kAdOcXqo1319nWa120mHfLQiwOQ639m4OzPk,16182
 pvnet/models/ensemble.py,sha256=1mFUEsl33kWcLL5d7zfDm9ypWxgAxBHgBiJLt0vwTeg,2363
@@ -22,11 +19,11 @@ pvnet/models/late_fusion/site_encoders/__init__.py,sha256=QoUiiWWFf12vEpdkw0gO4T
 pvnet/models/late_fusion/site_encoders/basic_blocks.py,sha256=iEB_N7ZL5HMQ1hZM6H32A71GCwP7YbErUx0oQF21PQM,1042
 pvnet/models/late_fusion/site_encoders/encoders.py,sha256=k4z690cfcP6J4pm2KtDujHN-W3uOl7QY0WvBIu1tM8c,11703
 pvnet/training/__init__.py,sha256=FKxmPZ59Vuj5_mXomN4saJ3En5M-aDMxSs6OttTQOcg,49
-pvnet/training/lightning_module.py,sha256=RFp0kOmRW0cYU__i7SRCJUIvlp57s5cZ_H3hy2JnBxE,12954
+pvnet/training/lightning_module.py,sha256=ByWv4m1QSUUYq2DNCqWCWqEBcXF5gEQ1g__8J6lOseo,12944
 pvnet/training/plots.py,sha256=4xID7TBA4IazaARaCN5AoG5fFPJF1wIprn0y6I0C31c,2469
-pvnet/training/train.py,sha256=1tDA34ianCRfilS0yEeIpR9nsQWaJiiZfTD2qRUmgEc,5214
-pvnet-5.0.26.dist-info/licenses/LICENSE,sha256=tKUnlSmcLBWMJWkHx3UjZGdrjs9LidGwLo0jsBUBAwU,1077
-pvnet-5.0.26.dist-info/METADATA,sha256=OcYyfKyx5U6tWv2Tq_L9yvV1ilsih4jtBwjsM0weN2Y,16707
-pvnet-5.0.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pvnet-5.0.26.dist-info/top_level.txt,sha256=4mg6WjeW05SR7pg3-Q4JRE2yAoutHYpspOsiUzYVNv0,6
-pvnet-5.0.26.dist-info/RECORD,,
+pvnet/training/train.py,sha256=Sry2wYgggUmtIB-k_umFts7xMr2roEL76NCu9ySbLUY,4107
+pvnet-5.1.1.dist-info/licenses/LICENSE,sha256=tKUnlSmcLBWMJWkHx3UjZGdrjs9LidGwLo0jsBUBAwU,1077
+pvnet-5.1.1.dist-info/METADATA,sha256=52p2kuR_y7YhoLdLDio_P2ljaaIu14vcQYth-jY00_c,16474
+pvnet-5.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pvnet-5.1.1.dist-info/top_level.txt,sha256=4mg6WjeW05SR7pg3-Q4JRE2yAoutHYpspOsiUzYVNv0,6
+pvnet-5.1.1.dist-info/RECORD,,

pvnet/data/__init__.py

@@ -1,3 +0,0 @@
-"""Data parts"""
-from .site_datamodule import SitePresavedDataModule, SiteStreamedDataModule
-from .uk_regional_datamodule import UKRegionalPresavedDataModule, UKRegionalStreamedDataModule

pvnet/data/site_datamodule.py

@@ -1,29 +0,0 @@
-""" Data module for pytorch lightning """
-
-from ocf_data_sampler.torch_datasets.datasets.site import SitesDataset
-from ocf_data_sampler.torch_datasets.sample.site import SiteSample
-from torch.utils.data import Dataset
-
-from pvnet.data.base_datamodule import (
-    BasePresavedDataModule,
-    BaseStreamedDataModule,
-    PresavedSamplesDataset,
-)
-
-
-class SitePresavedDataModule(BasePresavedDataModule):
-    """Datamodule for loading pre-saved samples."""
-
-    def _get_premade_samples_dataset(self, subdir: str) -> Dataset:
-        return PresavedSamplesDataset(f"{self.sample_dir}/{subdir}", SiteSample)
-
-
-class SiteStreamedDataModule(BaseStreamedDataModule):
-    """Datamodule which streams samples using sampler for ocf-data-sampler."""
-
-    def _get_streamed_samples_dataset(
-        self,
-        start_time: str | None,
-        end_time: str | None
-    ) -> Dataset:
-        return SitesDataset(self.configuration, start_time=start_time, end_time=end_time)

pvnet/data/uk_regional_datamodule.py

@@ -1,29 +0,0 @@
-""" Data module for pytorch lightning """
-
-from ocf_data_sampler.torch_datasets.datasets.pvnet_uk import PVNetUKRegionalDataset
-from ocf_data_sampler.torch_datasets.sample.uk_regional import UKRegionalSample
-from torch.utils.data import Dataset
-
-from pvnet.data.base_datamodule import (
-    BasePresavedDataModule,
-    BaseStreamedDataModule,
-    PresavedSamplesDataset,
-)
-
-
-class UKRegionalPresavedDataModule(BasePresavedDataModule):
-    """Datamodule for loading pre-saved samples."""
-
-    def _get_premade_samples_dataset(self, subdir: str) -> Dataset:
-        return PresavedSamplesDataset(f"{self.sample_dir}/{subdir}", UKRegionalSample)
-
-
-class UKRegionalStreamedDataModule(BaseStreamedDataModule):
-    """Datamodule which streams samples using sampler for ocf-data-sampler."""
-
-    def _get_streamed_samples_dataset(
-        self,
-        start_time: str | None,
-        end_time: str | None
-    ) -> Dataset:
-        return PVNetUKRegionalDataset(self.configuration, start_time=start_time, end_time=end_time)