PVNet 5.0.0__tar.gz → 5.0.2__tar.gz

{pvnet-5.0.0 → pvnet-5.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.0.0
+Version: 5.0.2
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: >=3.10

{pvnet-5.0.0 → pvnet-5.0.2}/PVNet.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.0.0
+Version: 5.0.2
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: >=3.10

{pvnet-5.0.0 → pvnet-5.0.2}/pvnet/data/base_datamodule.py

@@ -45,6 +45,7 @@ class BasePresavedDataModule(LightningDataModule):
         batch_size: int = 16,
         num_workers: int = 0,
         prefetch_factor: int | None = None,
+        persistent_workers: bool = False,
     ):
         """Base Datamodule for loading pre-saved samples
 
@@ -53,8 +54,9 @@ class BasePresavedDataModule(LightningDataModule):
             batch_size: Batch size.
             num_workers: Number of workers to use in multiprocess batch loading.
             prefetch_factor: Number of data will be prefetched at the end of each worker process.
-            train_period: Date range filter for train dataloader.
-            val_period: Date range filter for val dataloader.
+            persistent_workers: If True, the data loader will not shut down the worker processes 
+                after a dataset has been consumed once. This allows to maintain the workers Dataset 
+                instances alive.
         """
         super().__init__()
 
@@ -71,7 +73,7 @@ class BasePresavedDataModule(LightningDataModule):
             timeout=0,
             worker_init_fn=None,
             prefetch_factor=prefetch_factor,
-            persistent_workers=False,
+            persistent_workers=persistent_workers,
         )
 
     def _get_premade_samples_dataset(self, subdir: str) -> Dataset:
@@ -97,8 +99,11 @@ class BaseStreamedDataModule(LightningDataModule):
         batch_size: int = 16,
         num_workers: int = 0,
         prefetch_factor: int | None = None,
+        persistent_workers: bool = False,
         train_period: list[str | None] = [None, None],
         val_period: list[str | None] = [None, None],
+        seed: int | None = None,
+
     ):
         """Base Datamodule for streaming samples.
 
@@ -107,14 +112,19 @@ class BaseStreamedDataModule(LightningDataModule):
             batch_size: Batch size.
             num_workers: Number of workers to use in multiprocess batch loading.
             prefetch_factor: Number of data will be prefetched at the end of each worker process.
+            persistent_workers: If True, the data loader will not shut down the worker processes 
+                after a dataset has been consumed once. This allows to maintain the workers Dataset 
+                instances alive.
             train_period: Date range filter for train dataloader.
             val_period: Date range filter for val dataloader.
+            seed: Random seed used in shuffling datasets.
         """
         super().__init__()
 
         self.configuration = configuration
         self.train_period = train_period
         self.val_period = val_period
+        self.seed = seed
 
         self._common_dataloader_kwargs = dict(
             batch_size=batch_size,
@@ -126,7 +136,7 @@ class BaseStreamedDataModule(LightningDataModule):
             timeout=0,
             worker_init_fn=None,
             prefetch_factor=prefetch_factor,
-            persistent_workers=False,
+            persistent_workers=persistent_workers,
         )
 
     def setup(self, stage: str | None = None):
@@ -135,11 +145,16 @@ class BaseStreamedDataModule(LightningDataModule):
         # This logic runs only once at the start of training, therefore the val dataset is only
         # shuffled once
         if stage == "fit":
+
             # Prepare the train dataset
             self.train_dataset = self._get_streamed_samples_dataset(*self.train_period)
 
-            # Prepare and pre-shuffle the val dataset
+            # Prepare and pre-shuffle the val dataset and set seed for reproducibility
             val_dataset = self._get_streamed_samples_dataset(*self.val_period)
+
+            if self.seed is not None:
+                torch.manual_seed(self.seed)
+
             shuffled_indices = torch.randperm(len(val_dataset))
             self.val_dataset = Subset(val_dataset, shuffled_indices)
 

{pvnet-5.0.0 → pvnet-5.0.2}/pvnet/utils.py

@@ -43,6 +43,8 @@ def run_config_utilities(config: DictConfig) -> None:
             config.datamodule.pin_memory = False
         if config.datamodule.get("num_workers"):
             config.datamodule.num_workers = 0
+        if config.datamodule.get("prefetch_factor"):
+            config.datamodule.prefetch_factor = None
 
     # Disable adding new keys to config
     OmegaConf.set_struct(config, True)