PVNet 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

pvnet/data/base_datamodule.py

@@ -1,5 +1,6 @@
 """ Data module for pytorch lightning """
 
+import os
 from glob import glob
 
 import numpy as np
@@ -77,6 +78,7 @@ class BasePresavedDataModule(LightningDataModule):
             worker_init_fn=None,
             prefetch_factor=prefetch_factor,
             persistent_workers=persistent_workers,
+            multiprocessing_context="spawn" if num_workers>0 else None,
         )
 
     def _get_premade_samples_dataset(self, subdir: str) -> Dataset:
@@ -107,7 +109,7 @@ class BaseStreamedDataModule(LightningDataModule):
         train_period: list[str | None] = [None, None],
         val_period: list[str | None] = [None, None],
         seed: int | None = None,
-
+        dataset_pickle_dir: str | None = None,
     ):
         """Base Datamodule for streaming samples.
 
@@ -124,6 +126,8 @@ class BaseStreamedDataModule(LightningDataModule):
             train_period: Date range filter for train dataloader.
             val_period: Date range filter for val dataloader.
             seed: Random seed used in shuffling datasets.
+            dataset_pickle_dir: Directory in which the val and train set will be presaved as
+                pickle objects. Setting this speeds up instantiation of multiple workers a lot.
         """
         super().__init__()
 
@@ -131,6 +135,7 @@ class BaseStreamedDataModule(LightningDataModule):
         self.train_period = train_period
         self.val_period = val_period
         self.seed = seed
+        self.dataset_pickle_dir = dataset_pickle_dir
 
         self._common_dataloader_kwargs = dict(
             batch_size=batch_size,
@@ -143,6 +148,7 @@ class BaseStreamedDataModule(LightningDataModule):
             worker_init_fn=None,
             prefetch_factor=prefetch_factor,
             persistent_workers=persistent_workers,
+            multiprocessing_context="spawn" if num_workers>0 else None,
         )
 
     def setup(self, stage: str | None = None):
@@ -160,6 +166,33 @@ class BaseStreamedDataModule(LightningDataModule):
 
             shuffled_indices = np.random.default_rng(seed=self.seed).permutation(len(val_dataset))
             self.val_dataset = Subset(val_dataset, shuffled_indices)
+        
+            if self.dataset_pickle_dir is not None:
+                os.makedirs(self.dataset_pickle_dir, exist_ok=True)
+                train_dataset_path = f"{self.dataset_pickle_dir}/train_dataset.pkl"
+                val_dataset_path = f"{self.dataset_pickle_dir}/val_dataset.pkl"
+
+                # For safety, these pickled datasets cannot be overwritten.
+                # See: https://github.com/openclimatefix/pvnet/pull/445
+                for path in [train_dataset_path, val_dataset_path]:
+                    if os.path.exists(path):
+                        raise FileExistsError(
+                            f"The pickled dataset path '{path}' already exists. Make sure that "
+                            "this can be safely deleted (i.e. not currently being used by any "
+                            "training run) and delete it manually. Else change the "
+                            "`dataset_pickle_dir` to a different directory."
+                        )
+
+                self.train_dataset.presave_pickle(train_dataset_path)
+                self.train_dataset.presave_pickle(val_dataset_path)
+
+    def teardown(self, stage: str | None = None) -> None:
+        """Clean up the pickled datasets"""
+        if self.dataset_pickle_dir is not None:
+            for filename in ["val_dataset.pkl", "train_dataset.pkl"]:
+                filepath = f"{self.dataset_pickle_dir}/{filename}"
+                if os.path.exists(filepath):
+                    os.remove(filepath)
 
     def _get_streamed_samples_dataset(
         self,

pvnet/training/lightning_module.py

@@ -105,8 +105,9 @@ class PVNetLightningModule(pl.LightningModule):
         """Run training step"""
         y_hat = self.model(batch)
 
-        # Batch is adapted in the model forward method, but needs to be adapted here too
-        batch = self.model._adapt_batch(batch)
+        # Batch may be adapted in the model forward method, would need adapting here too
+        if self.model.adapt_batches:
+            batch = self.model._adapt_batch(batch)
 
         y = batch[self.model._target_key][:, -self.model.forecast_len :]
 
@@ -211,8 +212,9 @@ class PVNetLightningModule(pl.LightningModule):
             with torch.no_grad():
                 y_hat = self.model(batch)
             
-            # Batch is adapted in the model forward method, but needs to be adapted here too
-            batch = self.model._adapt_batch(batch)
+            # Batch may be adapted in the model forward method, would need adapting here too
+            if self.model.adapt_batches:
+                batch = self.model._adapt_batch(batch)
             
             fig = plot_sample_forecasts(
                 batch,
@@ -223,7 +225,9 @@ class PVNetLightningModule(pl.LightningModule):
 
             plot_name = f"val_forecast_samples/sample_set_{plot_num}"
 
-            self.logger.experiment.log({plot_name: wandb.Image(fig)})
+            # Disabled for testing or using no logger
+            if self.logger:
+                self.logger.experiment.log({plot_name: wandb.Image(fig)})
 
             plt.close(fig)
 
@@ -231,8 +235,9 @@ class PVNetLightningModule(pl.LightningModule):
         """Run validation step"""
 
         y_hat = self.model(batch)
-        # Batch is adapted in the model forward method, but needs to be adapted here too
-        batch = self.model._adapt_batch(batch)
+        # Batch may be adapted in the model forward method, would need adapting here too
+        if self.model.adapt_batches:
+            batch = self.model._adapt_batch(batch)
 
         # Internally store the val predictions
         self._store_val_predictions(batch, y_hat)

{pvnet-5.0.7.dist-info → pvnet-5.0.9.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.0.7
+Version: 5.0.9
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ocf-data-sampler>=0.2.32
+Requires-Dist: ocf-data-sampler>=0.5.20
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: matplotlib

{pvnet-5.0.7.dist-info → pvnet-5.0.9.dist-info}/RECORD

@@ -3,7 +3,7 @@ pvnet/load_model.py,sha256=LzN06O3oXzqhj1Dh_VlschDTxOq_Eea0OWDxrrboSKw,3726
 pvnet/optimizers.py,sha256=1N4b-Xd6QiIrcUU8cbU326bbFC0BvMNIV8VYWtGILJc,6548
 pvnet/utils.py,sha256=h4w9nmx6V_IAtiRp6VQ90TmQZGGTdMzU63WfGmL-pPs,2666
 pvnet/data/__init__.py,sha256=FFD2tkLwEw9YiAVDam3tmaXNWMKiKVMHcnIz7zXCtrg,191
-pvnet/data/base_datamodule.py,sha256=A5x6HsVWyXSZopNqyJM2qKxUzhlGiggs_K187leVaQ4,6678
+pvnet/data/base_datamodule.py,sha256=Ibz0RoSr15HT6tMCs6ftXTpMa-NOKAmEd5ky55MqEK0,8615
 pvnet/data/site_datamodule.py,sha256=-KGxirGCBXVwcCREsjFkF7JDfa6NICv8bBDV6EILF_Q,962
 pvnet/data/uk_regional_datamodule.py,sha256=KA2_7DYuSggmD5b-XiXshXq8xmu36BjtFmy_pS7e4QE,1017
 pvnet/models/__init__.py,sha256=owzZ9xkD0DRTT51mT2Dx_p96oJjwDz57xo_MaMIEosk,145
@@ -22,11 +22,11 @@ pvnet/models/late_fusion/site_encoders/__init__.py,sha256=QoUiiWWFf12vEpdkw0gO4T
 pvnet/models/late_fusion/site_encoders/basic_blocks.py,sha256=iEB_N7ZL5HMQ1hZM6H32A71GCwP7YbErUx0oQF21PQM,1042
 pvnet/models/late_fusion/site_encoders/encoders.py,sha256=k4z690cfcP6J4pm2KtDujHN-W3uOl7QY0WvBIu1tM8c,11703
 pvnet/training/__init__.py,sha256=FKxmPZ59Vuj5_mXomN4saJ3En5M-aDMxSs6OttTQOcg,49
-pvnet/training/lightning_module.py,sha256=8UXOeL4mnsnHTZf1kHNbRxP8aeasTI-clASegSxuWzI,13029
+pvnet/training/lightning_module.py,sha256=GVqdi5ALFo9-_WRYeyMMj2qH_k4gPxQ2sG6FhL_wRFE,13242
 pvnet/training/plots.py,sha256=4xID7TBA4IazaARaCN5AoG5fFPJF1wIprn0y6I0C31c,2469
 pvnet/training/train.py,sha256=zj9JMi9C6W68vGsQUBapWkJ4aDzDuJFMv0IVjO73s1k,5215
-pvnet-5.0.7.dist-info/licenses/LICENSE,sha256=tKUnlSmcLBWMJWkHx3UjZGdrjs9LidGwLo0jsBUBAwU,1077
-pvnet-5.0.7.dist-info/METADATA,sha256=UyYMlgXH6SCla4n2ez-SVmPW6doYcYKeS2f9M5SqsRY,18043
-pvnet-5.0.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pvnet-5.0.7.dist-info/top_level.txt,sha256=4mg6WjeW05SR7pg3-Q4JRE2yAoutHYpspOsiUzYVNv0,6
-pvnet-5.0.7.dist-info/RECORD,,
+pvnet-5.0.9.dist-info/licenses/LICENSE,sha256=tKUnlSmcLBWMJWkHx3UjZGdrjs9LidGwLo0jsBUBAwU,1077
+pvnet-5.0.9.dist-info/METADATA,sha256=Z909YP0GU68upJxamWI1BlHI4vZjbSUAAlEnIt2V_fc,18043
+pvnet-5.0.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pvnet-5.0.9.dist-info/top_level.txt,sha256=4mg6WjeW05SR7pg3-Q4JRE2yAoutHYpspOsiUzYVNv0,6
+pvnet-5.0.9.dist-info/RECORD,,