PVNet 5.0.14__tar.gz → 5.0.16__tar.gz

{pvnet-5.0.14 → pvnet-5.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.0.14
+Version: 5.0.16
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: >=3.11
@@ -14,7 +14,6 @@ Requires-Dist: xarray
 Requires-Dist: h5netcdf
 Requires-Dist: torch>=2.0.0
 Requires-Dist: lightning
-Requires-Dist: torchvision
 Requires-Dist: typer
 Requires-Dist: sqlalchemy
 Requires-Dist: fsspec[s3]

{pvnet-5.0.14 → pvnet-5.0.16}/PVNet.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.0.14
+Version: 5.0.16
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: >=3.11
@@ -14,7 +14,6 @@ Requires-Dist: xarray
 Requires-Dist: h5netcdf
 Requires-Dist: torch>=2.0.0
 Requires-Dist: lightning
-Requires-Dist: torchvision
 Requires-Dist: typer
 Requires-Dist: sqlalchemy
 Requires-Dist: fsspec[s3]

{pvnet-5.0.14 → pvnet-5.0.16}/PVNet.egg-info/requires.txt

@@ -6,7 +6,6 @@ xarray
 h5netcdf
 torch>=2.0.0
 lightning
-torchvision
 typer
 sqlalchemy
 fsspec[s3]

{pvnet-5.0.14 → pvnet-5.0.16}/pvnet/models/base_model.py

@@ -1,5 +1,4 @@
 """Base model for all PVNet submodels"""
-import copy
 import logging
 import os
 import shutil
@@ -12,9 +11,7 @@ import torch
 import yaml
 from huggingface_hub import ModelCard, ModelCardData, snapshot_download
 from huggingface_hub.hf_api import HfApi
-from ocf_data_sampler.numpy_sample.common_types import TensorBatch
 from safetensors.torch import load_file, save_file
-from torchvision.transforms.functional import center_crop
 
 from pvnet.utils import (
     DATA_CONFIG_NAME,
@@ -437,69 +434,6 @@ class BaseModel(torch.nn.Module, HuggingfaceMixin):
         else:
             self.num_output_features = self.forecast_len
 
-    def _adapt_batch(self, batch: TensorBatch) -> TensorBatch:
-        """Slice batches into appropriate shapes for model.
-
-        Returns a new batch dictionary with adapted data, leaving the original batch unchanged.
-        We make some specific assumptions about the original batch and the derived sliced batch:
-        - We are only limiting the future projections. I.e. we are never shrinking the batch from
-          the left hand side of the time axis, only slicing it from the right
-        - We are only shrinking the spatial crop of the satellite and NWP data
-
-        """
-        # Create a copy of the batch to avoid modifying the original
-        new_batch = {key: copy.deepcopy(value) for key, value in batch.items()}
-
-        if "gsp" in new_batch.keys():
-            # Slice off the end of the GSP data
-            gsp_len = self.forecast_len + self.history_len + 1
-            new_batch["gsp"] = new_batch["gsp"][:, :gsp_len]
-            new_batch["gsp_time_utc"] = new_batch["gsp_time_utc"][:, :gsp_len]
-
-        if "site" in new_batch.keys():
-            # Slice off the end of the site data
-            site_len = self.forecast_len + self.history_len + 1
-            new_batch["site"] = new_batch["site"][:, :site_len]
-
-            # Slice all site related datetime coordinates and features
-            site_time_keys = [
-                "site_time_utc",
-                "site_date_sin",
-                "site_date_cos",
-                "site_time_sin",
-                "site_time_cos",
-            ]
-
-            for key in site_time_keys:
-                if key in new_batch.keys():
-                    new_batch[key] = new_batch[key][:, :site_len]
-
-        if self.include_sat:
-            # Slice off the end of the satellite data and spatially crop
-            # Shape: batch_size, seq_length, channel, height, width
-            new_batch["satellite_actual"] = center_crop(
-                new_batch["satellite_actual"][:, : self.sat_sequence_len],
-                output_size=self.sat_encoder.image_size_pixels,
-            )
-
-        if self.include_nwp:
-            # Slice off the end of the NWP data and spatially crop
-            for nwp_source in self.nwp_encoders_dict:
-                # shape: batch_size, seq_len, n_chans, height, width
-                new_batch["nwp"][nwp_source]["nwp"] = center_crop(
-                    new_batch["nwp"][nwp_source]["nwp"],
-                    output_size=self.nwp_encoders_dict[nwp_source].image_size_pixels,
-                )[:, : self.nwp_encoders_dict[nwp_source].sequence_length]
-
-        if self.include_sun:
-            sun_len = self.forecast_len + self.history_len + 1
-            # Slice off end of solar coords
-            for s in ["solar_azimuth", "solar_elevation"]:
-                if s in new_batch.keys():
-                    new_batch[s] = new_batch[s][:, :sun_len]
-
-        return new_batch
-
     def _quantiles_to_prediction(self, y_quantiles: torch.Tensor) -> torch.Tensor:
         """
         Convert network prediction into a point prediction.
@@ -517,4 +451,4 @@ class BaseModel(torch.nn.Module, HuggingfaceMixin):
         """
         # y_quantiles Shape: batch_size, seq_length, num_quantiles
         idx = self.output_quantiles.index(0.5)
-        return y_quantiles[..., idx]
+        return y_quantiles[..., idx]

{pvnet-5.0.14 → pvnet-5.0.16}/pvnet/models/late_fusion/encoders/basic_blocks.py

@@ -31,6 +31,7 @@ class AbstractNWPSatelliteEncoder(nn.Module, metaclass=ABCMeta):
         self.out_features = out_features
         self.image_size_pixels = image_size_pixels
         self.sequence_length = sequence_length
+        self.in_channels = in_channels
 
     @abstractmethod
     def forward(self):

{pvnet-5.0.14 → pvnet-5.0.16}/pvnet/models/late_fusion/late_fusion.py

@@ -61,7 +61,6 @@ class LateFusionModel(BaseModel):
         nwp_interval_minutes: DictConfig | None = None,
         pv_interval_minutes: int = 5,
         sat_interval_minutes: int = 5,
-        adapt_batches: bool = False,
     ):
         """Neural network which combines information from different sources.
 
@@ -110,9 +109,6 @@ class LateFusionModel(BaseModel):
                 data for each source
             pv_interval_minutes: The interval between each sample of the PV data
             sat_interval_minutes: The interval between each sample of the satellite data
-            adapt_batches: If set to true, we attempt to slice the batches to the expected shape for
-                the model to use. This allows us to overprepare batches and slice from them for the
-                data we need for a model run.
         """
         super().__init__(
             history_minutes=history_minutes,
@@ -134,7 +130,6 @@ class LateFusionModel(BaseModel):
         self.add_image_embedding_channel = add_image_embedding_channel
         self.interval_minutes = interval_minutes
         self.min_sat_delay_minutes = min_sat_delay_minutes
-        self.adapt_batches = adapt_batches
 
         if self.location_id_mapping is None:
             logger.warning(
@@ -272,9 +267,6 @@ class LateFusionModel(BaseModel):
     def forward(self, x: TensorBatch) -> torch.Tensor:
         """Run model forward"""
 
-        if self.adapt_batches:
-            x = self._adapt_batch(x)
-
         if self.use_id_embedding:
             # eg: x['gsp_id'] = [1] with location_id_mapping = {1:0}, would give [0]
             id = torch.tensor(

{pvnet-5.0.14 → pvnet-5.0.16}/pvnet/training/lightning_module.py

@@ -15,6 +15,7 @@ from pvnet.data.base_datamodule import collate_fn
 from pvnet.models.base_model import BaseModel
 from pvnet.optimizers import AbstractOptimizer
 from pvnet.training.plots import plot_sample_forecasts, wandb_line_plot
+from pvnet.utils import validate_batch_against_config
 
 
 class PVNetLightningModule(pl.LightningModule):
@@ -42,9 +43,6 @@ class PVNetLightningModule(pl.LightningModule):
         # This setting is only used when lr is tuned with callback
         self.lr = None
 
-        # Set up store for all all validation results so we can log these
-        self.save_all_validation_results = save_all_validation_results
-
     def transfer_batch_to_device(
         self, 
         batch: TensorBatch, 
@@ -105,10 +103,6 @@ class PVNetLightningModule(pl.LightningModule):
         """Run training step"""
         y_hat = self.model(batch)
 
-        # Batch may be adapted in the model forward method, would need adapting here too
-        if self.model.adapt_batches:
-            batch = self.model._adapt_batch(batch)
-
         y = batch[self.model._target_key][:, -self.model.forecast_len :]
 
         losses = self._calculate_common_losses(y, y_hat)
@@ -193,7 +187,7 @@ class PVNetLightningModule(pl.LightningModule):
         self._val_horizon_maes: list[np.array] = []
         if self.current_epoch==0:
             self._val_persistence_horizon_maes: list[np.array] = []
-        
+
         # Plot some sample forecasts
         val_dataset = self.trainer.val_dataloaders.dataset
 
@@ -209,13 +203,17 @@ class PVNetLightningModule(pl.LightningModule):
 
             batch = collate_fn([val_dataset[i] for i in idxs])
             batch = self.transfer_batch_to_device(batch, self.device, dataloader_idx=0)
+
+            # Batch validation check only during sanity check phase - use first batch
+            if self.trainer.sanity_checking and plot_num == 0:
+                validate_batch_against_config(
+                    batch=batch,
+                    model=self.model
+                )
+            
             with torch.no_grad():
                 y_hat = self.model(batch)
             
-            # Batch may be adapted in the model forward method, would need adapting here too
-            if self.model.adapt_batches:
-                batch = self.model._adapt_batch(batch)
-            
             fig = plot_sample_forecasts(
                 batch,
                 y_hat,
@@ -235,9 +233,6 @@ class PVNetLightningModule(pl.LightningModule):
         """Run validation step"""
 
         y_hat = self.model(batch)
-        # Batch may be adapted in the model forward method, would need adapting here too
-        if self.model.adapt_batches:
-            batch = self.model._adapt_batch(batch)
 
         # Internally store the val predictions
         self._store_val_predictions(batch, y_hat)
@@ -317,7 +312,7 @@ class PVNetLightningModule(pl.LightningModule):
             self.log_dict(extreme_error_metrics, on_step=False, on_epoch=True)
 
             # Optionally save all validation results - these are overridden each epoch
-            if self.save_all_validation_results:
+            if self.hparams.save_all_validation_results:
                 # Add attributes
                 ds_val_results.attrs["epoch"] = self.current_epoch
 

{pvnet-5.0.14 → pvnet-5.0.16}/pvnet/training/train.py

@@ -26,7 +26,6 @@ from pvnet.utils import (
 log = logging.getLogger(__name__)
 
 
-
 def resolve_monitor_loss(output_quantiles: list | None) -> str:
     """Return the desired metric to monitor based on whether quantile regression is being used.
 

pvnet-5.0.16/pvnet/utils.py

@@ -0,0 +1,157 @@
+"""Utils"""
+import logging
+from typing import TYPE_CHECKING
+
+import rich.syntax
+import rich.tree
+from lightning.pytorch.utilities import rank_zero_only
+from omegaconf import DictConfig, OmegaConf
+
+if TYPE_CHECKING:
+    from pvnet.models.base_model import BaseModel
+
+logger = logging.getLogger(__name__)
+
+
+PYTORCH_WEIGHTS_NAME = "model_weights.safetensors"
+MODEL_CONFIG_NAME = "model_config.yaml"
+DATA_CONFIG_NAME = "data_config.yaml"
+DATAMODULE_CONFIG_NAME = "datamodule_config.yaml"
+FULL_CONFIG_NAME =  "full_experiment_config.yaml"
+MODEL_CARD_NAME = "README.md"
+
+
+
+def run_config_utilities(config: DictConfig) -> None:
+    """A couple of optional utilities.
+
+    Controlled by main config file:
+    - forcing debug friendly configuration
+
+    Modifies DictConfig in place.
+
+    Args:
+        config (DictConfig): Configuration composed by Hydra.
+    """
+
+    # Enable adding new keys to config
+    OmegaConf.set_struct(config, False)
+
+    # Force debugger friendly configuration if <config.trainer.fast_dev_run=True>
+    if config.trainer.get("fast_dev_run"):
+        logger.info("Forcing debugger friendly configuration! <config.trainer.fast_dev_run=True>")
+        # Debuggers don't like GPUs or multiprocessing
+        if config.trainer.get("gpus"):
+            config.trainer.gpus = 0
+        if config.datamodule.get("pin_memory"):
+            config.datamodule.pin_memory = False
+        if config.datamodule.get("num_workers"):
+            config.datamodule.num_workers = 0
+        if config.datamodule.get("prefetch_factor"):
+            config.datamodule.prefetch_factor = None
+
+    # Disable adding new keys to config
+    OmegaConf.set_struct(config, True)
+
+
+@rank_zero_only
+def print_config(
+    config: DictConfig,
+    fields: tuple[str] = (
+        "trainer",
+        "model",
+        "datamodule",
+        "callbacks",
+        "logger",
+        "seed",
+    ),
+    resolve: bool = True,
+) -> None:
+    """Prints content of DictConfig using Rich library and its tree structure.
+
+    Args:
+        config (DictConfig): Configuration composed by Hydra.
+        fields (Sequence[str], optional): Determines which main fields from config will
+        be printed and in what order.
+        resolve (bool, optional): Whether to resolve reference fields of DictConfig.
+    """
+
+    style = "dim"
+    tree = rich.tree.Tree("CONFIG", style=style, guide_style=style)
+
+    for field in fields:
+        branch = tree.add(field, style=style, guide_style=style)
+
+        config_section = config.get(field)
+
+        branch_content = str(config_section)
+        if isinstance(config_section, DictConfig):
+            branch_content = OmegaConf.to_yaml(config_section, resolve=resolve)
+
+        branch.add(rich.syntax.Syntax(branch_content, "yaml"))
+
+    rich.print(tree)
+
+
+def validate_batch_against_config(
+    batch: dict, 
+    model: "BaseModel",
+) -> None:
+    """Validates tensor shapes in batch against model configuration."""
+    logger.info("Performing batch shape validation against model config.")
+    
+    # NWP validation
+    if hasattr(model, 'nwp_encoders_dict'):
+        if "nwp" not in batch:
+            raise ValueError(
+                "Model configured with 'nwp_encoders_dict' but 'nwp' data missing from batch."
+            )
+            
+        for source, nwp_data in batch["nwp"].items():
+            if source in model.nwp_encoders_dict:
+
+                enc = model.nwp_encoders_dict[source]                
+                expected_channels = enc.in_channels
+                if model.add_image_embedding_channel:
+                    expected_channels -= 1
+
+                expected = (nwp_data["nwp"].shape[0], enc.sequence_length, 
+                           expected_channels, enc.image_size_pixels, enc.image_size_pixels)
+                if tuple(nwp_data["nwp"].shape) != expected:
+                    actual_shape = tuple(nwp_data['nwp'].shape)
+                    raise ValueError(
+                        f"NWP.{source} shape mismatch: expected {expected}, got {actual_shape}"
+                    )
+
+    # Satellite validation
+    if hasattr(model, 'sat_encoder'):
+        if "satellite_actual" not in batch:
+            raise ValueError(
+                "Model configured with 'sat_encoder' but 'satellite_actual' missing from batch."
+            )
+
+        enc = model.sat_encoder
+        expected_channels = enc.in_channels
+        if model.add_image_embedding_channel:
+            expected_channels -= 1
+
+        expected = (batch["satellite_actual"].shape[0], enc.sequence_length, expected_channels, 
+                enc.image_size_pixels, enc.image_size_pixels)
+        if tuple(batch["satellite_actual"].shape) != expected:
+            actual_shape = tuple(batch['satellite_actual'].shape)
+            raise ValueError(f"Satellite shape mismatch: expected {expected}, got {actual_shape}")
+
+    # GSP/Site validation
+    key = model._target_key
+    if key in batch:
+        total_minutes = model.history_minutes + model.forecast_minutes
+        interval = model.interval_minutes
+        expected_len = total_minutes // interval + 1
+        expected = (batch[key].shape[0], expected_len)
+        if tuple(batch[key].shape) != expected:
+            actual_shape = tuple(batch[key].shape)
+            raise ValueError(
+                f"{key.upper()} shape mismatch: expected {expected}, got {actual_shape}"
+            )
+
+    logger.info("Batch shape validation successful!")

{pvnet-5.0.14 → pvnet-5.0.16}/pyproject.toml

@@ -20,7 +20,6 @@ dependencies = [
     "h5netcdf",
     "torch>=2.0.0",
     "lightning",
-    "torchvision",
     "typer",
     "sqlalchemy",
     "fsspec[s3]",

{pvnet-5.0.14 → pvnet-5.0.16}/tests/test_end2end.py

@@ -1,7 +1,8 @@
 import lightning
-from pvnet.data import  UKRegionalStreamedDataModule
-from pvnet.training.lightning_module import PVNetLightningModule
+
+from pvnet.data import UKRegionalStreamedDataModule
 from pvnet.optimizers import EmbAdamWReduceLROnPlateau
+from pvnet.training.lightning_module import PVNetLightningModule
 
 
 def test_model_trainer_fit(session_tmp_path, uk_data_config_path, late_fusion_model):
@@ -15,7 +16,7 @@ def test_model_trainer_fit(session_tmp_path, uk_data_config_path, late_fusion_mo
         dataset_pickle_dir=f"{session_tmp_path}/dataset_pickles"
     )
 
-    ligtning_model = PVNetLightningModule(
+    lightning_model = PVNetLightningModule(
         model=late_fusion_model,
         optimizer=EmbAdamWReduceLROnPlateau(),
     )
@@ -29,4 +30,4 @@ def test_model_trainer_fit(session_tmp_path, uk_data_config_path, late_fusion_mo
         logger=False, 
         enable_checkpointing=False, 
     )
-    trainer.fit(model=ligtning_model, datamodule=datamodule)
+    trainer.fit(model=lightning_model, datamodule=datamodule)

pvnet-5.0.14/pvnet/utils.py

@@ -1,88 +0,0 @@
-"""Utils"""
-import logging
-
-import rich.syntax
-import rich.tree
-from lightning.pytorch.utilities import rank_zero_only
-from omegaconf import DictConfig, OmegaConf
-
-logger = logging.getLogger(__name__)
-
-
-PYTORCH_WEIGHTS_NAME = "model_weights.safetensors"
-MODEL_CONFIG_NAME = "model_config.yaml"
-DATA_CONFIG_NAME = "data_config.yaml"
-DATAMODULE_CONFIG_NAME = "datamodule_config.yaml"
-FULL_CONFIG_NAME =  "full_experiment_config.yaml"
-MODEL_CARD_NAME = "README.md"
-
-
-
-def run_config_utilities(config: DictConfig) -> None:
-    """A couple of optional utilities.
-
-    Controlled by main config file:
-    - forcing debug friendly configuration
-
-    Modifies DictConfig in place.
-
-    Args:
-        config (DictConfig): Configuration composed by Hydra.
-    """
-
-    # Enable adding new keys to config
-    OmegaConf.set_struct(config, False)
-
-    # Force debugger friendly configuration if <config.trainer.fast_dev_run=True>
-    if config.trainer.get("fast_dev_run"):
-        logger.info("Forcing debugger friendly configuration! <config.trainer.fast_dev_run=True>")
-        # Debuggers don't like GPUs or multiprocessing
-        if config.trainer.get("gpus"):
-            config.trainer.gpus = 0
-        if config.datamodule.get("pin_memory"):
-            config.datamodule.pin_memory = False
-        if config.datamodule.get("num_workers"):
-            config.datamodule.num_workers = 0
-        if config.datamodule.get("prefetch_factor"):
-            config.datamodule.prefetch_factor = None
-
-    # Disable adding new keys to config
-    OmegaConf.set_struct(config, True)
-
-
-@rank_zero_only
-def print_config(
-    config: DictConfig,
-    fields: tuple[str] = (
-        "trainer",
-        "model",
-        "datamodule",
-        "callbacks",
-        "logger",
-        "seed",
-    ),
-    resolve: bool = True,
-) -> None:
-    """Prints content of DictConfig using Rich library and its tree structure.
-
-    Args:
-        config (DictConfig): Configuration composed by Hydra.
-        fields (Sequence[str], optional): Determines which main fields from config will
-        be printed and in what order.
-        resolve (bool, optional): Whether to resolve reference fields of DictConfig.
-    """
-
-    style = "dim"
-    tree = rich.tree.Tree("CONFIG", style=style, guide_style=style)
-
-    for field in fields:
-        branch = tree.add(field, style=style, guide_style=style)
-
-        config_section = config.get(field)
-        branch_content = str(config_section)
-        if isinstance(config_section, DictConfig):
-            branch_content = OmegaConf.to_yaml(config_section, resolve=resolve)
-
-        branch.add(rich.syntax.Syntax(branch_content, "yaml"))
-
-    rich.print(tree)