PVNet 5.3.1__tar.gz → 5.3.6__tar.gz

{pvnet-5.3.1 → pvnet-5.3.6}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.3.1
+Version: 5.3.6
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: <3.14,>=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ocf-data-sampler>=0.6.0
+Requires-Dist: ocf-data-sampler>=1.0.9
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: matplotlib

{pvnet-5.3.1 → pvnet-5.3.6}/PVNet.egg-info/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 5.3.1
+Version: 5.3.6
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Requires-Python: <3.14,>=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ocf-data-sampler>=0.6.0
+Requires-Dist: ocf-data-sampler>=1.0.9
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: matplotlib

{pvnet-5.3.1 → pvnet-5.3.6}/PVNet.egg-info/requires.txt

@@ -1,4 +1,4 @@
-ocf-data-sampler>=0.6.0
+ocf-data-sampler>=1.0.9
 numpy
 pandas
 matplotlib

{pvnet-5.3.1 → pvnet-5.3.6}/pvnet/models/late_fusion/late_fusion.py

@@ -46,6 +46,7 @@ class LateFusionModel(BaseModel):
         include_generation_history: bool = False,
         include_sun: bool = True,
         include_time: bool = False,
+        t0_embedding_dim: int = 0,
         location_id_mapping: dict[Any, int] | None = None,
         embedding_dim: int = 16,
         forecast_minutes: int = 30,
@@ -85,6 +86,8 @@ class LateFusionModel(BaseModel):
             include_generation_history: Include generation yield data.
             include_sun: Include sun azimuth and altitude data.
             include_time: Include sine and cosine of dates and times.
+            t0_embedding_dim: Shape of the embedding of the init-time (t0) of the forecast. Not used
+                if set to 0.
             location_id_mapping: A dictionary mapping the location ID to an integer. ID embedding is
                 not used if this is not provided.
             embedding_dim: Number of embedding dimensions to use for location ID.
@@ -119,6 +122,7 @@ class LateFusionModel(BaseModel):
         self.include_pv = pv_encoder is not None
         self.include_sun = include_sun
         self.include_time = include_time
+        self.t0_embedding_dim = t0_embedding_dim
         self.location_id_mapping = location_id_mapping
         self.embedding_dim = embedding_dim
         self.add_image_embedding_channel = add_image_embedding_channel
@@ -246,6 +250,8 @@ class LateFusionModel(BaseModel):
             # Update num features
             fusion_input_features += 32
 
+        fusion_input_features += self.t0_embedding_dim
+
         if include_generation_history:
             # Update num features
             fusion_input_features += self.history_len + 1
@@ -321,6 +327,9 @@ class LateFusionModel(BaseModel):
             time = self.time_fc1(time)
             modes["time"] = time
 
+        if self.t0_embedding_dim>0:
+            modes["t0_embed"] = x["t0_embedding"]
+
         out = self.output_network(modes)
 
         if self.use_quantile_regression:

{pvnet-5.3.1 → pvnet-5.3.6}/pvnet/optimizers.py

@@ -65,7 +65,7 @@ class AbstractOptimizer(ABC):
     """
 
     @abstractmethod
-    def __call__(self):
+    def __call__(self, model: Module):
         """Abstract call"""
         pass
 
@@ -129,19 +129,18 @@ class EmbAdamWReduceLROnPlateau(AbstractOptimizer):
             {"params": decay, "weight_decay": self.weight_decay},
             {"params": no_decay, "weight_decay": 0.0},
         ]
+        monitor = "quantile_loss/val" if model.use_quantile_regression else "MAE/val"
         opt = torch.optim.AdamW(optim_groups, lr=self.lr, **self.opt_kwargs)
-
         sch = torch.optim.lr_scheduler.ReduceLROnPlateau(
             opt,
             factor=self.factor,
             patience=self.patience,
             threshold=self.threshold,
         )
-        sch = {
-            "scheduler": sch,
-            "monitor": "quantile_loss/val" if model.use_quantile_regression else "MAE/val",
+        return {
+            "optimizer": opt,
+            "lr_scheduler": {"scheduler": sch, "monitor": monitor},
         }
-        return [opt], [sch]
 
 
 class AdamWReduceLROnPlateau(AbstractOptimizer):
@@ -153,15 +152,13 @@ class AdamWReduceLROnPlateau(AbstractOptimizer):
         patience: int = 3, 
         factor: float = 0.5, 
         threshold: float = 2e-4, 
-        step_freq=None, 
         **opt_kwargs,
     ):
         """AdamW optimizer and reduce on plateau scheduler"""
-        self._lr = lr
+        self.lr = lr
         self.patience = patience
         self.factor = factor
         self.threshold = threshold
-        self.step_freq = step_freq
         self.opt_kwargs = opt_kwargs
 
     def _call_multi(self, model):
@@ -169,7 +166,7 @@ class AdamWReduceLROnPlateau(AbstractOptimizer):
 
         group_args = []
 
-        for key in self._lr.keys():
+        for key in self.lr.keys():
             if key == "default":
                 continue
 
@@ -178,43 +175,38 @@ class AdamWReduceLROnPlateau(AbstractOptimizer):
                 if param_name.startswith(key):
                     submodule_params += [remaining_params.pop(param_name)]
 
-            group_args += [{"params": submodule_params, "lr": self._lr[key]}]
+            group_args += [{"params": submodule_params, "lr": self.lr[key]}]
 
         remaining_params = [p for k, p in remaining_params.items()]
         group_args += [{"params": remaining_params}]
-
-        opt = torch.optim.AdamW(
-            group_args, 
-            lr=self._lr["default"] if model.lr is None else model.lr, 
-            **self.opt_kwargs,
+        monitor = "quantile_loss/val" if model.use_quantile_regression else "MAE/val"
+        opt = torch.optim.AdamW(group_args, lr=self.lr["default"], **self.opt_kwargs)
+        sch = torch.optim.lr_scheduler.ReduceLROnPlateau(
+            opt,
+            factor=self.factor,
+            patience=self.patience,
+            threshold=self.threshold,
         )
-        sch = {
-            "scheduler": torch.optim.lr_scheduler.ReduceLROnPlateau(
-                opt,
-                factor=self.factor,
-                patience=self.patience,
-                threshold=self.threshold,
-            ),
-            "monitor": "quantile_loss/val" if model.use_quantile_regression else "MAE/val",
+        return {
+            "optimizer": opt,
+            "lr_scheduler": {"scheduler": sch, "monitor": monitor},
         }
 
-        return [opt], [sch]
 
     def __call__(self, model):
         """Return optimizer"""
-        if not isinstance(self._lr, float):
+        if isinstance(self.lr, dict):
             return self._call_multi(model)
         else:
-            default_lr = self._lr if model.lr is None else model.lr
-            opt = torch.optim.AdamW(model.parameters(), lr=default_lr, **self.opt_kwargs)
+            monitor = "quantile_loss/val" if model.use_quantile_regression else "MAE/val"
+            opt = torch.optim.AdamW(model.parameters(), lr=self.lr, **self.opt_kwargs)
             sch = torch.optim.lr_scheduler.ReduceLROnPlateau(
                 opt,
                 factor=self.factor,
                 patience=self.patience,
                 threshold=self.threshold,
             )
-            sch = {
-                "scheduler": sch,
-                "monitor": "quantile_loss/val" if model.use_quantile_regression else "MAE/val",
+            return {
+                "optimizer": opt,
+                "lr_scheduler": {"scheduler": sch, "monitor": monitor},
             }
-            return [opt], [sch]

{pvnet-5.3.1 → pvnet-5.3.6}/pvnet/training/lightning_module.py

@@ -109,7 +109,7 @@ class PVNetLightningModule(pl.LightningModule):
         losses = self._calculate_common_losses(y, y_hat)
         losses = {f"{k}/train": v for k, v in losses.items()}
 
-        self.log_dict(losses, on_step=True, on_epoch=True)
+        self.log_dict(losses, on_step=True, on_epoch=True, batch_size=y.size(0))
 
         if self.model.use_quantile_regression:
             opt_target = losses["quantile_loss/train"]
@@ -256,11 +256,28 @@ class PVNetLightningModule(pl.LightningModule):
 
         # Calculate the persistance losses - we only need to do this once per training run
         # not every epoch
-        if self.current_epoch == 0:
+        if self.current_epoch==0:
+            # Need to find last valid value before forecast
+            target_data = batch["generation"]
+            history_data = target_data[:, :-(self.model.forecast_len)]
+            
+            # Find where values aren't dropped
+            valid_mask = history_data >= 0
+            
+            # Last valid value index for each sample
+            flipped_mask = valid_mask.float().flip(dims=[1])
+            last_valid_indices_flipped = torch.argmax(flipped_mask, dim=1)
+            last_valid_indices = history_data.shape[1] - 1 - last_valid_indices_flipped
+            
+            # Grab those last valid values
+            batch_indices = torch.arange(
+                history_data.shape[0], 
+                device=history_data.device
+            )
+            last_valid_values = history_data[batch_indices, last_valid_indices]
+
             y_persist = (
-                batch["generation"][:, -(self.model.forecast_len + 1)]
-                .unsqueeze(1)
-                .expand(-1, self.model.forecast_len)
+                last_valid_values.unsqueeze(1).expand(-1, self.model.forecast_len)
             )
             mae_step_persist, mse_step_persist = self._calculate_step_metrics(y, y_persist)
             self._val_persistence_horizon_maes.append(mae_step_persist)
@@ -272,7 +289,7 @@ class PVNetLightningModule(pl.LightningModule):
             )
 
         # Log the metrics
-        self.log_dict(losses, on_step=False, on_epoch=True)
+        self.log_dict(losses, on_step=False, on_epoch=True, batch_size=y.size(0))
 
     def on_validation_epoch_end(self) -> None:
         """Run on epoch end"""

{pvnet-5.3.1 → pvnet-5.3.6}/pvnet/utils.py

@@ -101,66 +101,64 @@ def validate_batch_against_config(
     logger.info("Performing batch shape validation against model config.")
 
     # NWP validation
-    if hasattr(model, "nwp_encoders_dict"):
+    if model.include_nwp:
         if "nwp" not in batch:
-            raise ValueError(
-                "Model configured with 'nwp_encoders_dict' but 'nwp' data missing from batch."
-            )
+            raise ValueError("Model uses NWP data but 'nwp' missing from batch.")
 
-        for source, nwp_data in batch["nwp"].items():
-            if source in model.nwp_encoders_dict:
-                enc = model.nwp_encoders_dict[source]
-                expected_channels = enc.in_channels
-                if model.add_image_embedding_channel:
-                    expected_channels -= 1
-
-                expected = (
-                    nwp_data["nwp"].shape[0],
-                    enc.sequence_length,
-                    expected_channels,
-                    enc.image_size_pixels,
-                    enc.image_size_pixels,
+        for source in model.nwp_encoders_dict:
+            if source not in batch["nwp"]:
+                raise ValueError(
+                    f"Model uses NWP source '{source}' but it is missing from batch['nwp']."
+                )
+
+            enc = model.nwp_encoders_dict[source]
+            expected_channels = enc.in_channels - int(model.add_image_embedding_channel)
+            
+            expected_shape = (
+                batch["nwp"][source]["nwp"].shape[0],
+                enc.sequence_length,
+                expected_channels,
+                enc.image_size_pixels,
+                enc.image_size_pixels,
+            )
+            actual_shape = tuple(batch["nwp"][source]["nwp"].shape)
+            if actual_shape != expected_shape:
+                raise ValueError(
+                    f"NWP.{source} shape mismatch: expected {expected_shape}, got {actual_shape}"
                 )
-                if tuple(nwp_data["nwp"].shape) != expected:
-                    actual_shape = tuple(nwp_data["nwp"].shape)
-                    raise ValueError(
-                        f"NWP.{source} shape mismatch: expected {expected}, got {actual_shape}"
-                    )
 
     # Satellite validation
-    if hasattr(model, "sat_encoder"):
+    if model.include_sat:
         if "satellite_actual" not in batch:
             raise ValueError(
-                "Model configured with 'sat_encoder' but 'satellite_actual' missing from batch."
+                "Model uses satellite data but 'satellite_actual' missing from batch."
             )
 
         enc = model.sat_encoder
-        expected_channels = enc.in_channels
-        if model.add_image_embedding_channel:
-            expected_channels -= 1
+        expected_channels = enc.in_channels - int(model.add_image_embedding_channel)
 
-        expected = (
+        expected_shape = (
             batch["satellite_actual"].shape[0],
             enc.sequence_length,
             expected_channels,
             enc.image_size_pixels,
             enc.image_size_pixels,
         )
-        if tuple(batch["satellite_actual"].shape) != expected:
-            actual_shape = tuple(batch["satellite_actual"].shape)
-            raise ValueError(f"Satellite shape mismatch: expected {expected}, got {actual_shape}")
+        actual_shape = tuple(batch["satellite_actual"].shape)
+        if actual_shape != expected_shape:
+            raise ValueError(
+                f"Satellite shape mismatch: expected {expected_shape}, got {actual_shape}"
+            )
 
-    # generation validation
     key = "generation"
     if key in batch:
         total_minutes = model.history_minutes + model.forecast_minutes
-        interval = model.interval_minutes
-        expected_len = total_minutes // interval + 1
-        expected = (batch[key].shape[0], expected_len)
-        if tuple(batch[key].shape) != expected:
-            actual_shape = tuple(batch[key].shape)
+        expected_len = total_minutes // model.interval_minutes + 1
+        expected_shape = (batch[key].shape[0], expected_len)
+        actual_shape = tuple(batch[key].shape)
+        if actual_shape != expected_shape:
             raise ValueError(
-                f"{key.upper()} shape mismatch: expected {expected}, got {actual_shape}"
+                f"Generation data shape mismatch: expected {expected_shape}, got {actual_shape}"
             )
 
     logger.info("Batch shape validation successful!")

{pvnet-5.3.1 → pvnet-5.3.6}/pyproject.toml

@@ -12,7 +12,7 @@ readme = {file="README.md", content-type="text/markdown"}
 requires-python = ">=3.11,<3.14"
 
 dependencies = [
-    "ocf-data-sampler>=0.6.0",
+    "ocf-data-sampler>=1.0.9",
     "numpy",
     "pandas",
     "matplotlib",