PVNet 4.1.6__tar.gz → 4.1.8__tar.gz

{pvnet-4.1.6 → pvnet-4.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 4.1.6
+Version: 4.1.8
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Description-Content-Type: text/markdown

{pvnet-4.1.6 → pvnet-4.1.8}/PVNet.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: PVNet
-Version: 4.1.6
+Version: 4.1.8
 Summary: PVNet
 Author-email: Peter Dudfield <info@openclimatefix.org>
 Description-Content-Type: text/markdown

pvnet-4.1.8/pvnet/__init__.py

@@ -0,0 +1,2 @@
+"""PVNet"""
+__version__ = "4.1.8"

{pvnet-4.1.6 → pvnet-4.1.8}/pvnet/models/base_model.py

@@ -107,13 +107,12 @@ def minimize_data_config(input_path, output_path, model):
                     nwp_config["image_size_pixels_height"] = nwp_pixel_size
                     nwp_config["image_size_pixels_width"] = nwp_pixel_size
 
-                    # Replace the forecast minutes
-                    nwp_config["forecast_minutes"] = (
-                        model.nwp_encoders_dict[nwp_source].sequence_length
-                        - nwp_config["interval_start_minutes"]
-                        / nwp_config["time_resolution_minutes"]
-                        - 1
-                    ) * nwp_config["time_resolution_minutes"]
+                    # Replace the interval_end_minutes minutes
+                    nwp_config["interval_end_minutes"] = (
+                        nwp_config["interval_start_minutes"] +
+                        (model.nwp_encoders_dict[nwp_source].sequence_length - 1)
+                        * nwp_config["time_resolution_minutes"]
+                    )
 
     if "satellite" in config["input_data"]:
         if not model.include_sat:
@@ -123,11 +122,15 @@ def minimize_data_config(input_path, output_path, model):
 
             # Replace the image size
             sat_pixel_size = model.sat_encoder.image_size_pixels
-            sat_config["satellite_image_size_pixels_height"] = sat_pixel_size
-            sat_config["satellite_image_size_pixels_width"] = sat_pixel_size
-
-            # Replace the satellite delay
-            sat_config["live_delay_minutes"] = model.min_sat_delay_minutes
+            sat_config["image_size_pixels_height"] = sat_pixel_size
+            sat_config["image_size_pixels_width"] = sat_pixel_size
+
+            # Replace the interval_end_minutes minutes
+            sat_config["interval_end_minutes"] = (
+                sat_config["interval_start_minutes"] +
+                (model.sat_encoder.sequence_length - 1)
+                * sat_config["time_resolution_minutes"]
+            )
 
     if "pv" in config["input_data"]:
         if not model.include_pv:
@@ -137,7 +140,7 @@ def minimize_data_config(input_path, output_path, model):
         gsp_config = config["input_data"]["gsp"]
 
         # Replace the forecast minutes
-        gsp_config["forecast_minutes"] = model.forecast_minutes
+        gsp_config["interval_end_minutes"] = model.forecast_minutes
 
     with open(output_path, "w") as outfile:
         yaml.dump(config, outfile, default_flow_style=False)
@@ -318,7 +321,6 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
         wandb_repo: Optional[str] = None,
         wandb_ids: Optional[Union[list[str], str]] = None,
         card_template_path: Optional[Path] = None,
-        revision: str = "main",
         **kwargs,
     ) -> Optional[str]:
         """
@@ -340,8 +342,6 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
             wandb_ids: Identifier(s) of the model on wandb.
             card_template_path: Path to the HuggingFace model card template. Defaults to card in
                 PVNet library if set to None.
-            revision (`str`, *optional*, defaults to `"main"`):
-                The revision to push the model to. Only used if `push_to_hub=True`.
             kwargs:
                 Additional key word arguments passed along to the
                 [`~ModelHubMixin._from_pretrained`] method.
@@ -367,10 +367,9 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
             # Taylor the data config to the model being saved
             minimize_data_config(new_data_config_path, new_data_config_path, self)
 
-        card = self.create_hugging_face_model_card(repo_id,
-                                                  wandb_repo,
-                                                  wandb_ids,
-                                                  card_template_path)
+        card = self.create_hugging_face_model_card(
+            repo_id, wandb_repo, wandb_ids, card_template_path
+        )
 
         (save_directory / "README.md").write_text(str(card))
 
@@ -381,16 +380,29 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
                 repo_id=repo_id,
                 repo_type="model",
                 folder_path=save_directory,
-                revision=revision,
             )
 
+            # Print the most recent commit hash
+            c = api.list_repo_commits(repo_id=repo_id, repo_type="model")[0]
+
+            message = (
+                f"The latest commit is now: \n"
+                f"    date: {c.created_at} \n"
+                f"    commit hash: {c.commit_id}\n"
+                f"    by: {c.authors}\n"
+                f"    title: {c.title}\n"
+            )
+
+            print(message)
+
         return None
 
     @staticmethod
-    def create_hugging_face_model_card(repo_id: Optional[str] = None,
-                                      wandb_repo: Optional[str] = None,
-                                      wandb_ids: Optional[Union[list[str], str]] = None,
-                                      card_template_path: Optional[Path] = None
+    def create_hugging_face_model_card(
+        repo_id: Optional[str] = None,
+        wandb_repo: Optional[str] = None,
+        wandb_ids: Optional[Union[list[str], str]] = None,
+        card_template_path: Optional[Path] = None,
     ) -> ModelCard:
         """
         Creates Hugging Face model card
@@ -443,14 +455,13 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
         for package, version in packages_and_versions.items():
             package_versions_markdown += f" - {package}=={version}\n"
 
-        card = ModelCard.from_template(
+        return ModelCard.from_template(
             card_data,
             template_path=card_template_path,
             wandb_links=wandb_links,
             package_versions=package_versions_markdown
         )
 
-        return card
 
 class BaseModel(pl.LightningModule, PVNetModelHubMixin):
     """Abstract base class for PVNet submodels"""

{pvnet-4.1.6 → pvnet-4.1.8}/pvnet/models/multimodal/multimodal.py

@@ -1,7 +1,7 @@
 """The default composite model architecture for PVNet"""
 
 from collections import OrderedDict
-from typing import Optional
+from typing import Any, Optional
 
 import torch
 from omegaconf import DictConfig
@@ -47,6 +47,7 @@ class Model(MultimodalBaseModel):
         include_gsp_yield_history: bool = True,
         include_sun: bool = True,
         include_time: bool = False,
+        location_id_mapping: Optional[dict[Any, int]] = None,
         embedding_dim: Optional[int] = 16,
         forecast_minutes: int = 30,
         history_minutes: int = 60,
@@ -64,7 +65,6 @@ class Model(MultimodalBaseModel):
         pv_interval_minutes: int = 5,
         sat_interval_minutes: int = 5,
         sensor_interval_minutes: int = 30,
-        num_embeddings: Optional[int] = 318,
         timestep_intervals_to_plot: Optional[list[int]] = None,
         adapt_batches: Optional[bool] = False,
         forecast_minutes_ignore: Optional[int] = 0,
@@ -94,8 +94,9 @@ class Model(MultimodalBaseModel):
             include_gsp_yield_history: Include GSP yield data.
             include_sun: Include sun azimuth and altitude data.
             include_time: Include sine and cosine of dates and times.
-            embedding_dim: Number of embedding dimensions to use for GSP ID. Not included if set to
-                `None`.
+            location_id_mapping: A dictionary mapping the location ID to an integer. ID embedding is
+                not used if this is not provided.
+            embedding_dim: Number of embedding dimensions to use for GSP ID.
             forecast_minutes: The amount of minutes that should be forecasted.
             history_minutes: The default amount of historical minutes that are used.
             sat_history_minutes: Length of recent observations used for satellite inputs. Defaults
@@ -116,7 +117,6 @@ class Model(MultimodalBaseModel):
             pv_interval_minutes: The interval between each sample of the PV data
             sat_interval_minutes: The interval between each sample of the satellite data
             sensor_interval_minutes: The interval between each sample of the sensor data
-            num_embeddings: The number of dimensions to use for the image embedding
             timestep_intervals_to_plot: Intervals, in timesteps, to plot in
             addition to the full forecast
             sensor_encoder: Encoder for sensor data
@@ -136,12 +136,24 @@ class Model(MultimodalBaseModel):
         self.include_sun = include_sun
         self.include_time = include_time
         self.include_sensor = sensor_encoder is not None
+        self.location_id_mapping = location_id_mapping
         self.embedding_dim = embedding_dim
         self.add_image_embedding_channel = add_image_embedding_channel
         self.interval_minutes = interval_minutes
         self.min_sat_delay_minutes = min_sat_delay_minutes
         self.adapt_batches = adapt_batches
 
+        if location_id_mapping is None and add_image_embedding_channel:
+            raise ValueError(
+                "If `add_image_embedding_channel` is set to True, `location_id_mapping` must be "
+                "provided."
+            )
+
+        self.use_id_embedding = location_id_mapping is not None
+
+        if self.use_id_embedding:
+            num_embeddings = max(location_id_mapping.values())
+
         super().__init__(
             history_minutes=history_minutes,
             forecast_minutes=forecast_minutes,
@@ -246,7 +258,7 @@ class Model(MultimodalBaseModel):
             # Update num features
             fusion_input_features += self.sensor_encoder.out_features
 
-        if self.embedding_dim:
+        if self.use_id_embedding:
             self.embed = nn.Embedding(num_embeddings=num_embeddings, embedding_dim=embedding_dim)
 
             # Update num features
@@ -289,6 +301,14 @@ class Model(MultimodalBaseModel):
         if self.adapt_batches:
             x = self._adapt_batch(x)
 
+        if self.use_id_embedding:
+            # eg: x['gsp_id] = [1] with location_id_mapping = {1:0}, would give [0]
+            id = torch.tensor(
+                [self.location_id_mapping[i.item()] for i in x[f"{self._target_key}_id"]],
+                device=self.device,
+                dtype=torch.int64,
+            )
+
         modes = OrderedDict()
         # ******************* Satellite imagery *************************
         if self.include_sat:
@@ -297,7 +317,6 @@ class Model(MultimodalBaseModel):
             sat_data = torch.swapaxes(sat_data, 1, 2).float()  # switch time and channels
 
             if self.add_image_embedding_channel:
-                id = x[f"{self._target_key}_id"].int()
                 sat_data = self.sat_embed(sat_data, id)
             modes["sat"] = self.sat_encoder(sat_data)
 
@@ -313,7 +332,6 @@ class Model(MultimodalBaseModel):
                 nwp_data = torch.clip(nwp_data, min=-50, max=50)
 
                 if self.add_image_embedding_channel:
-                    id = x[f"{self._target_key}_id"].int()
                     nwp_data = self.nwp_embed_dict[nwp_source](nwp_data, id)
 
                 nwp_out = self.nwp_encoders_dict[nwp_source](nwp_data)
@@ -339,10 +357,8 @@ class Model(MultimodalBaseModel):
             modes["gsp"] = gsp_history
 
         # ********************** Embedding of GSP/Site ID ********************
-        if self.embedding_dim:
-            id = x[f"{self._target_key}_id"].int()
-            id_embedding = self.embed(id)
-            modes["id"] = id_embedding
+        if self.use_id_embedding:
+            modes["id"] = self.embed(id)
 
         if self.include_sun:
             # Use only new direct keys

{pvnet-4.1.6 → pvnet-4.1.8}/pvnet/models/multimodal/unimodal_teacher.py

@@ -2,7 +2,7 @@
 
 import glob
 from collections import OrderedDict
-from typing import Optional
+from typing import Any, Optional
 
 import hydra
 import torch
@@ -44,6 +44,7 @@ class Model(MultimodalBaseModel):
         output_quantiles: Optional[list[float]] = None,
         include_gsp_yield_history: bool = True,
         include_sun: bool = True,
+        location_id_mapping: Optional[dict[Any, int]] = None,
         embedding_dim: Optional[int] = 16,
         forecast_minutes: int = 30,
         history_minutes: int = 60,
@@ -74,8 +75,9 @@ class Model(MultimodalBaseModel):
                 None the output is a single value.
             include_gsp_yield_history: Include GSP yield data.
             include_sun: Include sun azimuth and altitude data.
-            embedding_dim: Number of embedding dimensions to use for GSP ID. Not included if set to
-                `None`.
+            location_id_mapping: A dictionary mapping the location ID to an integer. ID embedding is
+                not used if this is not provided.
+            embedding_dim: Number of embedding dimensions to use for GSP ID
             forecast_minutes: The amount of minutes that should be forecasted.
             history_minutes: The default amount of historical minutes that are used.
             optimizer: Optimizer factory function used for network.
@@ -93,6 +95,7 @@ class Model(MultimodalBaseModel):
 
         self.include_gsp_yield_history = include_gsp_yield_history
         self.include_sun = include_sun
+        self.location_id_mapping = location_id_mapping
         self.embedding_dim = embedding_dim
         self.enc_loss_frac = enc_loss_frac
         self.include_sat = False
@@ -100,6 +103,11 @@ class Model(MultimodalBaseModel):
         self.include_pv = False
         self.adapt_batches = adapt_batches
 
+        self.use_id_embedding = location_id_mapping is not None
+
+        if self.use_id_embedding:
+            num_embeddings = max(location_id_mapping.values())
+
         # This is set but modified later based on the teachers
         self.add_image_embedding_channel = False
 
@@ -167,7 +175,7 @@ class Model(MultimodalBaseModel):
                 fusion_input_features += self.nwp_encoders_dict[nwp_source].out_features
 
         if self.embedding_dim:
-            self.embed = nn.Embedding(num_embeddings=318, embedding_dim=embedding_dim)
+            self.embed = nn.Embedding(num_embeddings=num_embeddings, embedding_dim=embedding_dim)
             fusion_input_features += embedding_dim
 
         if self.include_sun:
@@ -209,6 +217,15 @@ class Model(MultimodalBaseModel):
 
     def teacher_forward(self, x):
         """Run the teacher models and return their encodings"""
+
+        if self.use_id_embedding:
+            # eg: x['gsp_id] = [1] with location_id_mapping = {1:0}, would give [0]
+            id = torch.tensor(
+                [self.location_id_mapping[i.item()] for i in x[f"{self._target_key}_id"]],
+                device=self.device,
+                dtype=torch.int64,
+            )
+
         modes = OrderedDict()
         for mode, teacher_model in self.teacher_models.items():
             # ******************* Satellite imagery *************************
@@ -218,7 +235,6 @@ class Model(MultimodalBaseModel):
                 sat_data = torch.swapaxes(sat_data, 1, 2).float()  # switch time and channels
 
                 if self.add_image_embedding_channel:
-                    id = x["gsp_id"].int()
                     sat_data = teacher_model.sat_embed(sat_data, id)
 
                 modes[mode] = teacher_model.sat_encoder(sat_data)
@@ -232,7 +248,6 @@ class Model(MultimodalBaseModel):
                 nwp_data = torch.swapaxes(nwp_data, 1, 2)  # switch time and channels
                 nwp_data = torch.clip(nwp_data, min=-50, max=50)
                 if teacher_model.add_image_embedding_channel:
-                    id = x["gsp_id"].int()
                     nwp_data = teacher_model.nwp_embed_dict[nwp_source](nwp_data, id)
 
                 nwp_out = teacher_model.nwp_encoders_dict[nwp_source](nwp_data)
@@ -251,6 +266,14 @@ class Model(MultimodalBaseModel):
         if self.adapt_batches:
             x = self._adapt_batch(x)
 
+        if self.use_id_embedding:
+            # eg: x['gsp_id] = [1] with location_id_mapping = {1:0}, would give [0]
+            id = torch.tensor(
+                [self.location_id_mapping[i.item()] for i in x[f"{self._target_key}_id"]],
+                device=self.device,
+                dtype=torch.int64,
+            )
+
         modes = OrderedDict()
         # ******************* Satellite imagery *************************
         if self.include_sat:
@@ -259,7 +282,6 @@ class Model(MultimodalBaseModel):
             sat_data = torch.swapaxes(sat_data, 1, 2).float()  # switch time and channels
 
             if self.add_image_embedding_channel:
-                id = x["gsp_id"].int()
                 sat_data = self.sat_embed(sat_data, id)
             modes["sat"] = self.sat_encoder(sat_data)
 
@@ -275,7 +297,6 @@ class Model(MultimodalBaseModel):
                 nwp_data = torch.clip(nwp_data, min=-50, max=50)
 
                 if self.add_image_embedding_channel:
-                    id = x["gsp_id"].int()
                     nwp_data = self.nwp_embed_dict[nwp_source](nwp_data, id)
 
                 nwp_out = self.nwp_encoders_dict[nwp_source](nwp_data)
@@ -299,10 +320,8 @@ class Model(MultimodalBaseModel):
             modes["gsp"] = gsp_history
 
         # ********************** Embedding of GSP ID ********************
-        if self.embedding_dim:
-            id = x["gsp_id"].int()
-            id_embedding = self.embed(id)
-            modes["id"] = id_embedding
+        if self.use_id_embedding:
+            modes["id"] = self.embed(id)
 
         if self.include_sun:
             # Use only new direct keys

pvnet-4.1.6/pvnet/__init__.py

@@ -1,2 +0,0 @@
-"""PVNet"""
-__version__ = "4.1.6"