PVNet 2.6.9__tar.gz → 2.6.10__tar.gz

{PVNet-2.6.9 → PVNet-2.6.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PVNet
-Version: 2.6.9
+Version: 2.6.10
 Summary: PVNet
 Author: Peter Dudfield
 Author-email: info@openclimatefix.org
@@ -32,6 +32,7 @@ Requires-Dist: hydra-core
 Requires-Dist: python-dotenv
 Requires-Dist: hydra-optuna-sweeper
 Requires-Dist: rich
+Requires-Dist: gcsfs
 Provides-Extra: all-models
 Requires-Dist: pytorch-tabnet; extra == "all-models"
 Requires-Dist: efficientnet_pytorch; extra == "all-models"
@@ -66,12 +67,26 @@ cd PVNet
 pip install -r requirements.txt
 ```
 
+The commit history is extensive. To save download time, use a depth of 1:
+```bash
+git clone --depth 1 https://github.com/openclimatefix/PVNet.git
+```
+This means only the latest commit and its associated files will be downloaded.
+
+Next, in the PVNet repo, install PVNet as an editable package:
+
+```bash
+pip install -e .
+```
+
 ### Additional development dependencies
 
 ```bash
 pip install -r requirements-dev.txt
 ```
 
+
+
 ## Getting started with running PVNet
 
 Before running any code in within PVNet, copy the example configuration to a
@@ -107,116 +122,103 @@ https://huggingface.co/datasets/openclimatefix/dwd-icon-eu which includes the UK
 OCF maintains a dataset of PV generation from 1311 private PV installations
 here: https://huggingface.co/datasets/openclimatefix/uk_pv
 
-### Generating pre-made batches of data for training/validation of PVNet
 
-PVNet contains a script for generating batches of data suitable for training the
-PVNet models.
+### Connecting with ocf_datapipes for batch creation
 
-To run the script you will need to make some modifications to the datamodule
-configuration.
+Outside the PVNet repo, clone the ocf-datapipes repo and exit the conda env created for PVNet: https://github.com/openclimatefix/ocf_datapipes
+```bash
+git clone --depth 1 https://github.com/openclimatefix/ocf_datapipes.git
+conda create -n ocf_datapipes python=3.10
+```
 
-1. First, create your new configuration file in
-   `./configs/datamodule/configiration/local_configuration.yaml` and paste the
-   sample config (shown below)
-2. Duplicate the `./configs/datamodule/ocf_datapipes.yaml` to
-   `./configs/datamodule/_local_ocf_datapipes.yaml` and ensure the
-   `configuration` key points to your newly created configuration file in
-   step 1.
-3. Also in this file, update the train, val & test periods to cover the data you
-   have access to.
-4. To get you started with your own configuration file, see the sample config
-   below. Update the data paths to the location of your local GSP, NWP and PV
-   datasets:
+Then go inside the ocf_datapipes repo to add packages
 
-```yaml
-general:
-  description: Demo config
-  name: demo_datamodule_config
-
-input_data:
-  default_history_minutes: 60
-  default_forecast_minutes: 120
-
-  gsp:
-    gsp_zarr_path: /path/to/gsp-data.zarr
-    history_minutes: 60
-    forecast_minutes: 120
-    time_resolution_minutes: 30
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    metadata_only: false
-
-  nwp:
-    ukv:
-      nwp_zarr_path: /path/to/nwp-data.zarr
-      history_minutes: 60
-      forecast_minutes: 120
-      time_resolution_minutes: 60
-      nwp_channels: # comment out channels as appropriate
-        - t         # 2-metre temperature
-        - dswrf     # downwards short-wave radiation flux
-        - dlwrf     # downwards long-wave radiation flux
-        - hcc       # high cloud cover
-        - mcc       # medium cloud cover
-        - lcc       # low cloud cover
-        - vis       # visability
-        - r         # relative humidity
-        - prate     # precipitation rate
-        - si10      # 10-metre wind speed | live = unknown
-      nwp_image_size_pixels_height: 24
-      nwp_image_size_pixels_width: 24
-      nwp_provider: ukv
-
-  pv:
-    pv_files_groups:
-      - label: pvoutput.org
-        pv_filename: /path/to/pv-data/pv.netcdf
-        pv_metadata_filename: /path/to/pv-data/metadata.csv
-    history_minutes: 60
-    forecast_minutes: 0 # PVNet assumes no future PV generation
-    time_resolution_minutes: 5
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    pv_image_size_meters_height: 24
-    pv_image_size_meters_width: 24
-    pv_ml_ids: [154,155,156,158,159,160,162,164,165,166,167,168,169,171,173,177,178,179,181,182,185,186,187,188,189,190,191,192,193,197,198,199,200,202,204,205,206,208,209,211,214,215,216,217,218,219,220,221,225,229,230,232,233,234,236,242,243,245,252,254,255,256,257,258,260,261,262,265,267,268,272,273,275,276,277,280,281,282,283,287,289,291,292,293,294,295,296,297,298,301,302,303,304,306,307,309,310,311,317,318,319,320,321,322,323,325,326,329,332,333,335,336,338,340,342,344,345,346,348,349,352,354,355,356,357,360,362,363,368,369,370,371,372,374,375,376,378,380,382,384,385,388,390,391,393,396,397,398,399,400,401,403,404,405,406,407,409,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,429,431,435,437,438,440,441,444,447,450,451,453,456,457,458,459,464,465,466,467,468,470,471,473,474,476,477,479,480,481,482,485,486,488,490,491,492,493,496,498,501,503,506,507,508,509,510,511,512,513,515,516,517,519,520,521,522,524,526,527,528,531,532,536,537,538,540,541,542,543,544,545,549,550,551,552,553,554,556,557,560,561,563,566,568,571,572,575,576,577,579,580,581,582,584,585,588,590,594,595,597,600,602,603,604,606,611,613,614,616,618,620,622,623,624,625,626,628,629,630,631,636,637,638,640,641,642,644,645,646,650,651,652,653,654,655,657,660,661,662,663,666,667,668,670,675,676,679,681,683,684,685,687,696,698,701,702,703,704,706,710,722,723,724,725,727,728,729,730,732,733,734,735,736,737,]
-    n_pv_systems_per_example: 128
-    get_center: false
-    is_live: false
-
-  satellite:
-    satellite_zarr_path: "" # Left empty to avoid using satellite data
-    history_minutes: 60
-    forecast_minutes: 0
-    live_delay_minutes: 30
-    time_resolution_minutes: 5
-    satellite_channels:
-      - IR_016
-      - IR_039
-      - IR_087
-      - IR_097
-      - IR_108
-      - IR_120
-      - IR_134
-      - VIS006
-      - VIS008
-      - WV_062
-      - WV_073
-    satellite_image_size_pixels_height: 24
-    satellite_image_size_pixels_width: 24
+```bash
+pip install -r requirements.txt requirements-dev.txt
 ```
 
+Then exit this environment, and enter back into the pvnet conda environment and install ocf_datapies in editable mode (-e). This means the package is directly linked to the source code in the ocf_datapies repo.
+
+```bash
+pip install -e <PATH-TO-ocf_datapipes-REPO>
+```
+
+## Generating pre-made batches of data for training/validation of PVNet
+
+PVNet contains a script for generating batches of data suitable for training the PVNet models. To run the script you will need to make some modifications to the datamodule configuration.
+
+Make sure you have copied the example configs (as already stated above):
+```
+cp -r configs.example configs
+```
+
+### Set up and config example for batch creation
+
+We will use the example of creating batches using data from gcp:
+`/PVNet/configs/datamodule/configuration/gcp_configuration.yaml`
+Ensure that the file paths are set to the correct locations in
+`gcp_configuration.yaml`.
+
+`PLACEHOLDER` is used to indcate where to input the location of the files.
+
+For OCF use cases, file locations can be found in `template_configuration.yaml` located alongside `gcp_configuration.yaml`.
+
+In these configurations you can update the train, val & test periods to cover the data you have access to.
+
+
 With your configuration in place, you can proceed to create batches. PVNet uses
 [hydra](https://hydra.cc/) which enables us to pass variables via the command
 line that will override the configuration defined in the `./configs` directory.
 
-Run the save_batches.py script to create batches with the following arguments as
-a minimum:
+When creating batches, an additional config is used which is passed into the batch creation script. This is the datamodule config located `PVNet/configs/datamodule`.
+
+For this example we will be using the `streamed_batches.yaml` config. Like before, a placeholder variable is used when specifing which configuration to use:
+
+`configuration: "PLACEHOLDER.yaml"`
+
+This should be given the whole path to the config on your local machine, such as for our example it should be changed to:
+
+`configuration: "/FULL-PATH-TO-REPO/PVNet/configs/datamodule/configuration/gcp_configuration.yaml"
+`
+
+Where `FULL-PATH-TO-REPO` represent the whole path to the PVNet repo on your local machine.
+
+### Running the batch creation script
+
+Run the save_batches.py script to create batches with the following example arguments as:
 
 ```
-python scripts/save_batches.py datamodule=local_ocf_datapipes +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
+python scripts/save_batches.py datamodule=streamed_batches +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
 ```
 
+In this function the datamodule argument looks for a config under `PVNet/configs/datamodule`. The examples here are either to use "premade_batches" or "streamed_batches".
+
+Its important that the dates set for the training, validation and testing in the datamodule (`streamed_batches.yaml`) config are within the ranges of the dates set for the input features in the configuration (`gcp_configuration.yaml`).
+
+If downloading private data from a gcp bucket make sure to authenticate gcloud (the public satellite data does not need authentication):
+
+```
+gcloud auth login
+```
+
+For files stored in multiple locations they can be added as list. For example from the gcp_configuration.yaml file we can change from satellite data stored on a bucket:
+
+```
+satellite:
+    satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr
+```
+
+To satellite data hosted by Google:
+
+```
+satellite:
+    satellite_zarr_paths:
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr"
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2021_nonhrv.zarr"
+```
+Datapipes is currently set up to use 11 channels from the satellite data, the 12th of which is HRV and is not included in these.
+
+
 ### Training PVNet
 
 How PVNet is run is determined by the extensive configuration in the config

{PVNet-2.6.9 → PVNet-2.6.10}/PVNet.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PVNet
-Version: 2.6.9
+Version: 2.6.10
 Summary: PVNet
 Author: Peter Dudfield
 Author-email: info@openclimatefix.org
@@ -32,6 +32,7 @@ Requires-Dist: hydra-core
 Requires-Dist: python-dotenv
 Requires-Dist: hydra-optuna-sweeper
 Requires-Dist: rich
+Requires-Dist: gcsfs
 Provides-Extra: all-models
 Requires-Dist: pytorch-tabnet; extra == "all-models"
 Requires-Dist: efficientnet_pytorch; extra == "all-models"
@@ -66,12 +67,26 @@ cd PVNet
 pip install -r requirements.txt
 ```
 
+The commit history is extensive. To save download time, use a depth of 1:
+```bash
+git clone --depth 1 https://github.com/openclimatefix/PVNet.git
+```
+This means only the latest commit and its associated files will be downloaded.
+
+Next, in the PVNet repo, install PVNet as an editable package:
+
+```bash
+pip install -e .
+```
+
 ### Additional development dependencies
 
 ```bash
 pip install -r requirements-dev.txt
 ```
 
+
+
 ## Getting started with running PVNet
 
 Before running any code in within PVNet, copy the example configuration to a
@@ -107,116 +122,103 @@ https://huggingface.co/datasets/openclimatefix/dwd-icon-eu which includes the UK
 OCF maintains a dataset of PV generation from 1311 private PV installations
 here: https://huggingface.co/datasets/openclimatefix/uk_pv
 
-### Generating pre-made batches of data for training/validation of PVNet
 
-PVNet contains a script for generating batches of data suitable for training the
-PVNet models.
+### Connecting with ocf_datapipes for batch creation
 
-To run the script you will need to make some modifications to the datamodule
-configuration.
+Outside the PVNet repo, clone the ocf-datapipes repo and exit the conda env created for PVNet: https://github.com/openclimatefix/ocf_datapipes
+```bash
+git clone --depth 1 https://github.com/openclimatefix/ocf_datapipes.git
+conda create -n ocf_datapipes python=3.10
+```
 
-1. First, create your new configuration file in
-   `./configs/datamodule/configiration/local_configuration.yaml` and paste the
-   sample config (shown below)
-2. Duplicate the `./configs/datamodule/ocf_datapipes.yaml` to
-   `./configs/datamodule/_local_ocf_datapipes.yaml` and ensure the
-   `configuration` key points to your newly created configuration file in
-   step 1.
-3. Also in this file, update the train, val & test periods to cover the data you
-   have access to.
-4. To get you started with your own configuration file, see the sample config
-   below. Update the data paths to the location of your local GSP, NWP and PV
-   datasets:
+Then go inside the ocf_datapipes repo to add packages
 
-```yaml
-general:
-  description: Demo config
-  name: demo_datamodule_config
-
-input_data:
-  default_history_minutes: 60
-  default_forecast_minutes: 120
-
-  gsp:
-    gsp_zarr_path: /path/to/gsp-data.zarr
-    history_minutes: 60
-    forecast_minutes: 120
-    time_resolution_minutes: 30
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    metadata_only: false
-
-  nwp:
-    ukv:
-      nwp_zarr_path: /path/to/nwp-data.zarr
-      history_minutes: 60
-      forecast_minutes: 120
-      time_resolution_minutes: 60
-      nwp_channels: # comment out channels as appropriate
-        - t         # 2-metre temperature
-        - dswrf     # downwards short-wave radiation flux
-        - dlwrf     # downwards long-wave radiation flux
-        - hcc       # high cloud cover
-        - mcc       # medium cloud cover
-        - lcc       # low cloud cover
-        - vis       # visability
-        - r         # relative humidity
-        - prate     # precipitation rate
-        - si10      # 10-metre wind speed | live = unknown
-      nwp_image_size_pixels_height: 24
-      nwp_image_size_pixels_width: 24
-      nwp_provider: ukv
-
-  pv:
-    pv_files_groups:
-      - label: pvoutput.org
-        pv_filename: /path/to/pv-data/pv.netcdf
-        pv_metadata_filename: /path/to/pv-data/metadata.csv
-    history_minutes: 60
-    forecast_minutes: 0 # PVNet assumes no future PV generation
-    time_resolution_minutes: 5
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    pv_image_size_meters_height: 24
-    pv_image_size_meters_width: 24
-    pv_ml_ids: [154,155,156,158,159,160,162,164,165,166,167,168,169,171,173,177,178,179,181,182,185,186,187,188,189,190,191,192,193,197,198,199,200,202,204,205,206,208,209,211,214,215,216,217,218,219,220,221,225,229,230,232,233,234,236,242,243,245,252,254,255,256,257,258,260,261,262,265,267,268,272,273,275,276,277,280,281,282,283,287,289,291,292,293,294,295,296,297,298,301,302,303,304,306,307,309,310,311,317,318,319,320,321,322,323,325,326,329,332,333,335,336,338,340,342,344,345,346,348,349,352,354,355,356,357,360,362,363,368,369,370,371,372,374,375,376,378,380,382,384,385,388,390,391,393,396,397,398,399,400,401,403,404,405,406,407,409,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,429,431,435,437,438,440,441,444,447,450,451,453,456,457,458,459,464,465,466,467,468,470,471,473,474,476,477,479,480,481,482,485,486,488,490,491,492,493,496,498,501,503,506,507,508,509,510,511,512,513,515,516,517,519,520,521,522,524,526,527,528,531,532,536,537,538,540,541,542,543,544,545,549,550,551,552,553,554,556,557,560,561,563,566,568,571,572,575,576,577,579,580,581,582,584,585,588,590,594,595,597,600,602,603,604,606,611,613,614,616,618,620,622,623,624,625,626,628,629,630,631,636,637,638,640,641,642,644,645,646,650,651,652,653,654,655,657,660,661,662,663,666,667,668,670,675,676,679,681,683,684,685,687,696,698,701,702,703,704,706,710,722,723,724,725,727,728,729,730,732,733,734,735,736,737,]
-    n_pv_systems_per_example: 128
-    get_center: false
-    is_live: false
-
-  satellite:
-    satellite_zarr_path: "" # Left empty to avoid using satellite data
-    history_minutes: 60
-    forecast_minutes: 0
-    live_delay_minutes: 30
-    time_resolution_minutes: 5
-    satellite_channels:
-      - IR_016
-      - IR_039
-      - IR_087
-      - IR_097
-      - IR_108
-      - IR_120
-      - IR_134
-      - VIS006
-      - VIS008
-      - WV_062
-      - WV_073
-    satellite_image_size_pixels_height: 24
-    satellite_image_size_pixels_width: 24
+```bash
+pip install -r requirements.txt requirements-dev.txt
 ```
 
+Then exit this environment, and enter back into the pvnet conda environment and install ocf_datapies in editable mode (-e). This means the package is directly linked to the source code in the ocf_datapies repo.
+
+```bash
+pip install -e <PATH-TO-ocf_datapipes-REPO>
+```
+
+## Generating pre-made batches of data for training/validation of PVNet
+
+PVNet contains a script for generating batches of data suitable for training the PVNet models. To run the script you will need to make some modifications to the datamodule configuration.
+
+Make sure you have copied the example configs (as already stated above):
+```
+cp -r configs.example configs
+```
+
+### Set up and config example for batch creation
+
+We will use the example of creating batches using data from gcp:
+`/PVNet/configs/datamodule/configuration/gcp_configuration.yaml`
+Ensure that the file paths are set to the correct locations in
+`gcp_configuration.yaml`.
+
+`PLACEHOLDER` is used to indcate where to input the location of the files.
+
+For OCF use cases, file locations can be found in `template_configuration.yaml` located alongside `gcp_configuration.yaml`.
+
+In these configurations you can update the train, val & test periods to cover the data you have access to.
+
+
 With your configuration in place, you can proceed to create batches. PVNet uses
 [hydra](https://hydra.cc/) which enables us to pass variables via the command
 line that will override the configuration defined in the `./configs` directory.
 
-Run the save_batches.py script to create batches with the following arguments as
-a minimum:
+When creating batches, an additional config is used which is passed into the batch creation script. This is the datamodule config located `PVNet/configs/datamodule`.
+
+For this example we will be using the `streamed_batches.yaml` config. Like before, a placeholder variable is used when specifing which configuration to use:
+
+`configuration: "PLACEHOLDER.yaml"`
+
+This should be given the whole path to the config on your local machine, such as for our example it should be changed to:
+
+`configuration: "/FULL-PATH-TO-REPO/PVNet/configs/datamodule/configuration/gcp_configuration.yaml"
+`
+
+Where `FULL-PATH-TO-REPO` represent the whole path to the PVNet repo on your local machine.
+
+### Running the batch creation script
+
+Run the save_batches.py script to create batches with the following example arguments as:
 
 ```
-python scripts/save_batches.py datamodule=local_ocf_datapipes +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
+python scripts/save_batches.py datamodule=streamed_batches +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
 ```
 
+In this function the datamodule argument looks for a config under `PVNet/configs/datamodule`. The examples here are either to use "premade_batches" or "streamed_batches".
+
+Its important that the dates set for the training, validation and testing in the datamodule (`streamed_batches.yaml`) config are within the ranges of the dates set for the input features in the configuration (`gcp_configuration.yaml`).
+
+If downloading private data from a gcp bucket make sure to authenticate gcloud (the public satellite data does not need authentication):
+
+```
+gcloud auth login
+```
+
+For files stored in multiple locations they can be added as list. For example from the gcp_configuration.yaml file we can change from satellite data stored on a bucket:
+
+```
+satellite:
+    satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr
+```
+
+To satellite data hosted by Google:
+
+```
+satellite:
+    satellite_zarr_paths:
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr"
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2021_nonhrv.zarr"
+```
+Datapipes is currently set up to use 11 channels from the satellite data, the 12th of which is HRV and is not included in these.
+
+
 ### Training PVNet
 
 How PVNet is run is determined by the extensive configuration in the config

{PVNet-2.6.9 → PVNet-2.6.10}/PVNet.egg-info/requires.txt

@@ -23,6 +23,7 @@ hydra-core
 python-dotenv
 hydra-optuna-sweeper
 rich
+gcsfs
 
 [all_models]
 pytorch-tabnet

{PVNet-2.6.9 → PVNet-2.6.10}/README.md

@@ -28,12 +28,26 @@ cd PVNet
 pip install -r requirements.txt
 ```
 
+The commit history is extensive. To save download time, use a depth of 1:
+```bash
+git clone --depth 1 https://github.com/openclimatefix/PVNet.git
+```
+This means only the latest commit and its associated files will be downloaded.
+
+Next, in the PVNet repo, install PVNet as an editable package:
+
+```bash
+pip install -e .
+```
+
 ### Additional development dependencies
 
 ```bash
 pip install -r requirements-dev.txt
 ```
 
+
+
 ## Getting started with running PVNet
 
 Before running any code in within PVNet, copy the example configuration to a
@@ -69,116 +83,103 @@ https://huggingface.co/datasets/openclimatefix/dwd-icon-eu which includes the UK
 OCF maintains a dataset of PV generation from 1311 private PV installations
 here: https://huggingface.co/datasets/openclimatefix/uk_pv
 
-### Generating pre-made batches of data for training/validation of PVNet
 
-PVNet contains a script for generating batches of data suitable for training the
-PVNet models.
+### Connecting with ocf_datapipes for batch creation
 
-To run the script you will need to make some modifications to the datamodule
-configuration.
+Outside the PVNet repo, clone the ocf-datapipes repo and exit the conda env created for PVNet: https://github.com/openclimatefix/ocf_datapipes
+```bash
+git clone --depth 1 https://github.com/openclimatefix/ocf_datapipes.git
+conda create -n ocf_datapipes python=3.10
+```
 
-1. First, create your new configuration file in
-   `./configs/datamodule/configiration/local_configuration.yaml` and paste the
-   sample config (shown below)
-2. Duplicate the `./configs/datamodule/ocf_datapipes.yaml` to
-   `./configs/datamodule/_local_ocf_datapipes.yaml` and ensure the
-   `configuration` key points to your newly created configuration file in
-   step 1.
-3. Also in this file, update the train, val & test periods to cover the data you
-   have access to.
-4. To get you started with your own configuration file, see the sample config
-   below. Update the data paths to the location of your local GSP, NWP and PV
-   datasets:
+Then go inside the ocf_datapipes repo to add packages
 
-```yaml
-general:
-  description: Demo config
-  name: demo_datamodule_config
-
-input_data:
-  default_history_minutes: 60
-  default_forecast_minutes: 120
-
-  gsp:
-    gsp_zarr_path: /path/to/gsp-data.zarr
-    history_minutes: 60
-    forecast_minutes: 120
-    time_resolution_minutes: 30
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    metadata_only: false
-
-  nwp:
-    ukv:
-      nwp_zarr_path: /path/to/nwp-data.zarr
-      history_minutes: 60
-      forecast_minutes: 120
-      time_resolution_minutes: 60
-      nwp_channels: # comment out channels as appropriate
-        - t         # 2-metre temperature
-        - dswrf     # downwards short-wave radiation flux
-        - dlwrf     # downwards long-wave radiation flux
-        - hcc       # high cloud cover
-        - mcc       # medium cloud cover
-        - lcc       # low cloud cover
-        - vis       # visability
-        - r         # relative humidity
-        - prate     # precipitation rate
-        - si10      # 10-metre wind speed | live = unknown
-      nwp_image_size_pixels_height: 24
-      nwp_image_size_pixels_width: 24
-      nwp_provider: ukv
-
-  pv:
-    pv_files_groups:
-      - label: pvoutput.org
-        pv_filename: /path/to/pv-data/pv.netcdf
-        pv_metadata_filename: /path/to/pv-data/metadata.csv
-    history_minutes: 60
-    forecast_minutes: 0 # PVNet assumes no future PV generation
-    time_resolution_minutes: 5
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    pv_image_size_meters_height: 24
-    pv_image_size_meters_width: 24
-    pv_ml_ids: [154,155,156,158,159,160,162,164,165,166,167,168,169,171,173,177,178,179,181,182,185,186,187,188,189,190,191,192,193,197,198,199,200,202,204,205,206,208,209,211,214,215,216,217,218,219,220,221,225,229,230,232,233,234,236,242,243,245,252,254,255,256,257,258,260,261,262,265,267,268,272,273,275,276,277,280,281,282,283,287,289,291,292,293,294,295,296,297,298,301,302,303,304,306,307,309,310,311,317,318,319,320,321,322,323,325,326,329,332,333,335,336,338,340,342,344,345,346,348,349,352,354,355,356,357,360,362,363,368,369,370,371,372,374,375,376,378,380,382,384,385,388,390,391,393,396,397,398,399,400,401,403,404,405,406,407,409,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,429,431,435,437,438,440,441,444,447,450,451,453,456,457,458,459,464,465,466,467,468,470,471,473,474,476,477,479,480,481,482,485,486,488,490,491,492,493,496,498,501,503,506,507,508,509,510,511,512,513,515,516,517,519,520,521,522,524,526,527,528,531,532,536,537,538,540,541,542,543,544,545,549,550,551,552,553,554,556,557,560,561,563,566,568,571,572,575,576,577,579,580,581,582,584,585,588,590,594,595,597,600,602,603,604,606,611,613,614,616,618,620,622,623,624,625,626,628,629,630,631,636,637,638,640,641,642,644,645,646,650,651,652,653,654,655,657,660,661,662,663,666,667,668,670,675,676,679,681,683,684,685,687,696,698,701,702,703,704,706,710,722,723,724,725,727,728,729,730,732,733,734,735,736,737,]
-    n_pv_systems_per_example: 128
-    get_center: false
-    is_live: false
-
-  satellite:
-    satellite_zarr_path: "" # Left empty to avoid using satellite data
-    history_minutes: 60
-    forecast_minutes: 0
-    live_delay_minutes: 30
-    time_resolution_minutes: 5
-    satellite_channels:
-      - IR_016
-      - IR_039
-      - IR_087
-      - IR_097
-      - IR_108
-      - IR_120
-      - IR_134
-      - VIS006
-      - VIS008
-      - WV_062
-      - WV_073
-    satellite_image_size_pixels_height: 24
-    satellite_image_size_pixels_width: 24
+```bash
+pip install -r requirements.txt requirements-dev.txt
 ```
 
+Then exit this environment, and enter back into the pvnet conda environment and install ocf_datapies in editable mode (-e). This means the package is directly linked to the source code in the ocf_datapies repo.
+
+```bash
+pip install -e <PATH-TO-ocf_datapipes-REPO>
+```
+
+## Generating pre-made batches of data for training/validation of PVNet
+
+PVNet contains a script for generating batches of data suitable for training the PVNet models. To run the script you will need to make some modifications to the datamodule configuration.
+
+Make sure you have copied the example configs (as already stated above):
+```
+cp -r configs.example configs
+```
+
+### Set up and config example for batch creation
+
+We will use the example of creating batches using data from gcp:
+`/PVNet/configs/datamodule/configuration/gcp_configuration.yaml`
+Ensure that the file paths are set to the correct locations in
+`gcp_configuration.yaml`.
+
+`PLACEHOLDER` is used to indcate where to input the location of the files.
+
+For OCF use cases, file locations can be found in `template_configuration.yaml` located alongside `gcp_configuration.yaml`.
+
+In these configurations you can update the train, val & test periods to cover the data you have access to.
+
+
 With your configuration in place, you can proceed to create batches. PVNet uses
 [hydra](https://hydra.cc/) which enables us to pass variables via the command
 line that will override the configuration defined in the `./configs` directory.
 
-Run the save_batches.py script to create batches with the following arguments as
-a minimum:
+When creating batches, an additional config is used which is passed into the batch creation script. This is the datamodule config located `PVNet/configs/datamodule`.
+
+For this example we will be using the `streamed_batches.yaml` config. Like before, a placeholder variable is used when specifing which configuration to use:
+
+`configuration: "PLACEHOLDER.yaml"`
+
+This should be given the whole path to the config on your local machine, such as for our example it should be changed to:
+
+`configuration: "/FULL-PATH-TO-REPO/PVNet/configs/datamodule/configuration/gcp_configuration.yaml"
+`
+
+Where `FULL-PATH-TO-REPO` represent the whole path to the PVNet repo on your local machine.
+
+### Running the batch creation script
+
+Run the save_batches.py script to create batches with the following example arguments as:
 
 ```
-python scripts/save_batches.py datamodule=local_ocf_datapipes +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
+python scripts/save_batches.py datamodule=streamed_batches +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
 ```
 
+In this function the datamodule argument looks for a config under `PVNet/configs/datamodule`. The examples here are either to use "premade_batches" or "streamed_batches".
+
+Its important that the dates set for the training, validation and testing in the datamodule (`streamed_batches.yaml`) config are within the ranges of the dates set for the input features in the configuration (`gcp_configuration.yaml`).
+
+If downloading private data from a gcp bucket make sure to authenticate gcloud (the public satellite data does not need authentication):
+
+```
+gcloud auth login
+```
+
+For files stored in multiple locations they can be added as list. For example from the gcp_configuration.yaml file we can change from satellite data stored on a bucket:
+
+```
+satellite:
+    satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr
+```
+
+To satellite data hosted by Google:
+
+```
+satellite:
+    satellite_zarr_paths:
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr"
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2021_nonhrv.zarr"
+```
+Datapipes is currently set up to use 11 channels from the satellite data, the 12th of which is HRV and is not included in these.
+
+
 ### Training PVNet
 
 How PVNet is run is determined by the extensive configuration in the config

PVNet-2.6.10/pvnet/__init__.py

@@ -0,0 +1,2 @@
+"""PVNet"""
+__version__ = "2.6.10"

{PVNet-2.6.9 → PVNet-2.6.10}/pvnet/models/base_model.py

@@ -234,6 +234,7 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
         forecast_minutes: int,
         optimizer: AbstractOptimizer,
         output_quantiles: Optional[list[float]] = None,
+        target_key: BatchKey = BatchKey.gsp,
     ):
         """Abtstract base class for PVNet submodels.
 
@@ -243,10 +244,12 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
             optimizer (AbstractOptimizer): Optimizer
             output_quantiles: A list of float (0.0, 1.0) quantiles to predict values for. If set to
                 None the output is a single value.
+            target_key: The key of the target variable in the batch
         """
         super().__init__()
 
         self._optimizer = optimizer
+        self._target_key = target_key
 
         # Model must have lr to allow tuning
         # This setting is only used when lr is tuned with callback
@@ -424,7 +427,7 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
     def training_step(self, batch, batch_idx):
         """Run training step"""
         y_hat = self(batch)
-        y = batch[BatchKey.gsp][:, -self.forecast_len_30 :, 0]
+        y = batch[self._target_key][:, -self.forecast_len_30 :, 0]
 
         losses = self._calculate_common_losses(y, y_hat)
         losses = {f"{k}/train": v for k, v in losses.items()}
@@ -440,7 +443,7 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
     def validation_step(self, batch: dict, batch_idx):
         """Run validation step"""
         y_hat = self(batch)
-        y = batch[BatchKey.gsp][:, -self.forecast_len_30 :, 0]
+        y = batch[self._target_key][:, -self.forecast_len_30 :, 0]
 
         losses = self._calculate_common_losses(y, y_hat)
         losses.update(self._calculate_val_losses(y, y_hat))
@@ -484,7 +487,7 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
     def test_step(self, batch, batch_idx):
         """Run test step"""
         y_hat = self(batch)
-        y = batch[BatchKey.gsp][:, -self.forecast_len_30 :, 0]
+        y = batch[self._target_key][:, -self.forecast_len_30 :, 0]
 
         losses = self._calculate_common_losses(y, y_hat)
         losses.update(self._calculate_val_losses(y, y_hat))

{PVNet-2.6.9 → PVNet-2.6.10}/pvnet/models/multimodal/multimodal.py

@@ -54,6 +54,7 @@ class Model(BaseModel):
         nwp_history_minutes: Optional[int] = None,
         pv_history_minutes: Optional[int] = None,
         optimizer: AbstractOptimizer = pvnet.optimizers.Adam(),
+        target_key: str = "gsp",
     ):
         """Neural network which combines information from different sources.
 
@@ -94,6 +95,7 @@ class Model(BaseModel):
             pv_history_minutes: Length of recent site-level PV data data used as input. Defaults to
                 `history_minutes` if not provided.
             optimizer: Optimizer factory function used for network.
+            target_key: The key of the target variable in the batch.
         """
 
         self.include_gsp_yield_history = include_gsp_yield_history
@@ -103,8 +105,15 @@ class Model(BaseModel):
         self.include_sun = include_sun
         self.embedding_dim = embedding_dim
         self.add_image_embedding_channel = add_image_embedding_channel
-
-        super().__init__(history_minutes, forecast_minutes, optimizer, output_quantiles)
+        self.target_key_name = target_key
+
+        super().__init__(
+            history_minutes=history_minutes,
+            forecast_minutes=forecast_minutes,
+            optimizer=optimizer,
+            output_quantiles=output_quantiles,
+            target_key=BatchKey.gsp if target_key == "gsp" else BatchKey.pv,
+        )
 
         # Number of features expected by the output_network
         # Add to this as network pices are constructed
@@ -228,7 +237,12 @@ class Model(BaseModel):
         # *********************** PV Data *************************************
         # Add site-level PV yield
         if self.include_pv:
-            modes["pv"] = self.pv_encoder(x)
+            if self.target_key_name != "pv":
+                modes["pv"] = self.pv_encoder(x)
+            else:
+                # Target is PV, so only take the history
+                pv_history = x[BatchKey.pv][:, : self.history_len_30].float()
+                modes["pv"] = self.pv_encoder(pv_history)
 
         # *********************** GSP Data ************************************
         # add gsp yield history

{PVNet-2.6.9 → PVNet-2.6.10}/pvnet/utils.py

@@ -242,19 +242,24 @@ def finish(
             wandb.finish()
 
 
-def plot_batch_forecasts(batch, y_hat, batch_idx=None, quantiles=None):
+def plot_batch_forecasts(batch, y_hat, batch_idx=None, quantiles=None, key_to_plot: str = "gsp"):
     """Plot a batch of data and the forecast from that batch"""
 
     def _get_numpy(key):
         return batch[key].cpu().numpy().squeeze()
 
-    y = batch[BatchKey.gsp].cpu().numpy()
+    y_key = BatchKey.gsp if key_to_plot == "gsp" else BatchKey.pv
+    y_id_key = BatchKey.gsp_id if key_to_plot == "gsp" else BatchKey.pv_id
+    t0_idx_key = BatchKey.gsp_t0_idx if key_to_plot == "gsp" else BatchKey.pv_t0_idx
+    time_utc_key = BatchKey.gsp_time_utc if key_to_plot == "gsp" else BatchKey.pv_time_utc
+    plotting_name = key_to_plot.upper()
+    y = batch[y_key].cpu().numpy()
     y_hat = y_hat.cpu().numpy()
 
-    gsp_ids = batch[BatchKey.gsp_id].cpu().numpy().squeeze()
-    batch[BatchKey.gsp_t0_idx]
+    gsp_ids = batch[y_id_key].cpu().numpy().squeeze()
+    batch[t0_idx_key]
 
-    times_utc = batch[BatchKey.gsp_time_utc].cpu().numpy().squeeze().astype("datetime64[s]")
+    times_utc = batch[time_utc_key].cpu().numpy().squeeze().astype("datetime64[s]")
     times_utc = [pd.to_datetime(t) for t in times_utc]
 
     batch_size = y.shape[0]
@@ -294,9 +299,9 @@ def plot_batch_forecasts(batch, y_hat, batch_idx=None, quantiles=None):
         ax.set_xlabel("Time (hour of day)")
 
     if batch_idx is not None:
-        title = f"Normed GSP output : batch_idx={batch_idx}"
+        title = f"Normed {plotting_name} output : batch_idx={batch_idx}"
     else:
-        title = "Normed GSP output"
+        title = f"Normed {plotting_name} output"
     plt.suptitle(title)
     plt.tight_layout()
 

{PVNet-2.6.9 → PVNet-2.6.10}/requirements.txt

@@ -23,3 +23,5 @@ hydra-core
 python-dotenv
 hydra-optuna-sweeper
 rich
+# gcsfs is only needed when getting data from Google Cloud Storage
+gcsfs

PVNet-2.6.9/pvnet/__init__.py

@@ -1,2 +0,0 @@
-"""PVNet"""
-__version__ = "2.6.9"