PVNet 2.6.8__tar.gz → 2.6.10__tar.gz

{PVNet-2.6.8 → PVNet-2.6.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PVNet
-Version: 2.6.8
+Version: 2.6.10
 Summary: PVNet
 Author: Peter Dudfield
 Author-email: info@openclimatefix.org
@@ -32,6 +32,7 @@ Requires-Dist: hydra-core
 Requires-Dist: python-dotenv
 Requires-Dist: hydra-optuna-sweeper
 Requires-Dist: rich
+Requires-Dist: gcsfs
 Provides-Extra: all-models
 Requires-Dist: pytorch-tabnet; extra == "all-models"
 Requires-Dist: efficientnet_pytorch; extra == "all-models"
@@ -66,12 +67,26 @@ cd PVNet
 pip install -r requirements.txt
 ```
 
+The commit history is extensive. To save download time, use a depth of 1:
+```bash
+git clone --depth 1 https://github.com/openclimatefix/PVNet.git
+```
+This means only the latest commit and its associated files will be downloaded.
+
+Next, in the PVNet repo, install PVNet as an editable package:
+
+```bash
+pip install -e .
+```
+
 ### Additional development dependencies
 
 ```bash
 pip install -r requirements-dev.txt
 ```
 
+
+
 ## Getting started with running PVNet
 
 Before running any code in within PVNet, copy the example configuration to a
@@ -107,116 +122,103 @@ https://huggingface.co/datasets/openclimatefix/dwd-icon-eu which includes the UK
 OCF maintains a dataset of PV generation from 1311 private PV installations
 here: https://huggingface.co/datasets/openclimatefix/uk_pv
 
-### Generating pre-made batches of data for training/validation of PVNet
 
-PVNet contains a script for generating batches of data suitable for training the
-PVNet models.
+### Connecting with ocf_datapipes for batch creation
 
-To run the script you will need to make some modifications to the datamodule
-configuration.
+Outside the PVNet repo, clone the ocf-datapipes repo and exit the conda env created for PVNet: https://github.com/openclimatefix/ocf_datapipes
+```bash
+git clone --depth 1 https://github.com/openclimatefix/ocf_datapipes.git
+conda create -n ocf_datapipes python=3.10
+```
 
-1. First, create your new configuration file in
-   `./configs/datamodule/configiration/local_configuration.yaml` and paste the
-   sample config (shown below)
-2. Duplicate the `./configs/datamodule/ocf_datapipes.yaml` to
-   `./configs/datamodule/_local_ocf_datapipes.yaml` and ensure the
-   `configuration` key points to your newly created configuration file in
-   step 1.
-3. Also in this file, update the train, val & test periods to cover the data you
-   have access to.
-4. To get you started with your own configuration file, see the sample config
-   below. Update the data paths to the location of your local GSP, NWP and PV
-   datasets:
+Then go inside the ocf_datapipes repo to add packages
 
-```yaml
-general:
-  description: Demo config
-  name: demo_datamodule_config
-
-input_data:
-  default_history_minutes: 60
-  default_forecast_minutes: 120
-
-  gsp:
-    gsp_zarr_path: /path/to/gsp-data.zarr
-    history_minutes: 60
-    forecast_minutes: 120
-    time_resolution_minutes: 30
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    metadata_only: false
-
-  nwp:
-    ukv:
-      nwp_zarr_path: /path/to/nwp-data.zarr
-      history_minutes: 60
-      forecast_minutes: 120
-      time_resolution_minutes: 60
-      nwp_channels: # comment out channels as appropriate
-        - t         # 2-metre temperature
-        - dswrf     # downwards short-wave radiation flux
-        - dlwrf     # downwards long-wave radiation flux
-        - hcc       # high cloud cover
-        - mcc       # medium cloud cover
-        - lcc       # low cloud cover
-        - vis       # visability
-        - r         # relative humidity
-        - prate     # precipitation rate
-        - si10      # 10-metre wind speed | live = unknown
-      nwp_image_size_pixels_height: 24
-      nwp_image_size_pixels_width: 24
-      nwp_provider: ukv
-
-  pv:
-    pv_files_groups:
-      - label: pvoutput.org
-        pv_filename: /path/to/pv-data/pv.netcdf
-        pv_metadata_filename: /path/to/pv-data/metadata.csv
-    history_minutes: 60
-    forecast_minutes: 0 # PVNet assumes no future PV generation
-    time_resolution_minutes: 5
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    pv_image_size_meters_height: 24
-    pv_image_size_meters_width: 24
-    pv_ml_ids: [154,155,156,158,159,160,162,164,165,166,167,168,169,171,173,177,178,179,181,182,185,186,187,188,189,190,191,192,193,197,198,199,200,202,204,205,206,208,209,211,214,215,216,217,218,219,220,221,225,229,230,232,233,234,236,242,243,245,252,254,255,256,257,258,260,261,262,265,267,268,272,273,275,276,277,280,281,282,283,287,289,291,292,293,294,295,296,297,298,301,302,303,304,306,307,309,310,311,317,318,319,320,321,322,323,325,326,329,332,333,335,336,338,340,342,344,345,346,348,349,352,354,355,356,357,360,362,363,368,369,370,371,372,374,375,376,378,380,382,384,385,388,390,391,393,396,397,398,399,400,401,403,404,405,406,407,409,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,429,431,435,437,438,440,441,444,447,450,451,453,456,457,458,459,464,465,466,467,468,470,471,473,474,476,477,479,480,481,482,485,486,488,490,491,492,493,496,498,501,503,506,507,508,509,510,511,512,513,515,516,517,519,520,521,522,524,526,527,528,531,532,536,537,538,540,541,542,543,544,545,549,550,551,552,553,554,556,557,560,561,563,566,568,571,572,575,576,577,579,580,581,582,584,585,588,590,594,595,597,600,602,603,604,606,611,613,614,616,618,620,622,623,624,625,626,628,629,630,631,636,637,638,640,641,642,644,645,646,650,651,652,653,654,655,657,660,661,662,663,666,667,668,670,675,676,679,681,683,684,685,687,696,698,701,702,703,704,706,710,722,723,724,725,727,728,729,730,732,733,734,735,736,737,]
-    n_pv_systems_per_example: 128
-    get_center: false
-    is_live: false
-
-  satellite:
-    satellite_zarr_path: "" # Left empty to avoid using satellite data
-    history_minutes: 60
-    forecast_minutes: 0
-    live_delay_minutes: 30
-    time_resolution_minutes: 5
-    satellite_channels:
-      - IR_016
-      - IR_039
-      - IR_087
-      - IR_097
-      - IR_108
-      - IR_120
-      - IR_134
-      - VIS006
-      - VIS008
-      - WV_062
-      - WV_073
-    satellite_image_size_pixels_height: 24
-    satellite_image_size_pixels_width: 24
+```bash
+pip install -r requirements.txt requirements-dev.txt
 ```
 
+Then exit this environment, and enter back into the pvnet conda environment and install ocf_datapies in editable mode (-e). This means the package is directly linked to the source code in the ocf_datapies repo.
+
+```bash
+pip install -e <PATH-TO-ocf_datapipes-REPO>
+```
+
+## Generating pre-made batches of data for training/validation of PVNet
+
+PVNet contains a script for generating batches of data suitable for training the PVNet models. To run the script you will need to make some modifications to the datamodule configuration.
+
+Make sure you have copied the example configs (as already stated above):
+```
+cp -r configs.example configs
+```
+
+### Set up and config example for batch creation
+
+We will use the example of creating batches using data from gcp:
+`/PVNet/configs/datamodule/configuration/gcp_configuration.yaml`
+Ensure that the file paths are set to the correct locations in
+`gcp_configuration.yaml`.
+
+`PLACEHOLDER` is used to indcate where to input the location of the files.
+
+For OCF use cases, file locations can be found in `template_configuration.yaml` located alongside `gcp_configuration.yaml`.
+
+In these configurations you can update the train, val & test periods to cover the data you have access to.
+
+
 With your configuration in place, you can proceed to create batches. PVNet uses
 [hydra](https://hydra.cc/) which enables us to pass variables via the command
 line that will override the configuration defined in the `./configs` directory.
 
-Run the save_batches.py script to create batches with the following arguments as
-a minimum:
+When creating batches, an additional config is used which is passed into the batch creation script. This is the datamodule config located `PVNet/configs/datamodule`.
+
+For this example we will be using the `streamed_batches.yaml` config. Like before, a placeholder variable is used when specifing which configuration to use:
+
+`configuration: "PLACEHOLDER.yaml"`
+
+This should be given the whole path to the config on your local machine, such as for our example it should be changed to:
+
+`configuration: "/FULL-PATH-TO-REPO/PVNet/configs/datamodule/configuration/gcp_configuration.yaml"
+`
+
+Where `FULL-PATH-TO-REPO` represent the whole path to the PVNet repo on your local machine.
+
+### Running the batch creation script
+
+Run the save_batches.py script to create batches with the following example arguments as:
 
 ```
-python scripts/save_batches.py datamodule=local_ocf_datapipes +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
+python scripts/save_batches.py datamodule=streamed_batches +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
 ```
 
+In this function the datamodule argument looks for a config under `PVNet/configs/datamodule`. The examples here are either to use "premade_batches" or "streamed_batches".
+
+Its important that the dates set for the training, validation and testing in the datamodule (`streamed_batches.yaml`) config are within the ranges of the dates set for the input features in the configuration (`gcp_configuration.yaml`).
+
+If downloading private data from a gcp bucket make sure to authenticate gcloud (the public satellite data does not need authentication):
+
+```
+gcloud auth login
+```
+
+For files stored in multiple locations they can be added as list. For example from the gcp_configuration.yaml file we can change from satellite data stored on a bucket:
+
+```
+satellite:
+    satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr
+```
+
+To satellite data hosted by Google:
+
+```
+satellite:
+    satellite_zarr_paths:
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr"
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2021_nonhrv.zarr"
+```
+Datapipes is currently set up to use 11 channels from the satellite data, the 12th of which is HRV and is not included in these.
+
+
 ### Training PVNet
 
 How PVNet is run is determined by the extensive configuration in the config

{PVNet-2.6.8 → PVNet-2.6.10}/PVNet.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PVNet
-Version: 2.6.8
+Version: 2.6.10
 Summary: PVNet
 Author: Peter Dudfield
 Author-email: info@openclimatefix.org
@@ -32,6 +32,7 @@ Requires-Dist: hydra-core
 Requires-Dist: python-dotenv
 Requires-Dist: hydra-optuna-sweeper
 Requires-Dist: rich
+Requires-Dist: gcsfs
 Provides-Extra: all-models
 Requires-Dist: pytorch-tabnet; extra == "all-models"
 Requires-Dist: efficientnet_pytorch; extra == "all-models"
@@ -66,12 +67,26 @@ cd PVNet
 pip install -r requirements.txt
 ```
 
+The commit history is extensive. To save download time, use a depth of 1:
+```bash
+git clone --depth 1 https://github.com/openclimatefix/PVNet.git
+```
+This means only the latest commit and its associated files will be downloaded.
+
+Next, in the PVNet repo, install PVNet as an editable package:
+
+```bash
+pip install -e .
+```
+
 ### Additional development dependencies
 
 ```bash
 pip install -r requirements-dev.txt
 ```
 
+
+
 ## Getting started with running PVNet
 
 Before running any code in within PVNet, copy the example configuration to a
@@ -107,116 +122,103 @@ https://huggingface.co/datasets/openclimatefix/dwd-icon-eu which includes the UK
 OCF maintains a dataset of PV generation from 1311 private PV installations
 here: https://huggingface.co/datasets/openclimatefix/uk_pv
 
-### Generating pre-made batches of data for training/validation of PVNet
 
-PVNet contains a script for generating batches of data suitable for training the
-PVNet models.
+### Connecting with ocf_datapipes for batch creation
 
-To run the script you will need to make some modifications to the datamodule
-configuration.
+Outside the PVNet repo, clone the ocf-datapipes repo and exit the conda env created for PVNet: https://github.com/openclimatefix/ocf_datapipes
+```bash
+git clone --depth 1 https://github.com/openclimatefix/ocf_datapipes.git
+conda create -n ocf_datapipes python=3.10
+```
 
-1. First, create your new configuration file in
-   `./configs/datamodule/configiration/local_configuration.yaml` and paste the
-   sample config (shown below)
-2. Duplicate the `./configs/datamodule/ocf_datapipes.yaml` to
-   `./configs/datamodule/_local_ocf_datapipes.yaml` and ensure the
-   `configuration` key points to your newly created configuration file in
-   step 1.
-3. Also in this file, update the train, val & test periods to cover the data you
-   have access to.
-4. To get you started with your own configuration file, see the sample config
-   below. Update the data paths to the location of your local GSP, NWP and PV
-   datasets:
+Then go inside the ocf_datapipes repo to add packages
 
-```yaml
-general:
-  description: Demo config
-  name: demo_datamodule_config
-
-input_data:
-  default_history_minutes: 60
-  default_forecast_minutes: 120
-
-  gsp:
-    gsp_zarr_path: /path/to/gsp-data.zarr
-    history_minutes: 60
-    forecast_minutes: 120
-    time_resolution_minutes: 30
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    metadata_only: false
-
-  nwp:
-    ukv:
-      nwp_zarr_path: /path/to/nwp-data.zarr
-      history_minutes: 60
-      forecast_minutes: 120
-      time_resolution_minutes: 60
-      nwp_channels: # comment out channels as appropriate
-        - t         # 2-metre temperature
-        - dswrf     # downwards short-wave radiation flux
-        - dlwrf     # downwards long-wave radiation flux
-        - hcc       # high cloud cover
-        - mcc       # medium cloud cover
-        - lcc       # low cloud cover
-        - vis       # visability
-        - r         # relative humidity
-        - prate     # precipitation rate
-        - si10      # 10-metre wind speed | live = unknown
-      nwp_image_size_pixels_height: 24
-      nwp_image_size_pixels_width: 24
-      nwp_provider: ukv
-
-  pv:
-    pv_files_groups:
-      - label: pvoutput.org
-        pv_filename: /path/to/pv-data/pv.netcdf
-        pv_metadata_filename: /path/to/pv-data/metadata.csv
-    history_minutes: 60
-    forecast_minutes: 0 # PVNet assumes no future PV generation
-    time_resolution_minutes: 5
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    pv_image_size_meters_height: 24
-    pv_image_size_meters_width: 24
-    pv_ml_ids: [154,155,156,158,159,160,162,164,165,166,167,168,169,171,173,177,178,179,181,182,185,186,187,188,189,190,191,192,193,197,198,199,200,202,204,205,206,208,209,211,214,215,216,217,218,219,220,221,225,229,230,232,233,234,236,242,243,245,252,254,255,256,257,258,260,261,262,265,267,268,272,273,275,276,277,280,281,282,283,287,289,291,292,293,294,295,296,297,298,301,302,303,304,306,307,309,310,311,317,318,319,320,321,322,323,325,326,329,332,333,335,336,338,340,342,344,345,346,348,349,352,354,355,356,357,360,362,363,368,369,370,371,372,374,375,376,378,380,382,384,385,388,390,391,393,396,397,398,399,400,401,403,404,405,406,407,409,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,429,431,435,437,438,440,441,444,447,450,451,453,456,457,458,459,464,465,466,467,468,470,471,473,474,476,477,479,480,481,482,485,486,488,490,491,492,493,496,498,501,503,506,507,508,509,510,511,512,513,515,516,517,519,520,521,522,524,526,527,528,531,532,536,537,538,540,541,542,543,544,545,549,550,551,552,553,554,556,557,560,561,563,566,568,571,572,575,576,577,579,580,581,582,584,585,588,590,594,595,597,600,602,603,604,606,611,613,614,616,618,620,622,623,624,625,626,628,629,630,631,636,637,638,640,641,642,644,645,646,650,651,652,653,654,655,657,660,661,662,663,666,667,668,670,675,676,679,681,683,684,685,687,696,698,701,702,703,704,706,710,722,723,724,725,727,728,729,730,732,733,734,735,736,737,]
-    n_pv_systems_per_example: 128
-    get_center: false
-    is_live: false
-
-  satellite:
-    satellite_zarr_path: "" # Left empty to avoid using satellite data
-    history_minutes: 60
-    forecast_minutes: 0
-    live_delay_minutes: 30
-    time_resolution_minutes: 5
-    satellite_channels:
-      - IR_016
-      - IR_039
-      - IR_087
-      - IR_097
-      - IR_108
-      - IR_120
-      - IR_134
-      - VIS006
-      - VIS008
-      - WV_062
-      - WV_073
-    satellite_image_size_pixels_height: 24
-    satellite_image_size_pixels_width: 24
+```bash
+pip install -r requirements.txt requirements-dev.txt
 ```
 
+Then exit this environment, and enter back into the pvnet conda environment and install ocf_datapies in editable mode (-e). This means the package is directly linked to the source code in the ocf_datapies repo.
+
+```bash
+pip install -e <PATH-TO-ocf_datapipes-REPO>
+```
+
+## Generating pre-made batches of data for training/validation of PVNet
+
+PVNet contains a script for generating batches of data suitable for training the PVNet models. To run the script you will need to make some modifications to the datamodule configuration.
+
+Make sure you have copied the example configs (as already stated above):
+```
+cp -r configs.example configs
+```
+
+### Set up and config example for batch creation
+
+We will use the example of creating batches using data from gcp:
+`/PVNet/configs/datamodule/configuration/gcp_configuration.yaml`
+Ensure that the file paths are set to the correct locations in
+`gcp_configuration.yaml`.
+
+`PLACEHOLDER` is used to indcate where to input the location of the files.
+
+For OCF use cases, file locations can be found in `template_configuration.yaml` located alongside `gcp_configuration.yaml`.
+
+In these configurations you can update the train, val & test periods to cover the data you have access to.
+
+
 With your configuration in place, you can proceed to create batches. PVNet uses
 [hydra](https://hydra.cc/) which enables us to pass variables via the command
 line that will override the configuration defined in the `./configs` directory.
 
-Run the save_batches.py script to create batches with the following arguments as
-a minimum:
+When creating batches, an additional config is used which is passed into the batch creation script. This is the datamodule config located `PVNet/configs/datamodule`.
+
+For this example we will be using the `streamed_batches.yaml` config. Like before, a placeholder variable is used when specifing which configuration to use:
+
+`configuration: "PLACEHOLDER.yaml"`
+
+This should be given the whole path to the config on your local machine, such as for our example it should be changed to:
+
+`configuration: "/FULL-PATH-TO-REPO/PVNet/configs/datamodule/configuration/gcp_configuration.yaml"
+`
+
+Where `FULL-PATH-TO-REPO` represent the whole path to the PVNet repo on your local machine.
+
+### Running the batch creation script
+
+Run the save_batches.py script to create batches with the following example arguments as:
 
 ```
-python scripts/save_batches.py datamodule=local_ocf_datapipes +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
+python scripts/save_batches.py datamodule=streamed_batches +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
 ```
 
+In this function the datamodule argument looks for a config under `PVNet/configs/datamodule`. The examples here are either to use "premade_batches" or "streamed_batches".
+
+Its important that the dates set for the training, validation and testing in the datamodule (`streamed_batches.yaml`) config are within the ranges of the dates set for the input features in the configuration (`gcp_configuration.yaml`).
+
+If downloading private data from a gcp bucket make sure to authenticate gcloud (the public satellite data does not need authentication):
+
+```
+gcloud auth login
+```
+
+For files stored in multiple locations they can be added as list. For example from the gcp_configuration.yaml file we can change from satellite data stored on a bucket:
+
+```
+satellite:
+    satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr
+```
+
+To satellite data hosted by Google:
+
+```
+satellite:
+    satellite_zarr_paths:
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr"
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2021_nonhrv.zarr"
+```
+Datapipes is currently set up to use 11 channels from the satellite data, the 12th of which is HRV and is not included in these.
+
+
 ### Training PVNet
 
 How PVNet is run is determined by the extensive configuration in the config

{PVNet-2.6.8 → PVNet-2.6.10}/PVNet.egg-info/requires.txt

@@ -23,6 +23,7 @@ hydra-core
 python-dotenv
 hydra-optuna-sweeper
 rich
+gcsfs
 
 [all_models]
 pytorch-tabnet

{PVNet-2.6.8 → PVNet-2.6.10}/README.md

@@ -28,12 +28,26 @@ cd PVNet
 pip install -r requirements.txt
 ```
 
+The commit history is extensive. To save download time, use a depth of 1:
+```bash
+git clone --depth 1 https://github.com/openclimatefix/PVNet.git
+```
+This means only the latest commit and its associated files will be downloaded.
+
+Next, in the PVNet repo, install PVNet as an editable package:
+
+```bash
+pip install -e .
+```
+
 ### Additional development dependencies
 
 ```bash
 pip install -r requirements-dev.txt
 ```
 
+
+
 ## Getting started with running PVNet
 
 Before running any code in within PVNet, copy the example configuration to a
@@ -69,116 +83,103 @@ https://huggingface.co/datasets/openclimatefix/dwd-icon-eu which includes the UK
 OCF maintains a dataset of PV generation from 1311 private PV installations
 here: https://huggingface.co/datasets/openclimatefix/uk_pv
 
-### Generating pre-made batches of data for training/validation of PVNet
 
-PVNet contains a script for generating batches of data suitable for training the
-PVNet models.
+### Connecting with ocf_datapipes for batch creation
 
-To run the script you will need to make some modifications to the datamodule
-configuration.
+Outside the PVNet repo, clone the ocf-datapipes repo and exit the conda env created for PVNet: https://github.com/openclimatefix/ocf_datapipes
+```bash
+git clone --depth 1 https://github.com/openclimatefix/ocf_datapipes.git
+conda create -n ocf_datapipes python=3.10
+```
 
-1. First, create your new configuration file in
-   `./configs/datamodule/configiration/local_configuration.yaml` and paste the
-   sample config (shown below)
-2. Duplicate the `./configs/datamodule/ocf_datapipes.yaml` to
-   `./configs/datamodule/_local_ocf_datapipes.yaml` and ensure the
-   `configuration` key points to your newly created configuration file in
-   step 1.
-3. Also in this file, update the train, val & test periods to cover the data you
-   have access to.
-4. To get you started with your own configuration file, see the sample config
-   below. Update the data paths to the location of your local GSP, NWP and PV
-   datasets:
+Then go inside the ocf_datapipes repo to add packages
 
-```yaml
-general:
-  description: Demo config
-  name: demo_datamodule_config
-
-input_data:
-  default_history_minutes: 60
-  default_forecast_minutes: 120
-
-  gsp:
-    gsp_zarr_path: /path/to/gsp-data.zarr
-    history_minutes: 60
-    forecast_minutes: 120
-    time_resolution_minutes: 30
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    metadata_only: false
-
-  nwp:
-    ukv:
-      nwp_zarr_path: /path/to/nwp-data.zarr
-      history_minutes: 60
-      forecast_minutes: 120
-      time_resolution_minutes: 60
-      nwp_channels: # comment out channels as appropriate
-        - t         # 2-metre temperature
-        - dswrf     # downwards short-wave radiation flux
-        - dlwrf     # downwards long-wave radiation flux
-        - hcc       # high cloud cover
-        - mcc       # medium cloud cover
-        - lcc       # low cloud cover
-        - vis       # visability
-        - r         # relative humidity
-        - prate     # precipitation rate
-        - si10      # 10-metre wind speed | live = unknown
-      nwp_image_size_pixels_height: 24
-      nwp_image_size_pixels_width: 24
-      nwp_provider: ukv
-
-  pv:
-    pv_files_groups:
-      - label: pvoutput.org
-        pv_filename: /path/to/pv-data/pv.netcdf
-        pv_metadata_filename: /path/to/pv-data/metadata.csv
-    history_minutes: 60
-    forecast_minutes: 0 # PVNet assumes no future PV generation
-    time_resolution_minutes: 5
-    start_datetime: "2019-01-01T00:00:00"
-    end_datetime: "2019-01-08T00:00:00"
-    pv_image_size_meters_height: 24
-    pv_image_size_meters_width: 24
-    pv_ml_ids: [154,155,156,158,159,160,162,164,165,166,167,168,169,171,173,177,178,179,181,182,185,186,187,188,189,190,191,192,193,197,198,199,200,202,204,205,206,208,209,211,214,215,216,217,218,219,220,221,225,229,230,232,233,234,236,242,243,245,252,254,255,256,257,258,260,261,262,265,267,268,272,273,275,276,277,280,281,282,283,287,289,291,292,293,294,295,296,297,298,301,302,303,304,306,307,309,310,311,317,318,319,320,321,322,323,325,326,329,332,333,335,336,338,340,342,344,345,346,348,349,352,354,355,356,357,360,362,363,368,369,370,371,372,374,375,376,378,380,382,384,385,388,390,391,393,396,397,398,399,400,401,403,404,405,406,407,409,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,429,431,435,437,438,440,441,444,447,450,451,453,456,457,458,459,464,465,466,467,468,470,471,473,474,476,477,479,480,481,482,485,486,488,490,491,492,493,496,498,501,503,506,507,508,509,510,511,512,513,515,516,517,519,520,521,522,524,526,527,528,531,532,536,537,538,540,541,542,543,544,545,549,550,551,552,553,554,556,557,560,561,563,566,568,571,572,575,576,577,579,580,581,582,584,585,588,590,594,595,597,600,602,603,604,606,611,613,614,616,618,620,622,623,624,625,626,628,629,630,631,636,637,638,640,641,642,644,645,646,650,651,652,653,654,655,657,660,661,662,663,666,667,668,670,675,676,679,681,683,684,685,687,696,698,701,702,703,704,706,710,722,723,724,725,727,728,729,730,732,733,734,735,736,737,]
-    n_pv_systems_per_example: 128
-    get_center: false
-    is_live: false
-
-  satellite:
-    satellite_zarr_path: "" # Left empty to avoid using satellite data
-    history_minutes: 60
-    forecast_minutes: 0
-    live_delay_minutes: 30
-    time_resolution_minutes: 5
-    satellite_channels:
-      - IR_016
-      - IR_039
-      - IR_087
-      - IR_097
-      - IR_108
-      - IR_120
-      - IR_134
-      - VIS006
-      - VIS008
-      - WV_062
-      - WV_073
-    satellite_image_size_pixels_height: 24
-    satellite_image_size_pixels_width: 24
+```bash
+pip install -r requirements.txt requirements-dev.txt
 ```
 
+Then exit this environment, and enter back into the pvnet conda environment and install ocf_datapies in editable mode (-e). This means the package is directly linked to the source code in the ocf_datapies repo.
+
+```bash
+pip install -e <PATH-TO-ocf_datapipes-REPO>
+```
+
+## Generating pre-made batches of data for training/validation of PVNet
+
+PVNet contains a script for generating batches of data suitable for training the PVNet models. To run the script you will need to make some modifications to the datamodule configuration.
+
+Make sure you have copied the example configs (as already stated above):
+```
+cp -r configs.example configs
+```
+
+### Set up and config example for batch creation
+
+We will use the example of creating batches using data from gcp:
+`/PVNet/configs/datamodule/configuration/gcp_configuration.yaml`
+Ensure that the file paths are set to the correct locations in
+`gcp_configuration.yaml`.
+
+`PLACEHOLDER` is used to indcate where to input the location of the files.
+
+For OCF use cases, file locations can be found in `template_configuration.yaml` located alongside `gcp_configuration.yaml`.
+
+In these configurations you can update the train, val & test periods to cover the data you have access to.
+
+
 With your configuration in place, you can proceed to create batches. PVNet uses
 [hydra](https://hydra.cc/) which enables us to pass variables via the command
 line that will override the configuration defined in the `./configs` directory.
 
-Run the save_batches.py script to create batches with the following arguments as
-a minimum:
+When creating batches, an additional config is used which is passed into the batch creation script. This is the datamodule config located `PVNet/configs/datamodule`.
+
+For this example we will be using the `streamed_batches.yaml` config. Like before, a placeholder variable is used when specifing which configuration to use:
+
+`configuration: "PLACEHOLDER.yaml"`
+
+This should be given the whole path to the config on your local machine, such as for our example it should be changed to:
+
+`configuration: "/FULL-PATH-TO-REPO/PVNet/configs/datamodule/configuration/gcp_configuration.yaml"
+`
+
+Where `FULL-PATH-TO-REPO` represent the whole path to the PVNet repo on your local machine.
+
+### Running the batch creation script
+
+Run the save_batches.py script to create batches with the following example arguments as:
 
 ```
-python scripts/save_batches.py datamodule=local_ocf_datapipes +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
+python scripts/save_batches.py datamodule=streamed_batches +batch_output_dir="./output" +num_train_batches=10 +num_val_batches=5
 ```
 
+In this function the datamodule argument looks for a config under `PVNet/configs/datamodule`. The examples here are either to use "premade_batches" or "streamed_batches".
+
+Its important that the dates set for the training, validation and testing in the datamodule (`streamed_batches.yaml`) config are within the ranges of the dates set for the input features in the configuration (`gcp_configuration.yaml`).
+
+If downloading private data from a gcp bucket make sure to authenticate gcloud (the public satellite data does not need authentication):
+
+```
+gcloud auth login
+```
+
+For files stored in multiple locations they can be added as list. For example from the gcp_configuration.yaml file we can change from satellite data stored on a bucket:
+
+```
+satellite:
+    satellite_zarr_path: gs://solar-pv-nowcasting-data/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr
+```
+
+To satellite data hosted by Google:
+
+```
+satellite:
+    satellite_zarr_paths:
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2020_nonhrv.zarr"
+      - "gs://public-datasets-eumetsat-solar-forecasting/satellite/EUMETSAT/SEVIRI_RSS/v4/2021_nonhrv.zarr"
+```
+Datapipes is currently set up to use 11 channels from the satellite data, the 12th of which is HRV and is not included in these.
+
+
 ### Training PVNet
 
 How PVNet is run is determined by the extensive configuration in the config

PVNet-2.6.10/pvnet/__init__.py

@@ -0,0 +1,2 @@
+"""PVNet"""
+__version__ = "2.6.10"

{PVNet-2.6.8 → PVNet-2.6.10}/pvnet/data/datamodule.py

@@ -23,7 +23,6 @@ class DataModule(LightningDataModule):
         train_period=[None, None],
         val_period=[None, None],
         test_period=[None, None],
-        block_nwp_and_sat=False,
         batch_dir=None,
     ):
         """Datamodule for training pvnet and using pvnet pipeline in `ocf_datapipes`.
@@ -39,8 +38,6 @@ class DataModule(LightningDataModule):
             train_period: Date range filter for train dataloader.
             val_period: Date range filter for val dataloader.
             test_period: Date range filter for test dataloader.
-            block_nwp_and_sat: If True, the dataloader does not load the requested NWP and sat data.
-                It instead returns an zero-array of the required shape. Useful for pretraining.
             batch_dir: Path to the directory of pre-saved batches. Cannot be used together with
                 `configuration` or 'train/val/test_period'.
 
@@ -48,7 +45,6 @@ class DataModule(LightningDataModule):
         super().__init__()
         self.configuration = configuration
         self.batch_size = batch_size
-        self.block_nwp_and_sat = block_nwp_and_sat
         self.batch_dir = batch_dir
 
         if not ((batch_dir is not None) ^ (configuration is not None)):
@@ -69,7 +65,6 @@ class DataModule(LightningDataModule):
         ]
 
         self._common_dataloader_kwargs = dict(
-            shuffle=False,  # shuffled in datapipe step
             batch_size=None,  # batched in datapipe step
             sampler=None,
             batch_sampler=None,
@@ -88,8 +83,6 @@ class DataModule(LightningDataModule):
             self.configuration,
             start_time=start_time,
             end_time=end_time,
-            block_sat=self.block_nwp_and_sat,
-            block_nwp=self.block_nwp_and_sat,
         )
 
         data_pipeline = (
@@ -131,7 +124,7 @@ class DataModule(LightningDataModule):
             datapipe = self._get_premade_batches_datapipe("train", shuffle=True)
         else:
             datapipe = self._get_datapipe(*self.train_period)
-        return DataLoader(datapipe, **self._common_dataloader_kwargs)
+        return DataLoader(datapipe, shuffle=True, **self._common_dataloader_kwargs)
 
     def val_dataloader(self):
         """Construct val dataloader"""
@@ -139,7 +132,7 @@ class DataModule(LightningDataModule):
             datapipe = self._get_premade_batches_datapipe("val")
         else:
             datapipe = self._get_datapipe(*self.val_period)
-        return DataLoader(datapipe, **self._common_dataloader_kwargs)
+        return DataLoader(datapipe, shuffle=False, **self._common_dataloader_kwargs)
 
     def test_dataloader(self):
         """Construct test dataloader"""
@@ -147,4 +140,4 @@ class DataModule(LightningDataModule):
             datapipe = self._get_premade_batches_datapipe("test")
         else:
             datapipe = self._get_datapipe(*self.test_period)
-        return DataLoader(datapipe, **self._common_dataloader_kwargs)
+        return DataLoader(datapipe, shuffle=False, **self._common_dataloader_kwargs)

{PVNet-2.6.8 → PVNet-2.6.10}/pvnet/data/wind_datamodule.py

@@ -61,7 +61,6 @@ class WindDataModule(LightningDataModule):
         ]
 
         self._common_dataloader_kwargs = dict(
-            shuffle=False,  # shuffled in datapipe step
             batch_size=None,  # batched in datapipe step
             sampler=None,
             batch_sampler=None,
@@ -123,7 +122,7 @@ class WindDataModule(LightningDataModule):
             datapipe = self._get_premade_batches_datapipe("train", shuffle=True)
         else:
             datapipe = self._get_datapipe(*self.train_period)
-        return DataLoader(datapipe, **self._common_dataloader_kwargs)
+        return DataLoader(datapipe, shuffle=True, **self._common_dataloader_kwargs)
 
     def val_dataloader(self):
         """Construct val dataloader"""
@@ -131,7 +130,7 @@ class WindDataModule(LightningDataModule):
             datapipe = self._get_premade_batches_datapipe("val")
         else:
             datapipe = self._get_datapipe(*self.val_period)
-        return DataLoader(datapipe, **self._common_dataloader_kwargs)
+        return DataLoader(datapipe, shuffle=False, **self._common_dataloader_kwargs)
 
     def test_dataloader(self):
         """Construct test dataloader"""
@@ -139,4 +138,4 @@ class WindDataModule(LightningDataModule):
             datapipe = self._get_premade_batches_datapipe("test")
         else:
             datapipe = self._get_datapipe(*self.test_period)
-        return DataLoader(datapipe, **self._common_dataloader_kwargs)
+        return DataLoader(datapipe, shuffle=False, **self._common_dataloader_kwargs)

{PVNet-2.6.8 → PVNet-2.6.10}/pvnet/models/base_model.py

@@ -234,6 +234,7 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
         forecast_minutes: int,
         optimizer: AbstractOptimizer,
         output_quantiles: Optional[list[float]] = None,
+        target_key: BatchKey = BatchKey.gsp,
     ):
         """Abtstract base class for PVNet submodels.
 
@@ -243,10 +244,12 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
             optimizer (AbstractOptimizer): Optimizer
             output_quantiles: A list of float (0.0, 1.0) quantiles to predict values for. If set to
                 None the output is a single value.
+            target_key: The key of the target variable in the batch
         """
         super().__init__()
 
         self._optimizer = optimizer
+        self._target_key = target_key
 
         # Model must have lr to allow tuning
         # This setting is only used when lr is tuned with callback
@@ -424,7 +427,7 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
     def training_step(self, batch, batch_idx):
         """Run training step"""
         y_hat = self(batch)
-        y = batch[BatchKey.gsp][:, -self.forecast_len_30 :, 0]
+        y = batch[self._target_key][:, -self.forecast_len_30 :, 0]
 
         losses = self._calculate_common_losses(y, y_hat)
         losses = {f"{k}/train": v for k, v in losses.items()}
@@ -440,7 +443,7 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
     def validation_step(self, batch: dict, batch_idx):
         """Run validation step"""
         y_hat = self(batch)
-        y = batch[BatchKey.gsp][:, -self.forecast_len_30 :, 0]
+        y = batch[self._target_key][:, -self.forecast_len_30 :, 0]
 
         losses = self._calculate_common_losses(y, y_hat)
         losses.update(self._calculate_val_losses(y, y_hat))
@@ -484,7 +487,7 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
     def test_step(self, batch, batch_idx):
         """Run test step"""
         y_hat = self(batch)
-        y = batch[BatchKey.gsp][:, -self.forecast_len_30 :, 0]
+        y = batch[self._target_key][:, -self.forecast_len_30 :, 0]
 
         losses = self._calculate_common_losses(y, y_hat)
         losses.update(self._calculate_val_losses(y, y_hat))

{PVNet-2.6.8 → PVNet-2.6.10}/pvnet/models/multimodal/multimodal.py

@@ -54,6 +54,7 @@ class Model(BaseModel):
         nwp_history_minutes: Optional[int] = None,
         pv_history_minutes: Optional[int] = None,
         optimizer: AbstractOptimizer = pvnet.optimizers.Adam(),
+        target_key: str = "gsp",
     ):
         """Neural network which combines information from different sources.
 
@@ -94,6 +95,7 @@ class Model(BaseModel):
             pv_history_minutes: Length of recent site-level PV data data used as input. Defaults to
                 `history_minutes` if not provided.
             optimizer: Optimizer factory function used for network.
+            target_key: The key of the target variable in the batch.
         """
 
         self.include_gsp_yield_history = include_gsp_yield_history
@@ -103,8 +105,15 @@ class Model(BaseModel):
         self.include_sun = include_sun
         self.embedding_dim = embedding_dim
         self.add_image_embedding_channel = add_image_embedding_channel
-
-        super().__init__(history_minutes, forecast_minutes, optimizer, output_quantiles)
+        self.target_key_name = target_key
+
+        super().__init__(
+            history_minutes=history_minutes,
+            forecast_minutes=forecast_minutes,
+            optimizer=optimizer,
+            output_quantiles=output_quantiles,
+            target_key=BatchKey.gsp if target_key == "gsp" else BatchKey.pv,
+        )
 
         # Number of features expected by the output_network
         # Add to this as network pices are constructed
@@ -228,7 +237,12 @@ class Model(BaseModel):
         # *********************** PV Data *************************************
         # Add site-level PV yield
         if self.include_pv:
-            modes["pv"] = self.pv_encoder(x)
+            if self.target_key_name != "pv":
+                modes["pv"] = self.pv_encoder(x)
+            else:
+                # Target is PV, so only take the history
+                pv_history = x[BatchKey.pv][:, : self.history_len_30].float()
+                modes["pv"] = self.pv_encoder(pv_history)
 
         # *********************** GSP Data ************************************
         # add gsp yield history

{PVNet-2.6.8 → PVNet-2.6.10}/pvnet/models/utils.py

@@ -27,7 +27,7 @@ class PredAccumulator:
 
     def append(self, y_hat: torch.Tensor):
         """Append a sub-batch of predictions"""
-        self._y_hats += [y_hat]
+        self._y_hats.append(y_hat)
 
     def flush(self) -> torch.Tensor:
         """Return all appended predictions as single tensor and remove from accumulated store."""
@@ -42,7 +42,7 @@ class DictListAccumulator:
     @staticmethod
     def _dict_list_append(d1, d2):
         for k, v in d2.items():
-            d1[k] += [v]
+            d1[k].append(v)
 
     @staticmethod
     def _dict_init_list(d):

{PVNet-2.6.8 → PVNet-2.6.10}/pvnet/training.py

@@ -139,12 +139,16 @@ def train(config: DictConfig) -> Optional[float]:
 
     if should_pretrain:
         # Pre-train the model
-        datamodule.block_nwp_and_sat = True
+        raise NotImplementedError("Pre-training is not yet supported")
+        # The parameter `block_nwp_and_sat` has been removed from datapipes
+        # If pretraining is re-supported in the future it is likely any pre-training logic should
+        # go here or perhaps in the callbacks
+        # datamodule.block_nwp_and_sat = True
+
         trainer.fit(model=model, datamodule=datamodule)
 
     _callbacks_to_phase(callbacks, "main")
 
-    datamodule.block_nwp_and_sat = False
     trainer.should_stop = False
 
     # Train the model completely

{PVNet-2.6.8 → PVNet-2.6.10}/pvnet/utils.py

@@ -242,22 +242,26 @@ def finish(
             wandb.finish()
 
 
-def plot_batch_forecasts(batch, y_hat, batch_idx=None, quantiles=None):
+def plot_batch_forecasts(batch, y_hat, batch_idx=None, quantiles=None, key_to_plot: str = "gsp"):
     """Plot a batch of data and the forecast from that batch"""
 
     def _get_numpy(key):
         return batch[key].cpu().numpy().squeeze()
 
-    y = batch[BatchKey.gsp].cpu().numpy()
+    y_key = BatchKey.gsp if key_to_plot == "gsp" else BatchKey.pv
+    y_id_key = BatchKey.gsp_id if key_to_plot == "gsp" else BatchKey.pv_id
+    t0_idx_key = BatchKey.gsp_t0_idx if key_to_plot == "gsp" else BatchKey.pv_t0_idx
+    time_utc_key = BatchKey.gsp_time_utc if key_to_plot == "gsp" else BatchKey.pv_time_utc
+    plotting_name = key_to_plot.upper()
+    y = batch[y_key].cpu().numpy()
     y_hat = y_hat.cpu().numpy()
 
-    gsp_ids = batch[BatchKey.gsp_id].cpu().numpy().squeeze()
-    t0_idx = batch[BatchKey.gsp_t0_idx]
+    gsp_ids = batch[y_id_key].cpu().numpy().squeeze()
+    batch[t0_idx_key]
 
-    times_utc = batch[BatchKey.gsp_time_utc].cpu().numpy().squeeze().astype("datetime64[s]")
+    times_utc = batch[time_utc_key].cpu().numpy().squeeze().astype("datetime64[s]")
     times_utc = [pd.to_datetime(t) for t in times_utc]
 
-    len(times_utc[0]) - t0_idx - 1
     batch_size = y.shape[0]
 
     fig, axes = plt.subplots(4, 4, figsize=(8, 8))
@@ -295,9 +299,9 @@ def plot_batch_forecasts(batch, y_hat, batch_idx=None, quantiles=None):
         ax.set_xlabel("Time (hour of day)")
 
     if batch_idx is not None:
-        title = f"Normed GSP output : batch_idx={batch_idx}"
+        title = f"Normed {plotting_name} output : batch_idx={batch_idx}"
     else:
-        title = "Normed GSP output"
+        title = f"Normed {plotting_name} output"
     plt.suptitle(title)
     plt.tight_layout()
 

{PVNet-2.6.8 → PVNet-2.6.10}/requirements.txt

@@ -23,3 +23,5 @@ hydra-core
 python-dotenv
 hydra-optuna-sweeper
 rich
+# gcsfs is only needed when getting data from Google Cloud Storage
+gcsfs

{PVNet-2.6.8 → PVNet-2.6.10}/tests/conftest.py

@@ -123,7 +123,6 @@ def sample_train_val_datamodule():
             train_period=[None, None],
             val_period=[None, None],
             test_period=[None, None],
-            block_nwp_and_sat=False,
             batch_dir=f"{tmpdirname}",
         )
         yield dm
@@ -139,7 +138,6 @@ def sample_datamodule():
         train_period=[None, None],
         val_period=[None, None],
         test_period=[None, None],
-        block_nwp_and_sat=False,
         batch_dir="tests/test_data/sample_batches",
     )
     return dm

PVNet-2.6.8/pvnet/__init__.py

@@ -1,2 +0,0 @@
-"""PVNet"""
-__version__ = "2.6.8"