PVNet 4.1.26__tar.gz → 4.1.28__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pvnet-4.1.26 → pvnet-4.1.28}/PKG-INFO +2 -5
- {pvnet-4.1.26 → pvnet-4.1.28}/PVNet.egg-info/PKG-INFO +2 -5
- {pvnet-4.1.26 → pvnet-4.1.28}/PVNet.egg-info/SOURCES.txt +1 -2
- {pvnet-4.1.26 → pvnet-4.1.28}/PVNet.egg-info/requires.txt +1 -5
- pvnet-4.1.28/pvnet/__init__.py +2 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/load_model.py +8 -3
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/base_model.py +61 -105
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/multimodal.py +0 -26
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/site_encoders/encoders.py +3 -5
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/utils.py +0 -2
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/utils.py +6 -57
- {pvnet-4.1.26 → pvnet-4.1.28}/pyproject.toml +1 -6
- pvnet-4.1.26/pvnet/__init__.py +0 -2
- pvnet-4.1.26/tests/test_utils.py +0 -38
- {pvnet-4.1.26 → pvnet-4.1.28}/LICENSE +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/MANIFEST.in +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/PVNet.egg-info/dependency_links.txt +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/PVNet.egg-info/top_level.txt +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/README.md +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/data/__init__.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/data/base_datamodule.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/data/site_datamodule.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/data/uk_regional_datamodule.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/__init__.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/ensemble.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/model_cards/pv_india_model_card_template.md +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/model_cards/pv_uk_regional_model_card_template.md +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/model_cards/wind_india_model_card_template.md +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/__init__.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/basic_blocks.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/encoders/__init__.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/encoders/basic_blocks.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/encoders/encoders3d.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/linear_networks/__init__.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/linear_networks/basic_blocks.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/linear_networks/networks.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/site_encoders/__init__.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/site_encoders/basic_blocks.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/multimodal/unimodal_teacher.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/optimizers.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/pvnet/training.py +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/setup.cfg +0 -0
- {pvnet-4.1.26 → pvnet-4.1.28}/tests/test_end2end.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: PVNet
|
|
3
|
-
Version: 4.1.
|
|
3
|
+
Version: 4.1.28
|
|
4
4
|
Summary: PVNet
|
|
5
5
|
Author-email: Peter Dudfield <info@openclimatefix.org>
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -33,11 +33,8 @@ Requires-Dist: mypy; extra == "dev"
|
|
|
33
33
|
Requires-Dist: pre-commit; extra == "dev"
|
|
34
34
|
Requires-Dist: pytest; extra == "dev"
|
|
35
35
|
Requires-Dist: pytest-cov; extra == "dev"
|
|
36
|
-
Provides-Extra: all-models
|
|
37
|
-
Requires-Dist: pytorch-tabnet; extra == "all-models"
|
|
38
|
-
Requires-Dist: efficientnet_pytorch; extra == "all-models"
|
|
39
36
|
Provides-Extra: all
|
|
40
|
-
Requires-Dist: PVNet[
|
|
37
|
+
Requires-Dist: PVNet[dev]; extra == "all"
|
|
41
38
|
Dynamic: license-file
|
|
42
39
|
|
|
43
40
|
# PVNet 2.1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: PVNet
|
|
3
|
-
Version: 4.1.
|
|
3
|
+
Version: 4.1.28
|
|
4
4
|
Summary: PVNet
|
|
5
5
|
Author-email: Peter Dudfield <info@openclimatefix.org>
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -33,11 +33,8 @@ Requires-Dist: mypy; extra == "dev"
|
|
|
33
33
|
Requires-Dist: pre-commit; extra == "dev"
|
|
34
34
|
Requires-Dist: pytest; extra == "dev"
|
|
35
35
|
Requires-Dist: pytest-cov; extra == "dev"
|
|
36
|
-
Provides-Extra: all-models
|
|
37
|
-
Requires-Dist: pytorch-tabnet; extra == "all-models"
|
|
38
|
-
Requires-Dist: efficientnet_pytorch; extra == "all-models"
|
|
39
36
|
Provides-Extra: all
|
|
40
|
-
Requires-Dist: PVNet[
|
|
37
|
+
Requires-Dist: PVNet[dev]; extra == "all"
|
|
41
38
|
Dynamic: license-file
|
|
42
39
|
|
|
43
40
|
# PVNet 2.1
|
|
@@ -36,5 +36,4 @@ pvnet/models/multimodal/linear_networks/networks.py
|
|
|
36
36
|
pvnet/models/multimodal/site_encoders/__init__.py
|
|
37
37
|
pvnet/models/multimodal/site_encoders/basic_blocks.py
|
|
38
38
|
pvnet/models/multimodal/site_encoders/encoders.py
|
|
39
|
-
tests/test_end2end.py
|
|
40
|
-
tests/test_utils.py
|
|
39
|
+
tests/test_end2end.py
|
|
@@ -10,6 +10,11 @@ from pyaml_env import parse_config
|
|
|
10
10
|
|
|
11
11
|
from pvnet.models.ensemble import Ensemble
|
|
12
12
|
from pvnet.models.multimodal.unimodal_teacher import Model as UMTModel
|
|
13
|
+
from pvnet.utils import (
|
|
14
|
+
DATA_CONFIG_NAME,
|
|
15
|
+
DATAMODULE_CONFIG_NAME,
|
|
16
|
+
MODEL_CONFIG_NAME,
|
|
17
|
+
)
|
|
13
18
|
|
|
14
19
|
|
|
15
20
|
def get_model_from_checkpoints(
|
|
@@ -35,7 +40,7 @@ def get_model_from_checkpoints(
|
|
|
35
40
|
|
|
36
41
|
for path in checkpoint_dir_paths:
|
|
37
42
|
# Load the model
|
|
38
|
-
model_config = parse_config(f"{path}/
|
|
43
|
+
model_config = parse_config(f"{path}/{MODEL_CONFIG_NAME}")
|
|
39
44
|
|
|
40
45
|
model = hydra.utils.instantiate(model_config)
|
|
41
46
|
|
|
@@ -60,7 +65,7 @@ def get_model_from_checkpoints(
|
|
|
60
65
|
models.append(model)
|
|
61
66
|
|
|
62
67
|
# Check for data config
|
|
63
|
-
data_config = f"{path}/
|
|
68
|
+
data_config = f"{path}/{DATA_CONFIG_NAME}"
|
|
64
69
|
|
|
65
70
|
if os.path.isfile(data_config):
|
|
66
71
|
data_configs.append(data_config)
|
|
@@ -68,7 +73,7 @@ def get_model_from_checkpoints(
|
|
|
68
73
|
data_configs.append(None)
|
|
69
74
|
|
|
70
75
|
# check for datamodule config
|
|
71
|
-
datamodule_config = f"{path}/
|
|
76
|
+
datamodule_config = f"{path}/{DATAMODULE_CONFIG_NAME}"
|
|
72
77
|
if os.path.isfile(datamodule_config):
|
|
73
78
|
datamodule_configs.append(datamodule_config)
|
|
74
79
|
else:
|
|
@@ -5,7 +5,6 @@ import os
|
|
|
5
5
|
import tempfile
|
|
6
6
|
import time
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import Dict, Optional, Union
|
|
9
8
|
|
|
10
9
|
import hydra
|
|
11
10
|
import lightning.pytorch as pl
|
|
@@ -16,25 +15,22 @@ import torch
|
|
|
16
15
|
import torch.nn.functional as F
|
|
17
16
|
import wandb
|
|
18
17
|
import yaml
|
|
19
|
-
from huggingface_hub import ModelCard, ModelCardData
|
|
20
|
-
from huggingface_hub.constants import PYTORCH_WEIGHTS_NAME
|
|
18
|
+
from huggingface_hub import ModelCard, ModelCardData
|
|
21
19
|
from huggingface_hub.file_download import hf_hub_download
|
|
22
20
|
from huggingface_hub.hf_api import HfApi
|
|
23
21
|
from ocf_data_sampler.torch_datasets.sample.base import copy_batch_to_device
|
|
24
22
|
from torchvision.transforms.functional import center_crop
|
|
25
23
|
|
|
26
|
-
from pvnet.models.utils import
|
|
27
|
-
BatchAccumulator,
|
|
28
|
-
MetricAccumulator,
|
|
29
|
-
PredAccumulator,
|
|
30
|
-
)
|
|
24
|
+
from pvnet.models.utils import BatchAccumulator, MetricAccumulator, PredAccumulator
|
|
31
25
|
from pvnet.optimizers import AbstractOptimizer
|
|
32
|
-
from pvnet.utils import
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
26
|
+
from pvnet.utils import (
|
|
27
|
+
DATA_CONFIG_NAME,
|
|
28
|
+
DATAMODULE_CONFIG_NAME,
|
|
29
|
+
MODEL_CARD_NAME,
|
|
30
|
+
MODEL_CONFIG_NAME,
|
|
31
|
+
PYTORCH_WEIGHTS_NAME,
|
|
32
|
+
plot_batch_forecasts,
|
|
33
|
+
)
|
|
38
34
|
|
|
39
35
|
logger = logging.getLogger(__name__)
|
|
40
36
|
|
|
@@ -57,7 +53,7 @@ def make_clean_data_config(input_path, output_path, placeholder="PLACEHOLDER"):
|
|
|
57
53
|
config["general"]["description"] = "Config for training the saved PVNet model"
|
|
58
54
|
config["general"]["name"] = "PVNet current"
|
|
59
55
|
|
|
60
|
-
for source in ["gsp", "satellite"
|
|
56
|
+
for source in ["gsp", "satellite"]:
|
|
61
57
|
if source in config["input_data"]:
|
|
62
58
|
# If not empty - i.e. if used
|
|
63
59
|
if config["input_data"][source]["zarr_path"] != "":
|
|
@@ -73,11 +69,6 @@ def make_clean_data_config(input_path, output_path, placeholder="PLACEHOLDER"):
|
|
|
73
69
|
d["pv_filename"] = f"{placeholder}.netcdf"
|
|
74
70
|
d["pv_metadata_filename"] = f"{placeholder}.csv"
|
|
75
71
|
|
|
76
|
-
if "sensor" in config["input_data"]:
|
|
77
|
-
# If not empty - i.e. if used
|
|
78
|
-
if config["input_data"][source][f"{source}_filename"] != "":
|
|
79
|
-
config["input_data"][source][f"{source}_filename"] = f"{placeholder}.nc"
|
|
80
|
-
|
|
81
72
|
with open(output_path, "w") as outfile:
|
|
82
73
|
yaml.dump(config, outfile, default_flow_style=False)
|
|
83
74
|
|
|
@@ -211,7 +202,7 @@ def download_hf_hub_with_retries(
|
|
|
211
202
|
time.sleep(wait_time)
|
|
212
203
|
|
|
213
204
|
|
|
214
|
-
class PVNetModelHubMixin
|
|
205
|
+
class PVNetModelHubMixin:
|
|
215
206
|
"""
|
|
216
207
|
Implementation of [`PyTorchModelHubMixin`] to provide model Hub upload/download capabilities.
|
|
217
208
|
"""
|
|
@@ -222,12 +213,12 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
|
|
|
222
213
|
*,
|
|
223
214
|
model_id: str,
|
|
224
215
|
revision: str,
|
|
225
|
-
cache_dir:
|
|
216
|
+
cache_dir: str | None = None,
|
|
226
217
|
force_download: bool = False,
|
|
227
|
-
proxies:
|
|
228
|
-
resume_download:
|
|
218
|
+
proxies: dict | None = None,
|
|
219
|
+
resume_download: bool | None = None,
|
|
229
220
|
local_files_only: bool = False,
|
|
230
|
-
token:
|
|
221
|
+
token: str | bool | None = None,
|
|
231
222
|
map_location: str = "cpu",
|
|
232
223
|
strict: bool = False,
|
|
233
224
|
):
|
|
@@ -284,12 +275,12 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
|
|
|
284
275
|
cls,
|
|
285
276
|
model_id: str,
|
|
286
277
|
revision: str,
|
|
287
|
-
cache_dir:
|
|
278
|
+
cache_dir: str | None = None,
|
|
288
279
|
force_download: bool = False,
|
|
289
|
-
proxies:
|
|
280
|
+
proxies: dict | None = None,
|
|
290
281
|
resume_download: bool = False,
|
|
291
282
|
local_files_only: bool = False,
|
|
292
|
-
token:
|
|
283
|
+
token: str | bool | None = None,
|
|
293
284
|
):
|
|
294
285
|
"""Load data config file."""
|
|
295
286
|
if os.path.isdir(model_id):
|
|
@@ -312,37 +303,35 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
|
|
|
312
303
|
|
|
313
304
|
return data_config_file
|
|
314
305
|
|
|
315
|
-
def
|
|
306
|
+
def _save_model_weights(self, save_directory: str) -> None:
|
|
316
307
|
"""Save weights from a Pytorch model to a local directory."""
|
|
317
|
-
|
|
318
|
-
torch.save(model_to_save.state_dict(), save_directory / PYTORCH_WEIGHTS_NAME)
|
|
308
|
+
torch.save(self.state_dict(), f"{save_directory}/{PYTORCH_WEIGHTS_NAME}")
|
|
319
309
|
|
|
320
310
|
def save_pretrained(
|
|
321
311
|
self,
|
|
322
|
-
save_directory:
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
312
|
+
save_directory: str,
|
|
313
|
+
model_config: dict,
|
|
314
|
+
data_config_path: str,
|
|
315
|
+
wandb_repo: str,
|
|
316
|
+
wandb_ids: list[str] | str,
|
|
317
|
+
card_template_path: str,
|
|
318
|
+
datamodule_config_path: str | None = None,
|
|
319
|
+
hf_repo_id: str | None = None,
|
|
327
320
|
push_to_hub: bool = False,
|
|
328
|
-
|
|
329
|
-
wandb_ids: Optional[Union[list[str], str]] = None,
|
|
330
|
-
card_template_path: Optional[Path] = None,
|
|
331
|
-
**kwargs,
|
|
332
|
-
) -> Optional[str]:
|
|
321
|
+
) -> None:
|
|
333
322
|
"""
|
|
334
323
|
Save weights in local directory.
|
|
335
324
|
|
|
336
325
|
Args:
|
|
337
|
-
save_directory
|
|
326
|
+
save_directory:
|
|
338
327
|
Path to directory in which the model weights and configuration will be saved.
|
|
339
|
-
|
|
328
|
+
model_config (`dict`):
|
|
340
329
|
Model configuration specified as a key/value dictionary.
|
|
341
|
-
|
|
330
|
+
data_config_path:
|
|
342
331
|
The path to the data config.
|
|
343
|
-
|
|
332
|
+
datamodule_config_path:
|
|
344
333
|
The path to the datamodule config.
|
|
345
|
-
|
|
334
|
+
hf_repo_id:
|
|
346
335
|
ID of your repository on the Hub. Used only if `push_to_hub=True`. Will default to
|
|
347
336
|
the folder name if not provided.
|
|
348
337
|
push_to_hub (`bool`, *optional*, defaults to `False`):
|
|
@@ -351,57 +340,42 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
|
|
|
351
340
|
wandb_ids: Identifier(s) of the model on wandb.
|
|
352
341
|
card_template_path: Path to the HuggingFace model card template. Defaults to card in
|
|
353
342
|
PVNet library if set to None.
|
|
354
|
-
kwargs:
|
|
355
|
-
Additional key word arguments passed along to the
|
|
356
|
-
[`~ModelHubMixin._from_pretrained`] method.
|
|
357
343
|
"""
|
|
358
344
|
|
|
359
345
|
save_directory = Path(save_directory)
|
|
360
346
|
save_directory.mkdir(parents=True, exist_ok=True)
|
|
361
347
|
|
|
362
|
-
#
|
|
363
|
-
self.
|
|
348
|
+
# Save model weights/files
|
|
349
|
+
self._save_model_weights(save_directory)
|
|
364
350
|
|
|
365
|
-
#
|
|
366
|
-
if isinstance(
|
|
351
|
+
# Save the model config and data config
|
|
352
|
+
if isinstance(model_config, dict):
|
|
367
353
|
with open(save_directory / MODEL_CONFIG_NAME, "w") as f:
|
|
368
|
-
yaml.dump(
|
|
354
|
+
yaml.dump(model_config, f, sort_keys=False, default_flow_style=False)
|
|
369
355
|
|
|
370
|
-
# Save cleaned configuration file
|
|
371
|
-
|
|
372
|
-
|
|
356
|
+
# Save cleaned input data configuration file
|
|
357
|
+
new_data_config_path = save_directory / DATA_CONFIG_NAME
|
|
358
|
+
make_clean_data_config(data_config_path, new_data_config_path)
|
|
359
|
+
minimize_data_config(new_data_config_path, new_data_config_path, self)
|
|
373
360
|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
# Taylor the data config to the model being saved
|
|
378
|
-
minimize_data_config(new_data_config_path, new_data_config_path, self)
|
|
379
|
-
|
|
380
|
-
if datamodule_config is not None:
|
|
381
|
-
with open(datamodule_config) as dm_cfg:
|
|
361
|
+
if datamodule_config_path is not None:
|
|
362
|
+
with open(datamodule_config_path) as dm_cfg:
|
|
382
363
|
datamodule_config = yaml.load(dm_cfg, Loader=yaml.FullLoader)
|
|
383
364
|
|
|
384
|
-
|
|
385
|
-
with open(new_datamodule_config_path, "w") as outfile:
|
|
365
|
+
with open(save_directory / DATAMODULE_CONFIG_NAME, "w") as outfile:
|
|
386
366
|
yaml.dump(datamodule_config, outfile, default_flow_style=False, sort_keys=False)
|
|
387
367
|
|
|
388
|
-
card = self.create_hugging_face_model_card(
|
|
389
|
-
repo_id, wandb_repo, wandb_ids, card_template_path
|
|
390
|
-
)
|
|
368
|
+
card = self.create_hugging_face_model_card(card_template_path, wandb_repo, wandb_ids)
|
|
391
369
|
|
|
392
|
-
(save_directory /
|
|
370
|
+
(save_directory / MODEL_CARD_NAME).write_text(str(card))
|
|
393
371
|
|
|
394
372
|
if push_to_hub:
|
|
395
373
|
api = HfApi()
|
|
396
374
|
|
|
397
|
-
api.upload_folder(
|
|
398
|
-
repo_id=repo_id,
|
|
399
|
-
repo_type="model",
|
|
400
|
-
folder_path=save_directory,
|
|
401
|
-
)
|
|
375
|
+
api.upload_folder(repo_id=hf_repo_id, repo_type="model", folder_path=save_directory)
|
|
402
376
|
|
|
403
377
|
# Print the most recent commit hash
|
|
404
|
-
c = api.list_repo_commits(repo_id=
|
|
378
|
+
c = api.list_repo_commits(repo_id=hf_repo_id, repo_type="model")[0]
|
|
405
379
|
|
|
406
380
|
message = (
|
|
407
381
|
f"The latest commit is now: \n"
|
|
@@ -413,46 +387,28 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
|
|
|
413
387
|
|
|
414
388
|
print(message)
|
|
415
389
|
|
|
416
|
-
return
|
|
390
|
+
return
|
|
417
391
|
|
|
418
392
|
@staticmethod
|
|
419
393
|
def create_hugging_face_model_card(
|
|
420
|
-
|
|
421
|
-
wandb_repo:
|
|
422
|
-
wandb_ids:
|
|
423
|
-
card_template_path: Optional[Path] = None,
|
|
394
|
+
card_template_path: str,
|
|
395
|
+
wandb_repo: str,
|
|
396
|
+
wandb_ids: list[str] | str,
|
|
424
397
|
) -> ModelCard:
|
|
425
398
|
"""
|
|
426
399
|
Creates Hugging Face model card
|
|
427
400
|
|
|
428
401
|
Args:
|
|
429
|
-
|
|
430
|
-
ID of your repository on the Hub. Used only if `push_to_hub=True`. Will default to
|
|
431
|
-
the folder name if not provided.
|
|
402
|
+
card_template_path: Path to the HuggingFace model card template
|
|
432
403
|
wandb_repo: Identifier of the repo on wandb.
|
|
433
404
|
wandb_ids: Identifier(s) of the model on wandb.
|
|
434
|
-
card_template_path: Path to the HuggingFace model card template. Defaults to card in
|
|
435
|
-
PVNet library if set to None.
|
|
436
405
|
|
|
437
406
|
Returns:
|
|
438
407
|
card: ModelCard - Hugging Face model card object
|
|
439
408
|
"""
|
|
440
409
|
|
|
441
|
-
# Get appropriate model card
|
|
442
|
-
model_name = repo_id.split("/")[1]
|
|
443
|
-
if model_name == "windnet_india":
|
|
444
|
-
model_card = "wind_india_model_card_template.md"
|
|
445
|
-
elif model_name == "pvnet_india":
|
|
446
|
-
model_card = "pv_india_model_card_template.md"
|
|
447
|
-
else:
|
|
448
|
-
model_card = "pv_uk_regional_model_card_template.md"
|
|
449
|
-
|
|
450
410
|
# Creating and saving model card.
|
|
451
411
|
card_data = ModelCardData(language="en", license="mit", library_name="pytorch")
|
|
452
|
-
if card_template_path is None:
|
|
453
|
-
card_template_path = (
|
|
454
|
-
f"{os.path.dirname(os.path.abspath(__file__))}/model_cards/{model_card}"
|
|
455
|
-
)
|
|
456
412
|
|
|
457
413
|
if isinstance(wandb_ids, str):
|
|
458
414
|
wandb_ids = [wandb_ids]
|
|
@@ -477,7 +433,7 @@ class PVNetModelHubMixin(PyTorchModelHubMixin):
|
|
|
477
433
|
card_data,
|
|
478
434
|
template_path=card_template_path,
|
|
479
435
|
wandb_links=wandb_links,
|
|
480
|
-
package_versions=package_versions_markdown
|
|
436
|
+
package_versions=package_versions_markdown,
|
|
481
437
|
)
|
|
482
438
|
|
|
483
439
|
|
|
@@ -489,12 +445,12 @@ class BaseModel(pl.LightningModule, PVNetModelHubMixin):
|
|
|
489
445
|
history_minutes: int,
|
|
490
446
|
forecast_minutes: int,
|
|
491
447
|
optimizer: AbstractOptimizer,
|
|
492
|
-
output_quantiles:
|
|
448
|
+
output_quantiles: list[float] | None = None,
|
|
493
449
|
target_key: str = "gsp",
|
|
494
450
|
interval_minutes: int = 30,
|
|
495
|
-
timestep_intervals_to_plot:
|
|
496
|
-
forecast_minutes_ignore:
|
|
497
|
-
save_validation_results_csv:
|
|
451
|
+
timestep_intervals_to_plot: list[int] | None = None,
|
|
452
|
+
forecast_minutes_ignore: int = 0,
|
|
453
|
+
save_validation_results_csv: bool = False,
|
|
498
454
|
):
|
|
499
455
|
"""Abtstract base class for PVNet submodels.
|
|
500
456
|
|
|
@@ -45,7 +45,6 @@ class Model(BaseModel):
|
|
|
45
45
|
nwp_encoders_dict: Optional[dict[AbstractNWPSatelliteEncoder]] = None,
|
|
46
46
|
sat_encoder: Optional[AbstractNWPSatelliteEncoder] = None,
|
|
47
47
|
pv_encoder: Optional[AbstractSitesEncoder] = None,
|
|
48
|
-
sensor_encoder: Optional[AbstractSitesEncoder] = None,
|
|
49
48
|
add_image_embedding_channel: bool = False,
|
|
50
49
|
include_gsp_yield_history: bool = True,
|
|
51
50
|
include_site_yield_history: Optional[bool] = False,
|
|
@@ -60,15 +59,12 @@ class Model(BaseModel):
|
|
|
60
59
|
nwp_forecast_minutes: Optional[DictConfig] = None,
|
|
61
60
|
nwp_history_minutes: Optional[DictConfig] = None,
|
|
62
61
|
pv_history_minutes: Optional[int] = None,
|
|
63
|
-
sensor_history_minutes: Optional[int] = None,
|
|
64
|
-
sensor_forecast_minutes: Optional[int] = None,
|
|
65
62
|
optimizer: AbstractOptimizer = pvnet.optimizers.Adam(),
|
|
66
63
|
target_key: str = "gsp",
|
|
67
64
|
interval_minutes: int = 30,
|
|
68
65
|
nwp_interval_minutes: Optional[DictConfig] = None,
|
|
69
66
|
pv_interval_minutes: int = 5,
|
|
70
67
|
sat_interval_minutes: int = 5,
|
|
71
|
-
sensor_interval_minutes: int = 30,
|
|
72
68
|
timestep_intervals_to_plot: Optional[list[int]] = None,
|
|
73
69
|
adapt_batches: Optional[bool] = False,
|
|
74
70
|
forecast_minutes_ignore: Optional[int] = 0,
|
|
@@ -122,12 +118,8 @@ class Model(BaseModel):
|
|
|
122
118
|
data for each source
|
|
123
119
|
pv_interval_minutes: The interval between each sample of the PV data
|
|
124
120
|
sat_interval_minutes: The interval between each sample of the satellite data
|
|
125
|
-
sensor_interval_minutes: The interval between each sample of the sensor data
|
|
126
121
|
timestep_intervals_to_plot: Intervals, in timesteps, to plot in
|
|
127
122
|
addition to the full forecast
|
|
128
|
-
sensor_encoder: Encoder for sensor data
|
|
129
|
-
sensor_history_minutes: Length of recent sensor data used as input.
|
|
130
|
-
sensor_forecast_minutes: Length of forecast sensor data used as input.
|
|
131
123
|
adapt_batches: If set to true, we attempt to slice the batches to the expected shape for
|
|
132
124
|
the model to use. This allows us to overprepare batches and slice from them for the
|
|
133
125
|
data we need for a model run.
|
|
@@ -143,7 +135,6 @@ class Model(BaseModel):
|
|
|
143
135
|
self.include_pv = pv_encoder is not None
|
|
144
136
|
self.include_sun = include_sun
|
|
145
137
|
self.include_time = include_time
|
|
146
|
-
self.include_sensor = sensor_encoder is not None
|
|
147
138
|
self.location_id_mapping = location_id_mapping
|
|
148
139
|
self.embedding_dim = embedding_dim
|
|
149
140
|
self.add_image_embedding_channel = add_image_embedding_channel
|
|
@@ -254,23 +245,6 @@ class Model(BaseModel):
|
|
|
254
245
|
# Update num features
|
|
255
246
|
fusion_input_features += self.pv_encoder.out_features
|
|
256
247
|
|
|
257
|
-
if self.include_sensor:
|
|
258
|
-
if sensor_history_minutes is None:
|
|
259
|
-
sensor_history_minutes = history_minutes
|
|
260
|
-
if sensor_forecast_minutes is None:
|
|
261
|
-
sensor_forecast_minutes = forecast_minutes
|
|
262
|
-
|
|
263
|
-
self.sensor_encoder = sensor_encoder(
|
|
264
|
-
sequence_length=sensor_history_minutes // sensor_interval_minutes
|
|
265
|
-
+ sensor_forecast_minutes // sensor_interval_minutes
|
|
266
|
-
+ 1,
|
|
267
|
-
target_key_to_use=self._target_key,
|
|
268
|
-
input_key_to_use="sensor",
|
|
269
|
-
)
|
|
270
|
-
|
|
271
|
-
# Update num features
|
|
272
|
-
fusion_input_features += self.sensor_encoder.out_features
|
|
273
|
-
|
|
274
248
|
if self.use_id_embedding:
|
|
275
249
|
self.embed = nn.Embedding(num_embeddings=num_embeddings, embedding_dim=embedding_dim)
|
|
276
250
|
|
|
@@ -150,13 +150,11 @@ class SingleAttentionNetwork(AbstractSitesEncoder):
|
|
|
150
150
|
target_id_dim: The number of unique IDs.
|
|
151
151
|
target_key_to_use: The key to use for the target in the attention layer.
|
|
152
152
|
input_key_to_use: The key to use for the input in the attention layer.
|
|
153
|
-
num_channels: Number of channels in the input data
|
|
154
|
-
this will be 1, as there is not channel dimension, for Sensors,
|
|
155
|
-
this will probably be higher than that
|
|
153
|
+
num_channels: Number of channels in the input data
|
|
156
154
|
num_sites_in_inference: Number of sites to use in inference.
|
|
157
155
|
This is used to determine the number of sites to use in the
|
|
158
|
-
|
|
159
|
-
|
|
156
|
+
attention layer, for a single site, 1 works, while for multiple sites
|
|
157
|
+
this would be higher than that
|
|
160
158
|
|
|
161
159
|
"""
|
|
162
160
|
super().__init__(sequence_length, num_sites, out_features)
|
|
@@ -9,12 +9,16 @@ import pandas as pd
|
|
|
9
9
|
import pylab
|
|
10
10
|
import rich.syntax
|
|
11
11
|
import rich.tree
|
|
12
|
-
import xarray as xr
|
|
13
12
|
from lightning.pytorch.loggers import Logger
|
|
14
13
|
from lightning.pytorch.utilities import rank_zero_only
|
|
15
|
-
from ocf_data_sampler.select.location import Location
|
|
16
14
|
from omegaconf import DictConfig, OmegaConf
|
|
17
15
|
|
|
16
|
+
PYTORCH_WEIGHTS_NAME = "pytorch_model.bin"
|
|
17
|
+
MODEL_CONFIG_NAME = "model_config.yaml"
|
|
18
|
+
DATA_CONFIG_NAME = "data_config.yaml"
|
|
19
|
+
DATAMODULE_CONFIG_NAME = "datamodule_config.yaml"
|
|
20
|
+
MODEL_CARD_NAME = "README.md"
|
|
21
|
+
|
|
18
22
|
|
|
19
23
|
def get_logger(name=__name__, level=logging.INFO) -> logging.Logger:
|
|
20
24
|
"""Initializes multi-GPU-friendly python logger."""
|
|
@@ -38,61 +42,6 @@ def get_logger(name=__name__, level=logging.INFO) -> logging.Logger:
|
|
|
38
42
|
return logger
|
|
39
43
|
|
|
40
44
|
|
|
41
|
-
class GSPLocationLookup:
|
|
42
|
-
"""Query object for GSP location from GSP ID"""
|
|
43
|
-
|
|
44
|
-
def __init__(self, x_osgb: xr.DataArray, y_osgb: xr.DataArray):
|
|
45
|
-
"""Query object for GSP location from GSP ID
|
|
46
|
-
|
|
47
|
-
Args:
|
|
48
|
-
x_osgb: DataArray of the OSGB x-coordinate for any given GSP ID
|
|
49
|
-
y_osgb: DataArray of the OSGB y-coordinate for any given GSP ID
|
|
50
|
-
|
|
51
|
-
"""
|
|
52
|
-
self.x_osgb = x_osgb
|
|
53
|
-
self.y_osgb = y_osgb
|
|
54
|
-
|
|
55
|
-
def __call__(self, gsp_id: int) -> Location:
|
|
56
|
-
"""Returns the locations for the input GSP IDs.
|
|
57
|
-
|
|
58
|
-
Args:
|
|
59
|
-
gsp_id: Integer ID of the GSP
|
|
60
|
-
"""
|
|
61
|
-
return Location(
|
|
62
|
-
x=self.x_osgb.sel(gsp_id=gsp_id).item(),
|
|
63
|
-
y=self.y_osgb.sel(gsp_id=gsp_id).item(),
|
|
64
|
-
id=gsp_id,
|
|
65
|
-
)
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
class SiteLocationLookup:
|
|
69
|
-
"""Query object for site location from site ID"""
|
|
70
|
-
|
|
71
|
-
def __init__(self, long: xr.DataArray, lat: xr.DataArray):
|
|
72
|
-
"""Query object for site location from site ID
|
|
73
|
-
|
|
74
|
-
Args:
|
|
75
|
-
long: DataArray of the longitude coordinates for any given site ID
|
|
76
|
-
lat: DataArray of the latitude coordinates for any given site ID
|
|
77
|
-
|
|
78
|
-
"""
|
|
79
|
-
self.longitude = long
|
|
80
|
-
self.latitude = lat
|
|
81
|
-
|
|
82
|
-
def __call__(self, site_id: int) -> Location:
|
|
83
|
-
"""Returns the locations for the input site IDs.
|
|
84
|
-
|
|
85
|
-
Args:
|
|
86
|
-
site_id: Integer ID of the site
|
|
87
|
-
"""
|
|
88
|
-
return Location(
|
|
89
|
-
coordinate_system="lon_lat",
|
|
90
|
-
x=self.longitude.sel(pv_system_id=site_id).item(),
|
|
91
|
-
y=self.latitude.sel(pv_system_id=site_id).item(),
|
|
92
|
-
id=site_id,
|
|
93
|
-
)
|
|
94
|
-
|
|
95
|
-
|
|
96
45
|
def extras(config: DictConfig) -> None:
|
|
97
46
|
"""A couple of optional utilities.
|
|
98
47
|
|
|
@@ -45,11 +45,7 @@ dev=[
|
|
|
45
45
|
"pytest",
|
|
46
46
|
"pytest-cov",
|
|
47
47
|
]
|
|
48
|
-
|
|
49
|
-
"pytorch-tabnet",
|
|
50
|
-
"efficientnet_pytorch"
|
|
51
|
-
]
|
|
52
|
-
all=["PVNet[dev,all_models]"]
|
|
48
|
+
all=["PVNet[dev]"]
|
|
53
49
|
|
|
54
50
|
[tool.mypy]
|
|
55
51
|
exclude = [
|
|
@@ -114,7 +110,6 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
|
114
110
|
fixable = ["A", "B", "C", "D", "E", "F", "I"]
|
|
115
111
|
unfixable = []
|
|
116
112
|
select = ["E", "F", "D", "I"]
|
|
117
|
-
ignore-init-module-imports = true
|
|
118
113
|
# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
|
|
119
114
|
ignore = ["D200","D202","D210","D212","D415","D105",]
|
|
120
115
|
|
pvnet-4.1.26/pvnet/__init__.py
DELETED
pvnet-4.1.26/tests/test_utils.py
DELETED
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
from ocf_data_sampler.select.location import Location
|
|
2
|
-
from pvnet.utils import SiteLocationLookup
|
|
3
|
-
import xarray as xr
|
|
4
|
-
import pytest
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
@pytest.mark.parametrize(
|
|
8
|
-
"lookup_site_id, expected_x, expected_y, expected_id",
|
|
9
|
-
[
|
|
10
|
-
(1, -1.99106, 48.709865, 1),
|
|
11
|
-
(0, 0.30693, 51.509865, 0),
|
|
12
|
-
(2, -1.56106, 56.203865, 2),
|
|
13
|
-
],
|
|
14
|
-
)
|
|
15
|
-
def test_site_location_lookup(lookup_site_id, expected_x, expected_y, expected_id):
|
|
16
|
-
# setup
|
|
17
|
-
site_ids = [0, 1, 2]
|
|
18
|
-
longs = [0.30693, -1.99106, -1.56106]
|
|
19
|
-
lats = [51.509865, 48.709865, 56.203865]
|
|
20
|
-
da_long = xr.DataArray(
|
|
21
|
-
data=longs,
|
|
22
|
-
dims="pv_system_id",
|
|
23
|
-
coords=dict(site_id=(["pv_system_id"], site_ids), long=(["pv_system_id"], longs)),
|
|
24
|
-
)
|
|
25
|
-
da_lat = xr.DataArray(
|
|
26
|
-
data=lats,
|
|
27
|
-
dims="pv_system_id",
|
|
28
|
-
coords=dict(site_id=(["pv_system_id"], site_ids), long=(["pv_system_id"], lats)),
|
|
29
|
-
)
|
|
30
|
-
# Actual testing part
|
|
31
|
-
site_lookup = SiteLocationLookup(long=da_long, lat=da_lat)
|
|
32
|
-
|
|
33
|
-
# retrieve location of site 1
|
|
34
|
-
site_location: Location = site_lookup(lookup_site_id)
|
|
35
|
-
|
|
36
|
-
assert site_location.x == expected_x
|
|
37
|
-
assert site_location.y == expected_y
|
|
38
|
-
assert site_location.id == expected_id
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{pvnet-4.1.26 → pvnet-4.1.28}/pvnet/models/model_cards/pv_uk_regional_model_card_template.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|