PVNet 0.0.11__tar.gz → 0.0.13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {PVNet-0.0.11 → PVNet-0.0.13}/PKG-INFO +1 -1
  2. {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/PKG-INFO +1 -1
  3. {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/SOURCES.txt +3 -6
  4. {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/requires.txt +5 -2
  5. PVNet-0.0.13/pvnet/__init__.py +2 -0
  6. PVNet-0.0.13/pvnet/app.py +314 -0
  7. {PVNet-0.0.11 → PVNet-0.0.13}/pvnet/callbacks.py +32 -2
  8. {PVNet-0.0.11 → PVNet-0.0.13}/pvnet/optimizers.py +16 -1
  9. {PVNet-0.0.11 → PVNet-0.0.13}/pvnet/training.py +6 -3
  10. {PVNet-0.0.11 → PVNet-0.0.13}/pvnet/utils.py +10 -1
  11. {PVNet-0.0.11 → PVNet-0.0.13}/requirements.txt +5 -2
  12. {PVNet-0.0.11 → PVNet-0.0.13}/setup.py +1 -1
  13. PVNet-0.0.13/tests/conftest.py +271 -0
  14. PVNet-0.0.13/tests/test_app.py +46 -0
  15. PVNet-0.0.11/pvnet/data/__init__.py +0 -0
  16. PVNet-0.0.11/pvnet/data/datamodule.py +0 -180
  17. PVNet-0.0.11/pvnet/models/__init__.py +0 -0
  18. PVNet-0.0.11/pvnet/models/base_model.py +0 -321
  19. PVNet-0.0.11/pvnet/models/utils.py +0 -99
  20. PVNet-0.0.11/tests/__init__.py +0 -0
  21. PVNet-0.0.11/tests/conftest.py +0 -89
  22. {PVNet-0.0.11 → PVNet-0.0.13}/LICENSE +0 -0
  23. {PVNet-0.0.11 → PVNet-0.0.13}/MANIFEST.in +0 -0
  24. {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/dependency_links.txt +0 -0
  25. {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/top_level.txt +0 -0
  26. {PVNet-0.0.11 → PVNet-0.0.13}/README.md +0 -0
  27. {PVNet-0.0.11 → PVNet-0.0.13}/pyproject.toml +0 -0
  28. {PVNet-0.0.11 → PVNet-0.0.13}/requirements-dev.txt +0 -0
  29. {PVNet-0.0.11 → PVNet-0.0.13}/setup.cfg +0 -0
  30. {PVNet-0.0.11/pvnet → PVNet-0.0.13/tests}/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: PVNet
3
- Version: 0.0.11
3
+ Version: 0.0.13
4
4
  Summary: PVNet
5
5
  Author: Peter Dudfield
6
6
  Author-email: info@openclimatefix.org
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: PVNet
3
- Version: 0.0.11
3
+ Version: 0.0.13
4
4
  Summary: PVNet
5
5
  Author: Peter Dudfield
6
6
  Author-email: info@openclimatefix.org
@@ -11,14 +11,11 @@ PVNet.egg-info/dependency_links.txt
11
11
  PVNet.egg-info/requires.txt
12
12
  PVNet.egg-info/top_level.txt
13
13
  pvnet/__init__.py
14
+ pvnet/app.py
14
15
  pvnet/callbacks.py
15
16
  pvnet/optimizers.py
16
17
  pvnet/training.py
17
18
  pvnet/utils.py
18
- pvnet/data/__init__.py
19
- pvnet/data/datamodule.py
20
- pvnet/models/__init__.py
21
- pvnet/models/base_model.py
22
- pvnet/models/utils.py
23
19
  tests/__init__.py
24
- tests/conftest.py
20
+ tests/conftest.py
21
+ tests/test_app.py
@@ -1,4 +1,5 @@
1
1
  nowcasting_utils
2
+ nowcasting_datamodel
2
3
  ocf_datapipes
3
4
  ocf_ml_metrics
4
5
  numpy
@@ -9,12 +10,15 @@ ipykernel
9
10
  h5netcdf
10
11
  torch>=2.0
11
12
  lightning>=2.0.1
13
+ torchdata
12
14
  pytest
13
15
  pytest-cov
14
- flake8
16
+ typer
17
+ sqlalchemy
15
18
  jedi
16
19
  tables
17
20
  tilemapbase
21
+ testcontainers
18
22
  wandb
19
23
  tensorboard
20
24
  tqdm
@@ -23,7 +27,6 @@ omegaconf
23
27
  hydra-core
24
28
  python-dotenv
25
29
  hydra-optuna-sweeper
26
- black
27
30
 
28
31
  [all_models]
29
32
  pytorch-tabnet
@@ -0,0 +1,2 @@
1
+ """PVNet"""
2
+ __version__ = "0.0.8"
@@ -0,0 +1,314 @@
1
+ """App to run inference
2
+
3
+ This app expects these evironmental variables to be available:
4
+ - DB_URL
5
+ - NWP_ZARR_PATH
6
+ - SATELLITE_ZARR_PATH
7
+ """
8
+
9
+ import logging
10
+ import os
11
+ from datetime import datetime, timedelta, timezone
12
+
13
+ import fsspec
14
+ import numpy as np
15
+ import pandas as pd
16
+ import torch
17
+ import typer
18
+ from nowcasting_datamodel.connection import DatabaseConnection
19
+ from nowcasting_datamodel.models import (
20
+ ForecastSQL,
21
+ ForecastValue,
22
+ )
23
+ from nowcasting_datamodel.read.read import (
24
+ get_latest_input_data_last_updated,
25
+ get_location,
26
+ get_model,
27
+ )
28
+ from nowcasting_datamodel.save.save import save as save_sql_forecasts
29
+ from ocf_datapipes.load import OpenGSPFromDatabase
30
+ from ocf_datapipes.training.pvnet import construct_sliced_data_pipeline
31
+ from ocf_datapipes.transform.numpy.batch.sun_position import ELEVATION_MEAN, ELEVATION_STD
32
+ from ocf_datapipes.utils.consts import BatchKey, Location
33
+ from ocf_datapipes.utils.utils import stack_np_examples_into_batch
34
+ from sqlalchemy.orm import Session
35
+ from torchdata.dataloader2 import DataLoader2, MultiProcessingReadingService
36
+ from torchdata.datapipes.iter import IterableWrapper
37
+ from tqdm import tqdm
38
+
39
+ import pvnet
40
+ from pvnet.data.datamodule import batch_to_tensor
41
+ from pvnet.models.base_model import BaseModel
42
+
43
+ # ---------------------------------------------------------------------------
44
+ # GLOBAL SETTINGS
45
+
46
+ # TODO: Host data config alongside model?
47
+ this_dir = os.path.dirname(os.path.abspath(__file__))
48
+
49
+ data_config_filename = f"{this_dir}/../configs/datamodule/configuration/app_configuration.yaml"
50
+
51
+ # Model will use GPU if available
52
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
53
+
54
+ # Use multiple workers for data loading
55
+ num_workers = min(os.cpu_count() - 1, 16)
56
+
57
+ # If the solar elevation is less than this the predictions are set to zero
58
+ MIN_DAY_ELEVATION = 0
59
+
60
+ # Forecast made for these GSP IDs and summed to national with ID=>0
61
+ gsp_ids = np.arange(1, 318)
62
+
63
+ # Set up logger
64
+ logger = logging.getLogger("PVNet App")
65
+ logger.setLevel(logging.INFO)
66
+ logger.info(f"Using PVNet library version: {pvnet.__version__}")
67
+
68
+ # Huggingfacehub model repo and commit
69
+ model_name = "openclimatefix/pvnet_v2"
70
+ model_version = "7cc7e9f8e5fc472a753418c45b2af9f123547b6c"
71
+
72
+
73
+ # ---------------------------------------------------------------------------
74
+ # HELPER FUNCTIONS
75
+
76
+
77
+ def copy_batch_to_device(batch, device):
78
+ """Moves a dict-batch of tensors to new device."""
79
+ batch_copy = {}
80
+ for k in list(batch.keys()):
81
+ if isinstance(batch[k], torch.Tensor):
82
+ batch_copy[k] = batch[k].to(device)
83
+ else:
84
+ batch_copy[k] = batch[k]
85
+ return batch_copy
86
+
87
+
88
+ def id2loc(gsp_id, ds_gsp):
89
+ """Returns the locations for the input GSP IDs."""
90
+ return Location(
91
+ x=ds_gsp.x_osgb.sel(gsp_id=gsp_id).item(),
92
+ y=ds_gsp.y_osgb.sel(gsp_id=gsp_id).item(),
93
+ id=gsp_id,
94
+ )
95
+
96
+
97
+ def convert_df_to_forecasts(
98
+ forecast_values_df: pd.DataFrame, session: Session, model_name: str, version: str
99
+ ) -> list[ForecastSQL]:
100
+ """
101
+ Make a ForecastSQL object from a dataframe.
102
+
103
+ Args:
104
+ config (DictConfig): Configuration composed by Hydra.
105
+ forecast_values_df (Dataframe): Containing `target_datetime_utc` and `forecast_mw` columns
106
+ session: database session
107
+ model_name: the name of the model
108
+ version: the version of the model
109
+ Return:
110
+ List of ForecastSQL objects
111
+ """
112
+
113
+ logger.debug("Converting dataframe to list of ForecastSQL")
114
+
115
+ assert "target_datetime_utc" in forecast_values_df.columns
116
+ assert "forecast_mw" in forecast_values_df.columns
117
+ assert "gsp_id" in forecast_values_df.columns
118
+
119
+ # get last input data
120
+ input_data_last_updated = get_latest_input_data_last_updated(session=session)
121
+
122
+ # get model name
123
+ model = get_model(name=model_name, version=version, session=session)
124
+
125
+ forecasts = []
126
+
127
+ for gsp_id in forecast_values_df.gsp_id.unique():
128
+ # make forecast values
129
+ forecast_values = []
130
+
131
+ # get location
132
+ location = get_location(session=session, gsp_id=gsp_id)
133
+
134
+ gsp_forecast_values_df = forecast_values_df.query(f"gsp_id=={gsp_id}")
135
+
136
+ for i, forecast_value in gsp_forecast_values_df.iterrows():
137
+ # add timezone
138
+ target_time = forecast_value.target_datetime_utc.replace(tzinfo=timezone.utc)
139
+ forecast_value_sql = ForecastValue(
140
+ target_time=target_time,
141
+ expected_power_generation_megawatts=forecast_value.forecast_mw,
142
+ ).to_orm()
143
+ forecast_value_sql.adjust_mw = 0.0
144
+ forecast_values.append(forecast_value_sql)
145
+
146
+ # make forecast object
147
+ forecast = ForecastSQL(
148
+ model=model,
149
+ forecast_creation_time=datetime.now(tz=timezone.utc),
150
+ location=location,
151
+ input_data_last_updated=input_data_last_updated,
152
+ forecast_values=forecast_values,
153
+ historic=False,
154
+ )
155
+
156
+ forecasts.append(forecast)
157
+
158
+ return forecasts
159
+
160
+
161
+ def app(t0=None, apply_adjuster=False, gsp_ids=gsp_ids):
162
+ """Inference function for production
163
+
164
+ This app expects these evironmental variables to be available:
165
+ - DB_URL
166
+ - NWP_ZARR_PATH
167
+ - SATELLITE_ZARR_PATH
168
+ Args:
169
+ t0 (datetime): Datetime at which forecast is made
170
+ apply_adjuster (bool): Whether to apply the adjuster when saving forecast
171
+ gsp_ids (array_like): List of gsp_ids to make predictions for. This list of GSPs are summed
172
+ to national.
173
+ """
174
+ # ---------------------------------------------------------------------------
175
+ # 0. If inference datetime is None, round down to last 30 minutes
176
+ if t0 is None:
177
+ t0 = pd.Timestamp.now(tz=None).floor(timedelta(minutes=30))
178
+ else:
179
+ t0 = pd.to_datetime(t0).floor(timedelta(minutes=30))
180
+
181
+ logger.info(f"Making forecast for init time: {t0}")
182
+
183
+ # ---------------------------------------------------------------------------
184
+ # 1. Prepare data sources
185
+ logger.info("Loading GSP metadata")
186
+
187
+ ds_gsp = next(iter(OpenGSPFromDatabase()))
188
+
189
+ # DataFrame of most recent GSP capacities
190
+ gsp_capacities = (
191
+ ds_gsp.sel(
192
+ time_utc=t0,
193
+ method="ffill",
194
+ )
195
+ .sel(gsp_id=slice(1, None))
196
+ .to_dataframe()
197
+ .capacity_megawatt_power
198
+ )
199
+
200
+ # Download satellite data - can't load zipped zarr straight from s3 bucket
201
+ logger.info("Downloading zipped satellite data")
202
+ fs = fsspec.open(os.environ["SATELLITE_ZARR_PATH"]).fs
203
+ fs.get(os.environ["SATELLITE_ZARR_PATH"], "latest.zarr.zip")
204
+
205
+ # ---------------------------------------------------------------------------
206
+ # 2. Set up data loader
207
+ logger.info("Creating DataLoader")
208
+
209
+ # Location and time datapipes
210
+ location_pipe = IterableWrapper([id2loc(gsp_id, ds_gsp) for gsp_id in gsp_ids])
211
+ t0_datapipe = IterableWrapper([t0]).repeat(len(location_pipe))
212
+
213
+ location_pipe = location_pipe.sharding_filter()
214
+ t0_datapipe = t0_datapipe.sharding_filter()
215
+
216
+ # Batch datapipe
217
+ batch_datapipe = (
218
+ construct_sliced_data_pipeline(
219
+ config_filename=data_config_filename,
220
+ location_pipe=location_pipe,
221
+ t0_datapipe=t0_datapipe,
222
+ production=True,
223
+ )
224
+ .batch(10)
225
+ .map(stack_np_examples_into_batch)
226
+ )
227
+
228
+ # Set up dataloader for parallel loading
229
+ rs = MultiProcessingReadingService(
230
+ num_workers=num_workers,
231
+ multiprocessing_context="spawn",
232
+ worker_prefetch_cnt=0 if num_workers == 0 else 2,
233
+ )
234
+ dataloader = DataLoader2(batch_datapipe, reading_service=rs)
235
+
236
+ # ---------------------------------------------------------------------------
237
+ # 3. set up model
238
+ logger.info(f"Loading model: {model_name} - {model_version}")
239
+
240
+ model = BaseModel.from_pretrained(model_name, revision=model_version).to(device)
241
+
242
+ # 4. Make prediction
243
+ logger.info("Processing batches")
244
+ normed_preds = []
245
+
246
+ with torch.no_grad():
247
+ for batch in tqdm(dataloader):
248
+ # Run batch through model
249
+ device_batch = copy_batch_to_device(batch_to_tensor(batch), device)
250
+ preds = model(device_batch).detach().cpu().numpy()
251
+
252
+ # Calculate unnormalised elevation and sun-dowm mask
253
+ elevation = batch[BatchKey.gsp_solar_elevation] * ELEVATION_STD + ELEVATION_MEAN
254
+ # We only need elevation mask for forecasted values, not history
255
+ elevation = elevation[:, -preds.shape[-1] :]
256
+ sun_down_mask = elevation < MIN_DAY_ELEVATION
257
+
258
+ # Zero out after sundown
259
+ preds[sun_down_mask] = 0
260
+
261
+ normed_preds += [preds]
262
+
263
+ normed_preds = np.concatenate(normed_preds)
264
+
265
+ # ---------------------------------------------------------------------------
266
+ # 5. Merge batch results to pandas df
267
+ logger.info("Processing raw predictions to DataFrame")
268
+
269
+ n_times = normed_preds.shape[1]
270
+
271
+ df_normed = pd.DataFrame(
272
+ normed_preds.T,
273
+ columns=gsp_ids,
274
+ index=pd.Index(
275
+ [t0 + timedelta(minutes=30 * (i + 1)) for i in range(n_times)],
276
+ name="target_datetime_utc",
277
+ ),
278
+ )
279
+ # Multiply normalised forecasts by capacities and clip negatives
280
+ df_abs = df_normed.clip(0, None) * gsp_capacities.T
281
+
282
+ # ---------------------------------------------------------------------------
283
+ # 6. Make national total
284
+ logger.info("Summing to national forecast")
285
+ df_abs.insert(0, 0, df_abs.sum(axis=1))
286
+
287
+ # ---------------------------------------------------------------------------
288
+ # 7. Write predictions to database
289
+ logger.info("Writing to database")
290
+
291
+ # Flatten DataFrame
292
+ df = df_abs.reset_index().melt(
293
+ "target_datetime_utc",
294
+ var_name="gsp_id",
295
+ value_name="forecast_mw",
296
+ )
297
+
298
+ connection = DatabaseConnection(url=os.environ["DB_URL"])
299
+ with connection.get_session() as session:
300
+ sql_forecasts = convert_df_to_forecasts(
301
+ df, session, model_name=model_name, version=model_version
302
+ )
303
+
304
+ save_sql_forecasts(
305
+ forecasts=sql_forecasts,
306
+ session=session,
307
+ update_national=True,
308
+ update_gsp=True,
309
+ apply_adjuster=apply_adjuster,
310
+ )
311
+
312
+
313
+ if __name__ == "__main__":
314
+ app()
@@ -1,5 +1,4 @@
1
- """Custom callbacks developed to be able to use early stopping and learning rate finder even when
2
- pretraining parts of the network.
1
+ """Custom callbacks
3
2
  """
4
3
  from lightning.pytorch import Trainer
5
4
  from lightning.pytorch.callbacks import BaseFinetuning, EarlyStopping, LearningRateFinder
@@ -7,18 +6,26 @@ from lightning.pytorch.trainer.states import TrainerFn
7
6
 
8
7
 
9
8
  class PhaseEarlyStopping(EarlyStopping):
9
+ """Monitor a validation metric and stop training when it stops improving.
10
+
11
+ Only functions in a specific phase of training.
12
+ """
13
+
10
14
  training_phase = None
11
15
 
12
16
  def switch_phase(self, phase: str):
17
+ """Switch phase of callback"""
13
18
  if phase == self.training_phase:
14
19
  self.activate()
15
20
  else:
16
21
  self.deactivate()
17
22
 
18
23
  def deactivate(self):
24
+ """Deactivate callback"""
19
25
  self.active = False
20
26
 
21
27
  def activate(self):
28
+ """Activate callback"""
22
29
  self.active = True
23
30
 
24
31
  def _should_skip_check(self, trainer: Trainer) -> bool:
@@ -28,20 +35,34 @@ class PhaseEarlyStopping(EarlyStopping):
28
35
 
29
36
 
30
37
  class PretrainEarlyStopping(EarlyStopping):
38
+ """Monitor a validation metric and stop training when it stops improving.
39
+
40
+ Only functions in the 'pretrain' phase of training.
41
+ """
42
+
31
43
  training_phase = "pretrain"
32
44
 
33
45
 
34
46
  class MainEarlyStopping(EarlyStopping):
47
+ """Monitor a validation metric and stop training when it stops improving.
48
+
49
+ Only functions in the 'main' phase of training.
50
+ """
51
+
35
52
  training_phase = "main"
36
53
 
37
54
 
38
55
  class PretrainFreeze(BaseFinetuning):
56
+ """Freeze the satellite and NWP encoders during pretraining"""
57
+
39
58
  training_phase = "pretrain"
40
59
 
41
60
  def __init__(self):
61
+ """Freeze the satellite and NWP encoders during pretraining"""
42
62
  super().__init__()
43
63
 
44
64
  def freeze_before_training(self, pl_module):
65
+ """Freeze satellite and NWP encoders before training start"""
45
66
  # freeze any module you want
46
67
  modules = []
47
68
  if pl_module.include_sat:
@@ -51,6 +72,7 @@ class PretrainFreeze(BaseFinetuning):
51
72
  self.freeze(modules)
52
73
 
53
74
  def finetune_function(self, pl_module, current_epoch, optimizer):
75
+ """Unfreeze satellite and NWP encoders"""
54
76
  if not self.active:
55
77
  modules = []
56
78
  if pl_module.include_sat:
@@ -64,15 +86,18 @@ class PretrainFreeze(BaseFinetuning):
64
86
  )
65
87
 
66
88
  def switch_phase(self, phase: str):
89
+ """Switch phase of callback"""
67
90
  if phase == self.training_phase:
68
91
  self.activate()
69
92
  else:
70
93
  self.deactivate()
71
94
 
72
95
  def deactivate(self):
96
+ """Deactivate callback"""
73
97
  self.active = False
74
98
 
75
99
  def activate(self):
100
+ """Activate callback"""
76
101
  self.active = True
77
102
 
78
103
 
@@ -82,18 +107,23 @@ class PhasedLearningRateFinder(LearningRateFinder):
82
107
  active = True
83
108
 
84
109
  def on_fit_start(self, *args, **kwargs):
110
+ """Do nothing"""
85
111
  return
86
112
 
87
113
  def on_train_epoch_start(self, trainer, pl_module):
114
+ """Run learning rate finder on epoch start and then deactivate"""
88
115
  if self.active:
89
116
  self.lr_find(trainer, pl_module)
90
117
  self.deactivate()
91
118
 
92
119
  def switch_phase(self, phase: str):
120
+ """Switch training phase"""
93
121
  self.activate()
94
122
 
95
123
  def deactivate(self):
124
+ """Deactivate callback"""
96
125
  self.active = False
97
126
 
98
127
  def activate(self):
128
+ """Activate callback"""
99
129
  self.active = True
@@ -7,7 +7,9 @@ import torch
7
7
 
8
8
 
9
9
  class AbstractOptimizer(ABC):
10
- """Optimizer classes will be used by model like:
10
+ """Abstract class for optimizer
11
+
12
+ Optimizer classes will be used by model like:
11
13
  > OptimizerGenerator = AbstractOptimizer()
12
14
  > optimizer = OptimizerGenerator(model.parameters())
13
15
  The returned object `optimizer` must be something that may be returned by `pytorch_lightning`'s
@@ -19,29 +21,41 @@ class AbstractOptimizer(ABC):
19
21
 
20
22
  @abstractmethod
21
23
  def __call__(self):
24
+ """Abstract call"""
22
25
  pass
23
26
 
24
27
 
25
28
  class Adam(AbstractOptimizer):
29
+ """Adam optimizer"""
30
+
26
31
  def __init__(self, lr=0.0005, **kwargs):
32
+ """Adam optimizer"""
27
33
  self.lr = lr
28
34
  self.kwargs = kwargs
29
35
 
30
36
  def __call__(self, model_parameters):
37
+ """Return optimizer"""
31
38
  return torch.optim.Adam(model_parameters, lr=self.lr, **self.kwargs)
32
39
 
33
40
 
34
41
  class AdamW(AbstractOptimizer):
42
+ """AdamW optimizer"""
43
+
35
44
  def __init__(self, lr=0.0005, **kwargs):
45
+ """AdamW optimizer"""
36
46
  self.lr = lr
37
47
  self.kwargs = kwargs
38
48
 
39
49
  def __call__(self, model_parameters):
50
+ """Return optimizer"""
40
51
  return torch.optim.AdamW(model_parameters, lr=self.lr, **self.kwargs)
41
52
 
42
53
 
43
54
  class AdamWReduceLROnPlateau(AbstractOptimizer):
55
+ """AdamW optimizer and reduce on plateau scheduler"""
56
+
44
57
  def __init__(self, lr=0.0005, patience=3, factor=0.5, threshold=2e-4, **opt_kwargs):
58
+ """AdamW optimizer and reduce on plateau scheduler"""
45
59
  self.lr = lr
46
60
  self.patience = patience
47
61
  self.factor = factor
@@ -49,6 +63,7 @@ class AdamWReduceLROnPlateau(AbstractOptimizer):
49
63
  self.opt_kwargs = opt_kwargs
50
64
 
51
65
  def __call__(self, model_parameters):
66
+ """Return optimizer"""
52
67
  opt = torch.optim.AdamW(model_parameters, lr=self.lr, **self.opt_kwargs)
53
68
  sch = torch.optim.lr_scheduler.ReduceLROnPlateau(
54
69
  opt,
@@ -1,3 +1,5 @@
1
+ """Training"""
2
+
1
3
  from typing import Optional
2
4
 
3
5
  import hydra
@@ -19,7 +21,7 @@ log = utils.get_logger(__name__)
19
21
  torch.set_default_dtype(torch.float32)
20
22
 
21
23
 
22
- def callbacks_to_phase(callbacks, phase):
24
+ def _callbacks_to_phase(callbacks, phase):
23
25
  for c in callbacks:
24
26
  if hasattr(c, "switch_phase"):
25
27
  c.switch_phase(phase)
@@ -27,6 +29,7 @@ def callbacks_to_phase(callbacks, phase):
27
29
 
28
30
  def train(config: DictConfig) -> Optional[float]:
29
31
  """Contains training pipeline.
32
+
30
33
  Instantiates all PyTorch Lightning objects from config.
31
34
 
32
35
  Args:
@@ -69,7 +72,7 @@ def train(config: DictConfig) -> Optional[float]:
69
72
  should_pretrain |= hasattr(c, "training_phase") and c.training_phase == "pretrain"
70
73
 
71
74
  if should_pretrain:
72
- callbacks_to_phase(callbacks, "pretrain")
75
+ _callbacks_to_phase(callbacks, "pretrain")
73
76
 
74
77
  trainer: Trainer = hydra.utils.instantiate(
75
78
  config.trainer,
@@ -83,7 +86,7 @@ def train(config: DictConfig) -> Optional[float]:
83
86
  datamodule.block_nwp_and_sat = True
84
87
  trainer.fit(model=model, datamodule=datamodule)
85
88
 
86
- callbacks_to_phase(callbacks, "main")
89
+ _callbacks_to_phase(callbacks, "main")
87
90
 
88
91
  datamodule.block_nwp_and_sat = False
89
92
  trainer.should_stop = False
@@ -1,3 +1,5 @@
1
+ """Utils"""
2
+
1
3
  import logging
2
4
  import os
3
5
  import warnings
@@ -60,7 +62,9 @@ def get_logger(name=__name__, level=logging.INFO) -> logging.Logger:
60
62
 
61
63
 
62
64
  def extras(config: DictConfig) -> None:
63
- """A couple of optional utilities, controlled by main config file:
65
+ """A couple of optional utilities.
66
+
67
+ Controlled by main config file:
64
68
  - disabling warnings
65
69
  - easier access to debug mode
66
70
  - forcing debug friendly configuration
@@ -143,6 +147,7 @@ def print_config(
143
147
 
144
148
 
145
149
  def empty(*args, **kwargs):
150
+ """Returns nothing"""
146
151
  pass
147
152
 
148
153
 
@@ -209,6 +214,8 @@ def finish(
209
214
 
210
215
 
211
216
  def plot_batch_forecasts(batch, y_hat, batch_idx=None):
217
+ """Plot a batch of data and the forecast from that batch"""
218
+
212
219
  def _get_numpy(key):
213
220
  return batch[key].cpu().numpy().squeeze()
214
221
 
@@ -254,6 +261,8 @@ def plot_batch_forecasts(batch, y_hat, batch_idx=None):
254
261
 
255
262
 
256
263
  def construct_ocf_ml_metrics_batch_df(batch, y, y_hat):
264
+ """Helper function tot construct DataFrame for ocf_ml_metrics"""
265
+
257
266
  def _repeat(x):
258
267
  return np.repeat(x.squeeze(), n_times)
259
268
 
@@ -1,4 +1,5 @@
1
1
  nowcasting_utils
2
+ nowcasting_datamodel
2
3
  ocf_datapipes
3
4
  ocf_ml_metrics
4
5
  numpy
@@ -9,12 +10,15 @@ ipykernel
9
10
  h5netcdf
10
11
  torch>=2.0
11
12
  lightning>=2.0.1
13
+ torchdata
12
14
  pytest
13
15
  pytest-cov
14
- flake8
16
+ typer
17
+ sqlalchemy
15
18
  jedi
16
19
  tables
17
20
  tilemapbase
21
+ testcontainers
18
22
  wandb
19
23
  tensorboard
20
24
  tqdm
@@ -23,4 +27,3 @@ omegaconf
23
27
  hydra-core
24
28
  python-dotenv
25
29
  hydra-optuna-sweeper
26
- black
@@ -11,7 +11,7 @@ extras_require = {"all_models": ["pytorch-tabnet", "efficientnet_pytorch"]}
11
11
 
12
12
  setup(
13
13
  name="PVNet",
14
- version="0.0.11",
14
+ version="0.0.13",
15
15
  license="MIT",
16
16
  description="PVNet",
17
17
  author="Peter Dudfield",