PVNet 0.0.11__tar.gz → 0.0.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {PVNet-0.0.11 → PVNet-0.0.13}/PKG-INFO +1 -1
- {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/PKG-INFO +1 -1
- {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/SOURCES.txt +3 -6
- {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/requires.txt +5 -2
- PVNet-0.0.13/pvnet/__init__.py +2 -0
- PVNet-0.0.13/pvnet/app.py +314 -0
- {PVNet-0.0.11 → PVNet-0.0.13}/pvnet/callbacks.py +32 -2
- {PVNet-0.0.11 → PVNet-0.0.13}/pvnet/optimizers.py +16 -1
- {PVNet-0.0.11 → PVNet-0.0.13}/pvnet/training.py +6 -3
- {PVNet-0.0.11 → PVNet-0.0.13}/pvnet/utils.py +10 -1
- {PVNet-0.0.11 → PVNet-0.0.13}/requirements.txt +5 -2
- {PVNet-0.0.11 → PVNet-0.0.13}/setup.py +1 -1
- PVNet-0.0.13/tests/conftest.py +271 -0
- PVNet-0.0.13/tests/test_app.py +46 -0
- PVNet-0.0.11/pvnet/data/__init__.py +0 -0
- PVNet-0.0.11/pvnet/data/datamodule.py +0 -180
- PVNet-0.0.11/pvnet/models/__init__.py +0 -0
- PVNet-0.0.11/pvnet/models/base_model.py +0 -321
- PVNet-0.0.11/pvnet/models/utils.py +0 -99
- PVNet-0.0.11/tests/__init__.py +0 -0
- PVNet-0.0.11/tests/conftest.py +0 -89
- {PVNet-0.0.11 → PVNet-0.0.13}/LICENSE +0 -0
- {PVNet-0.0.11 → PVNet-0.0.13}/MANIFEST.in +0 -0
- {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/dependency_links.txt +0 -0
- {PVNet-0.0.11 → PVNet-0.0.13}/PVNet.egg-info/top_level.txt +0 -0
- {PVNet-0.0.11 → PVNet-0.0.13}/README.md +0 -0
- {PVNet-0.0.11 → PVNet-0.0.13}/pyproject.toml +0 -0
- {PVNet-0.0.11 → PVNet-0.0.13}/requirements-dev.txt +0 -0
- {PVNet-0.0.11 → PVNet-0.0.13}/setup.cfg +0 -0
- {PVNet-0.0.11/pvnet → PVNet-0.0.13/tests}/__init__.py +0 -0
|
@@ -11,14 +11,11 @@ PVNet.egg-info/dependency_links.txt
|
|
|
11
11
|
PVNet.egg-info/requires.txt
|
|
12
12
|
PVNet.egg-info/top_level.txt
|
|
13
13
|
pvnet/__init__.py
|
|
14
|
+
pvnet/app.py
|
|
14
15
|
pvnet/callbacks.py
|
|
15
16
|
pvnet/optimizers.py
|
|
16
17
|
pvnet/training.py
|
|
17
18
|
pvnet/utils.py
|
|
18
|
-
pvnet/data/__init__.py
|
|
19
|
-
pvnet/data/datamodule.py
|
|
20
|
-
pvnet/models/__init__.py
|
|
21
|
-
pvnet/models/base_model.py
|
|
22
|
-
pvnet/models/utils.py
|
|
23
19
|
tests/__init__.py
|
|
24
|
-
tests/conftest.py
|
|
20
|
+
tests/conftest.py
|
|
21
|
+
tests/test_app.py
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
nowcasting_utils
|
|
2
|
+
nowcasting_datamodel
|
|
2
3
|
ocf_datapipes
|
|
3
4
|
ocf_ml_metrics
|
|
4
5
|
numpy
|
|
@@ -9,12 +10,15 @@ ipykernel
|
|
|
9
10
|
h5netcdf
|
|
10
11
|
torch>=2.0
|
|
11
12
|
lightning>=2.0.1
|
|
13
|
+
torchdata
|
|
12
14
|
pytest
|
|
13
15
|
pytest-cov
|
|
14
|
-
|
|
16
|
+
typer
|
|
17
|
+
sqlalchemy
|
|
15
18
|
jedi
|
|
16
19
|
tables
|
|
17
20
|
tilemapbase
|
|
21
|
+
testcontainers
|
|
18
22
|
wandb
|
|
19
23
|
tensorboard
|
|
20
24
|
tqdm
|
|
@@ -23,7 +27,6 @@ omegaconf
|
|
|
23
27
|
hydra-core
|
|
24
28
|
python-dotenv
|
|
25
29
|
hydra-optuna-sweeper
|
|
26
|
-
black
|
|
27
30
|
|
|
28
31
|
[all_models]
|
|
29
32
|
pytorch-tabnet
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""App to run inference
|
|
2
|
+
|
|
3
|
+
This app expects these evironmental variables to be available:
|
|
4
|
+
- DB_URL
|
|
5
|
+
- NWP_ZARR_PATH
|
|
6
|
+
- SATELLITE_ZARR_PATH
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
from datetime import datetime, timedelta, timezone
|
|
12
|
+
|
|
13
|
+
import fsspec
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pandas as pd
|
|
16
|
+
import torch
|
|
17
|
+
import typer
|
|
18
|
+
from nowcasting_datamodel.connection import DatabaseConnection
|
|
19
|
+
from nowcasting_datamodel.models import (
|
|
20
|
+
ForecastSQL,
|
|
21
|
+
ForecastValue,
|
|
22
|
+
)
|
|
23
|
+
from nowcasting_datamodel.read.read import (
|
|
24
|
+
get_latest_input_data_last_updated,
|
|
25
|
+
get_location,
|
|
26
|
+
get_model,
|
|
27
|
+
)
|
|
28
|
+
from nowcasting_datamodel.save.save import save as save_sql_forecasts
|
|
29
|
+
from ocf_datapipes.load import OpenGSPFromDatabase
|
|
30
|
+
from ocf_datapipes.training.pvnet import construct_sliced_data_pipeline
|
|
31
|
+
from ocf_datapipes.transform.numpy.batch.sun_position import ELEVATION_MEAN, ELEVATION_STD
|
|
32
|
+
from ocf_datapipes.utils.consts import BatchKey, Location
|
|
33
|
+
from ocf_datapipes.utils.utils import stack_np_examples_into_batch
|
|
34
|
+
from sqlalchemy.orm import Session
|
|
35
|
+
from torchdata.dataloader2 import DataLoader2, MultiProcessingReadingService
|
|
36
|
+
from torchdata.datapipes.iter import IterableWrapper
|
|
37
|
+
from tqdm import tqdm
|
|
38
|
+
|
|
39
|
+
import pvnet
|
|
40
|
+
from pvnet.data.datamodule import batch_to_tensor
|
|
41
|
+
from pvnet.models.base_model import BaseModel
|
|
42
|
+
|
|
43
|
+
# ---------------------------------------------------------------------------
|
|
44
|
+
# GLOBAL SETTINGS
|
|
45
|
+
|
|
46
|
+
# TODO: Host data config alongside model?
|
|
47
|
+
this_dir = os.path.dirname(os.path.abspath(__file__))
|
|
48
|
+
|
|
49
|
+
data_config_filename = f"{this_dir}/../configs/datamodule/configuration/app_configuration.yaml"
|
|
50
|
+
|
|
51
|
+
# Model will use GPU if available
|
|
52
|
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
53
|
+
|
|
54
|
+
# Use multiple workers for data loading
|
|
55
|
+
num_workers = min(os.cpu_count() - 1, 16)
|
|
56
|
+
|
|
57
|
+
# If the solar elevation is less than this the predictions are set to zero
|
|
58
|
+
MIN_DAY_ELEVATION = 0
|
|
59
|
+
|
|
60
|
+
# Forecast made for these GSP IDs and summed to national with ID=>0
|
|
61
|
+
gsp_ids = np.arange(1, 318)
|
|
62
|
+
|
|
63
|
+
# Set up logger
|
|
64
|
+
logger = logging.getLogger("PVNet App")
|
|
65
|
+
logger.setLevel(logging.INFO)
|
|
66
|
+
logger.info(f"Using PVNet library version: {pvnet.__version__}")
|
|
67
|
+
|
|
68
|
+
# Huggingfacehub model repo and commit
|
|
69
|
+
model_name = "openclimatefix/pvnet_v2"
|
|
70
|
+
model_version = "7cc7e9f8e5fc472a753418c45b2af9f123547b6c"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
# HELPER FUNCTIONS
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def copy_batch_to_device(batch, device):
|
|
78
|
+
"""Moves a dict-batch of tensors to new device."""
|
|
79
|
+
batch_copy = {}
|
|
80
|
+
for k in list(batch.keys()):
|
|
81
|
+
if isinstance(batch[k], torch.Tensor):
|
|
82
|
+
batch_copy[k] = batch[k].to(device)
|
|
83
|
+
else:
|
|
84
|
+
batch_copy[k] = batch[k]
|
|
85
|
+
return batch_copy
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def id2loc(gsp_id, ds_gsp):
|
|
89
|
+
"""Returns the locations for the input GSP IDs."""
|
|
90
|
+
return Location(
|
|
91
|
+
x=ds_gsp.x_osgb.sel(gsp_id=gsp_id).item(),
|
|
92
|
+
y=ds_gsp.y_osgb.sel(gsp_id=gsp_id).item(),
|
|
93
|
+
id=gsp_id,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def convert_df_to_forecasts(
|
|
98
|
+
forecast_values_df: pd.DataFrame, session: Session, model_name: str, version: str
|
|
99
|
+
) -> list[ForecastSQL]:
|
|
100
|
+
"""
|
|
101
|
+
Make a ForecastSQL object from a dataframe.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
config (DictConfig): Configuration composed by Hydra.
|
|
105
|
+
forecast_values_df (Dataframe): Containing `target_datetime_utc` and `forecast_mw` columns
|
|
106
|
+
session: database session
|
|
107
|
+
model_name: the name of the model
|
|
108
|
+
version: the version of the model
|
|
109
|
+
Return:
|
|
110
|
+
List of ForecastSQL objects
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
logger.debug("Converting dataframe to list of ForecastSQL")
|
|
114
|
+
|
|
115
|
+
assert "target_datetime_utc" in forecast_values_df.columns
|
|
116
|
+
assert "forecast_mw" in forecast_values_df.columns
|
|
117
|
+
assert "gsp_id" in forecast_values_df.columns
|
|
118
|
+
|
|
119
|
+
# get last input data
|
|
120
|
+
input_data_last_updated = get_latest_input_data_last_updated(session=session)
|
|
121
|
+
|
|
122
|
+
# get model name
|
|
123
|
+
model = get_model(name=model_name, version=version, session=session)
|
|
124
|
+
|
|
125
|
+
forecasts = []
|
|
126
|
+
|
|
127
|
+
for gsp_id in forecast_values_df.gsp_id.unique():
|
|
128
|
+
# make forecast values
|
|
129
|
+
forecast_values = []
|
|
130
|
+
|
|
131
|
+
# get location
|
|
132
|
+
location = get_location(session=session, gsp_id=gsp_id)
|
|
133
|
+
|
|
134
|
+
gsp_forecast_values_df = forecast_values_df.query(f"gsp_id=={gsp_id}")
|
|
135
|
+
|
|
136
|
+
for i, forecast_value in gsp_forecast_values_df.iterrows():
|
|
137
|
+
# add timezone
|
|
138
|
+
target_time = forecast_value.target_datetime_utc.replace(tzinfo=timezone.utc)
|
|
139
|
+
forecast_value_sql = ForecastValue(
|
|
140
|
+
target_time=target_time,
|
|
141
|
+
expected_power_generation_megawatts=forecast_value.forecast_mw,
|
|
142
|
+
).to_orm()
|
|
143
|
+
forecast_value_sql.adjust_mw = 0.0
|
|
144
|
+
forecast_values.append(forecast_value_sql)
|
|
145
|
+
|
|
146
|
+
# make forecast object
|
|
147
|
+
forecast = ForecastSQL(
|
|
148
|
+
model=model,
|
|
149
|
+
forecast_creation_time=datetime.now(tz=timezone.utc),
|
|
150
|
+
location=location,
|
|
151
|
+
input_data_last_updated=input_data_last_updated,
|
|
152
|
+
forecast_values=forecast_values,
|
|
153
|
+
historic=False,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
forecasts.append(forecast)
|
|
157
|
+
|
|
158
|
+
return forecasts
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def app(t0=None, apply_adjuster=False, gsp_ids=gsp_ids):
|
|
162
|
+
"""Inference function for production
|
|
163
|
+
|
|
164
|
+
This app expects these evironmental variables to be available:
|
|
165
|
+
- DB_URL
|
|
166
|
+
- NWP_ZARR_PATH
|
|
167
|
+
- SATELLITE_ZARR_PATH
|
|
168
|
+
Args:
|
|
169
|
+
t0 (datetime): Datetime at which forecast is made
|
|
170
|
+
apply_adjuster (bool): Whether to apply the adjuster when saving forecast
|
|
171
|
+
gsp_ids (array_like): List of gsp_ids to make predictions for. This list of GSPs are summed
|
|
172
|
+
to national.
|
|
173
|
+
"""
|
|
174
|
+
# ---------------------------------------------------------------------------
|
|
175
|
+
# 0. If inference datetime is None, round down to last 30 minutes
|
|
176
|
+
if t0 is None:
|
|
177
|
+
t0 = pd.Timestamp.now(tz=None).floor(timedelta(minutes=30))
|
|
178
|
+
else:
|
|
179
|
+
t0 = pd.to_datetime(t0).floor(timedelta(minutes=30))
|
|
180
|
+
|
|
181
|
+
logger.info(f"Making forecast for init time: {t0}")
|
|
182
|
+
|
|
183
|
+
# ---------------------------------------------------------------------------
|
|
184
|
+
# 1. Prepare data sources
|
|
185
|
+
logger.info("Loading GSP metadata")
|
|
186
|
+
|
|
187
|
+
ds_gsp = next(iter(OpenGSPFromDatabase()))
|
|
188
|
+
|
|
189
|
+
# DataFrame of most recent GSP capacities
|
|
190
|
+
gsp_capacities = (
|
|
191
|
+
ds_gsp.sel(
|
|
192
|
+
time_utc=t0,
|
|
193
|
+
method="ffill",
|
|
194
|
+
)
|
|
195
|
+
.sel(gsp_id=slice(1, None))
|
|
196
|
+
.to_dataframe()
|
|
197
|
+
.capacity_megawatt_power
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
# Download satellite data - can't load zipped zarr straight from s3 bucket
|
|
201
|
+
logger.info("Downloading zipped satellite data")
|
|
202
|
+
fs = fsspec.open(os.environ["SATELLITE_ZARR_PATH"]).fs
|
|
203
|
+
fs.get(os.environ["SATELLITE_ZARR_PATH"], "latest.zarr.zip")
|
|
204
|
+
|
|
205
|
+
# ---------------------------------------------------------------------------
|
|
206
|
+
# 2. Set up data loader
|
|
207
|
+
logger.info("Creating DataLoader")
|
|
208
|
+
|
|
209
|
+
# Location and time datapipes
|
|
210
|
+
location_pipe = IterableWrapper([id2loc(gsp_id, ds_gsp) for gsp_id in gsp_ids])
|
|
211
|
+
t0_datapipe = IterableWrapper([t0]).repeat(len(location_pipe))
|
|
212
|
+
|
|
213
|
+
location_pipe = location_pipe.sharding_filter()
|
|
214
|
+
t0_datapipe = t0_datapipe.sharding_filter()
|
|
215
|
+
|
|
216
|
+
# Batch datapipe
|
|
217
|
+
batch_datapipe = (
|
|
218
|
+
construct_sliced_data_pipeline(
|
|
219
|
+
config_filename=data_config_filename,
|
|
220
|
+
location_pipe=location_pipe,
|
|
221
|
+
t0_datapipe=t0_datapipe,
|
|
222
|
+
production=True,
|
|
223
|
+
)
|
|
224
|
+
.batch(10)
|
|
225
|
+
.map(stack_np_examples_into_batch)
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
# Set up dataloader for parallel loading
|
|
229
|
+
rs = MultiProcessingReadingService(
|
|
230
|
+
num_workers=num_workers,
|
|
231
|
+
multiprocessing_context="spawn",
|
|
232
|
+
worker_prefetch_cnt=0 if num_workers == 0 else 2,
|
|
233
|
+
)
|
|
234
|
+
dataloader = DataLoader2(batch_datapipe, reading_service=rs)
|
|
235
|
+
|
|
236
|
+
# ---------------------------------------------------------------------------
|
|
237
|
+
# 3. set up model
|
|
238
|
+
logger.info(f"Loading model: {model_name} - {model_version}")
|
|
239
|
+
|
|
240
|
+
model = BaseModel.from_pretrained(model_name, revision=model_version).to(device)
|
|
241
|
+
|
|
242
|
+
# 4. Make prediction
|
|
243
|
+
logger.info("Processing batches")
|
|
244
|
+
normed_preds = []
|
|
245
|
+
|
|
246
|
+
with torch.no_grad():
|
|
247
|
+
for batch in tqdm(dataloader):
|
|
248
|
+
# Run batch through model
|
|
249
|
+
device_batch = copy_batch_to_device(batch_to_tensor(batch), device)
|
|
250
|
+
preds = model(device_batch).detach().cpu().numpy()
|
|
251
|
+
|
|
252
|
+
# Calculate unnormalised elevation and sun-dowm mask
|
|
253
|
+
elevation = batch[BatchKey.gsp_solar_elevation] * ELEVATION_STD + ELEVATION_MEAN
|
|
254
|
+
# We only need elevation mask for forecasted values, not history
|
|
255
|
+
elevation = elevation[:, -preds.shape[-1] :]
|
|
256
|
+
sun_down_mask = elevation < MIN_DAY_ELEVATION
|
|
257
|
+
|
|
258
|
+
# Zero out after sundown
|
|
259
|
+
preds[sun_down_mask] = 0
|
|
260
|
+
|
|
261
|
+
normed_preds += [preds]
|
|
262
|
+
|
|
263
|
+
normed_preds = np.concatenate(normed_preds)
|
|
264
|
+
|
|
265
|
+
# ---------------------------------------------------------------------------
|
|
266
|
+
# 5. Merge batch results to pandas df
|
|
267
|
+
logger.info("Processing raw predictions to DataFrame")
|
|
268
|
+
|
|
269
|
+
n_times = normed_preds.shape[1]
|
|
270
|
+
|
|
271
|
+
df_normed = pd.DataFrame(
|
|
272
|
+
normed_preds.T,
|
|
273
|
+
columns=gsp_ids,
|
|
274
|
+
index=pd.Index(
|
|
275
|
+
[t0 + timedelta(minutes=30 * (i + 1)) for i in range(n_times)],
|
|
276
|
+
name="target_datetime_utc",
|
|
277
|
+
),
|
|
278
|
+
)
|
|
279
|
+
# Multiply normalised forecasts by capacities and clip negatives
|
|
280
|
+
df_abs = df_normed.clip(0, None) * gsp_capacities.T
|
|
281
|
+
|
|
282
|
+
# ---------------------------------------------------------------------------
|
|
283
|
+
# 6. Make national total
|
|
284
|
+
logger.info("Summing to national forecast")
|
|
285
|
+
df_abs.insert(0, 0, df_abs.sum(axis=1))
|
|
286
|
+
|
|
287
|
+
# ---------------------------------------------------------------------------
|
|
288
|
+
# 7. Write predictions to database
|
|
289
|
+
logger.info("Writing to database")
|
|
290
|
+
|
|
291
|
+
# Flatten DataFrame
|
|
292
|
+
df = df_abs.reset_index().melt(
|
|
293
|
+
"target_datetime_utc",
|
|
294
|
+
var_name="gsp_id",
|
|
295
|
+
value_name="forecast_mw",
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
connection = DatabaseConnection(url=os.environ["DB_URL"])
|
|
299
|
+
with connection.get_session() as session:
|
|
300
|
+
sql_forecasts = convert_df_to_forecasts(
|
|
301
|
+
df, session, model_name=model_name, version=model_version
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
save_sql_forecasts(
|
|
305
|
+
forecasts=sql_forecasts,
|
|
306
|
+
session=session,
|
|
307
|
+
update_national=True,
|
|
308
|
+
update_gsp=True,
|
|
309
|
+
apply_adjuster=apply_adjuster,
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
if __name__ == "__main__":
|
|
314
|
+
app()
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""Custom callbacks
|
|
2
|
-
pretraining parts of the network.
|
|
1
|
+
"""Custom callbacks
|
|
3
2
|
"""
|
|
4
3
|
from lightning.pytorch import Trainer
|
|
5
4
|
from lightning.pytorch.callbacks import BaseFinetuning, EarlyStopping, LearningRateFinder
|
|
@@ -7,18 +6,26 @@ from lightning.pytorch.trainer.states import TrainerFn
|
|
|
7
6
|
|
|
8
7
|
|
|
9
8
|
class PhaseEarlyStopping(EarlyStopping):
|
|
9
|
+
"""Monitor a validation metric and stop training when it stops improving.
|
|
10
|
+
|
|
11
|
+
Only functions in a specific phase of training.
|
|
12
|
+
"""
|
|
13
|
+
|
|
10
14
|
training_phase = None
|
|
11
15
|
|
|
12
16
|
def switch_phase(self, phase: str):
|
|
17
|
+
"""Switch phase of callback"""
|
|
13
18
|
if phase == self.training_phase:
|
|
14
19
|
self.activate()
|
|
15
20
|
else:
|
|
16
21
|
self.deactivate()
|
|
17
22
|
|
|
18
23
|
def deactivate(self):
|
|
24
|
+
"""Deactivate callback"""
|
|
19
25
|
self.active = False
|
|
20
26
|
|
|
21
27
|
def activate(self):
|
|
28
|
+
"""Activate callback"""
|
|
22
29
|
self.active = True
|
|
23
30
|
|
|
24
31
|
def _should_skip_check(self, trainer: Trainer) -> bool:
|
|
@@ -28,20 +35,34 @@ class PhaseEarlyStopping(EarlyStopping):
|
|
|
28
35
|
|
|
29
36
|
|
|
30
37
|
class PretrainEarlyStopping(EarlyStopping):
|
|
38
|
+
"""Monitor a validation metric and stop training when it stops improving.
|
|
39
|
+
|
|
40
|
+
Only functions in the 'pretrain' phase of training.
|
|
41
|
+
"""
|
|
42
|
+
|
|
31
43
|
training_phase = "pretrain"
|
|
32
44
|
|
|
33
45
|
|
|
34
46
|
class MainEarlyStopping(EarlyStopping):
|
|
47
|
+
"""Monitor a validation metric and stop training when it stops improving.
|
|
48
|
+
|
|
49
|
+
Only functions in the 'main' phase of training.
|
|
50
|
+
"""
|
|
51
|
+
|
|
35
52
|
training_phase = "main"
|
|
36
53
|
|
|
37
54
|
|
|
38
55
|
class PretrainFreeze(BaseFinetuning):
|
|
56
|
+
"""Freeze the satellite and NWP encoders during pretraining"""
|
|
57
|
+
|
|
39
58
|
training_phase = "pretrain"
|
|
40
59
|
|
|
41
60
|
def __init__(self):
|
|
61
|
+
"""Freeze the satellite and NWP encoders during pretraining"""
|
|
42
62
|
super().__init__()
|
|
43
63
|
|
|
44
64
|
def freeze_before_training(self, pl_module):
|
|
65
|
+
"""Freeze satellite and NWP encoders before training start"""
|
|
45
66
|
# freeze any module you want
|
|
46
67
|
modules = []
|
|
47
68
|
if pl_module.include_sat:
|
|
@@ -51,6 +72,7 @@ class PretrainFreeze(BaseFinetuning):
|
|
|
51
72
|
self.freeze(modules)
|
|
52
73
|
|
|
53
74
|
def finetune_function(self, pl_module, current_epoch, optimizer):
|
|
75
|
+
"""Unfreeze satellite and NWP encoders"""
|
|
54
76
|
if not self.active:
|
|
55
77
|
modules = []
|
|
56
78
|
if pl_module.include_sat:
|
|
@@ -64,15 +86,18 @@ class PretrainFreeze(BaseFinetuning):
|
|
|
64
86
|
)
|
|
65
87
|
|
|
66
88
|
def switch_phase(self, phase: str):
|
|
89
|
+
"""Switch phase of callback"""
|
|
67
90
|
if phase == self.training_phase:
|
|
68
91
|
self.activate()
|
|
69
92
|
else:
|
|
70
93
|
self.deactivate()
|
|
71
94
|
|
|
72
95
|
def deactivate(self):
|
|
96
|
+
"""Deactivate callback"""
|
|
73
97
|
self.active = False
|
|
74
98
|
|
|
75
99
|
def activate(self):
|
|
100
|
+
"""Activate callback"""
|
|
76
101
|
self.active = True
|
|
77
102
|
|
|
78
103
|
|
|
@@ -82,18 +107,23 @@ class PhasedLearningRateFinder(LearningRateFinder):
|
|
|
82
107
|
active = True
|
|
83
108
|
|
|
84
109
|
def on_fit_start(self, *args, **kwargs):
|
|
110
|
+
"""Do nothing"""
|
|
85
111
|
return
|
|
86
112
|
|
|
87
113
|
def on_train_epoch_start(self, trainer, pl_module):
|
|
114
|
+
"""Run learning rate finder on epoch start and then deactivate"""
|
|
88
115
|
if self.active:
|
|
89
116
|
self.lr_find(trainer, pl_module)
|
|
90
117
|
self.deactivate()
|
|
91
118
|
|
|
92
119
|
def switch_phase(self, phase: str):
|
|
120
|
+
"""Switch training phase"""
|
|
93
121
|
self.activate()
|
|
94
122
|
|
|
95
123
|
def deactivate(self):
|
|
124
|
+
"""Deactivate callback"""
|
|
96
125
|
self.active = False
|
|
97
126
|
|
|
98
127
|
def activate(self):
|
|
128
|
+
"""Activate callback"""
|
|
99
129
|
self.active = True
|
|
@@ -7,7 +7,9 @@ import torch
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class AbstractOptimizer(ABC):
|
|
10
|
-
"""
|
|
10
|
+
"""Abstract class for optimizer
|
|
11
|
+
|
|
12
|
+
Optimizer classes will be used by model like:
|
|
11
13
|
> OptimizerGenerator = AbstractOptimizer()
|
|
12
14
|
> optimizer = OptimizerGenerator(model.parameters())
|
|
13
15
|
The returned object `optimizer` must be something that may be returned by `pytorch_lightning`'s
|
|
@@ -19,29 +21,41 @@ class AbstractOptimizer(ABC):
|
|
|
19
21
|
|
|
20
22
|
@abstractmethod
|
|
21
23
|
def __call__(self):
|
|
24
|
+
"""Abstract call"""
|
|
22
25
|
pass
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
class Adam(AbstractOptimizer):
|
|
29
|
+
"""Adam optimizer"""
|
|
30
|
+
|
|
26
31
|
def __init__(self, lr=0.0005, **kwargs):
|
|
32
|
+
"""Adam optimizer"""
|
|
27
33
|
self.lr = lr
|
|
28
34
|
self.kwargs = kwargs
|
|
29
35
|
|
|
30
36
|
def __call__(self, model_parameters):
|
|
37
|
+
"""Return optimizer"""
|
|
31
38
|
return torch.optim.Adam(model_parameters, lr=self.lr, **self.kwargs)
|
|
32
39
|
|
|
33
40
|
|
|
34
41
|
class AdamW(AbstractOptimizer):
|
|
42
|
+
"""AdamW optimizer"""
|
|
43
|
+
|
|
35
44
|
def __init__(self, lr=0.0005, **kwargs):
|
|
45
|
+
"""AdamW optimizer"""
|
|
36
46
|
self.lr = lr
|
|
37
47
|
self.kwargs = kwargs
|
|
38
48
|
|
|
39
49
|
def __call__(self, model_parameters):
|
|
50
|
+
"""Return optimizer"""
|
|
40
51
|
return torch.optim.AdamW(model_parameters, lr=self.lr, **self.kwargs)
|
|
41
52
|
|
|
42
53
|
|
|
43
54
|
class AdamWReduceLROnPlateau(AbstractOptimizer):
|
|
55
|
+
"""AdamW optimizer and reduce on plateau scheduler"""
|
|
56
|
+
|
|
44
57
|
def __init__(self, lr=0.0005, patience=3, factor=0.5, threshold=2e-4, **opt_kwargs):
|
|
58
|
+
"""AdamW optimizer and reduce on plateau scheduler"""
|
|
45
59
|
self.lr = lr
|
|
46
60
|
self.patience = patience
|
|
47
61
|
self.factor = factor
|
|
@@ -49,6 +63,7 @@ class AdamWReduceLROnPlateau(AbstractOptimizer):
|
|
|
49
63
|
self.opt_kwargs = opt_kwargs
|
|
50
64
|
|
|
51
65
|
def __call__(self, model_parameters):
|
|
66
|
+
"""Return optimizer"""
|
|
52
67
|
opt = torch.optim.AdamW(model_parameters, lr=self.lr, **self.opt_kwargs)
|
|
53
68
|
sch = torch.optim.lr_scheduler.ReduceLROnPlateau(
|
|
54
69
|
opt,
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
"""Training"""
|
|
2
|
+
|
|
1
3
|
from typing import Optional
|
|
2
4
|
|
|
3
5
|
import hydra
|
|
@@ -19,7 +21,7 @@ log = utils.get_logger(__name__)
|
|
|
19
21
|
torch.set_default_dtype(torch.float32)
|
|
20
22
|
|
|
21
23
|
|
|
22
|
-
def
|
|
24
|
+
def _callbacks_to_phase(callbacks, phase):
|
|
23
25
|
for c in callbacks:
|
|
24
26
|
if hasattr(c, "switch_phase"):
|
|
25
27
|
c.switch_phase(phase)
|
|
@@ -27,6 +29,7 @@ def callbacks_to_phase(callbacks, phase):
|
|
|
27
29
|
|
|
28
30
|
def train(config: DictConfig) -> Optional[float]:
|
|
29
31
|
"""Contains training pipeline.
|
|
32
|
+
|
|
30
33
|
Instantiates all PyTorch Lightning objects from config.
|
|
31
34
|
|
|
32
35
|
Args:
|
|
@@ -69,7 +72,7 @@ def train(config: DictConfig) -> Optional[float]:
|
|
|
69
72
|
should_pretrain |= hasattr(c, "training_phase") and c.training_phase == "pretrain"
|
|
70
73
|
|
|
71
74
|
if should_pretrain:
|
|
72
|
-
|
|
75
|
+
_callbacks_to_phase(callbacks, "pretrain")
|
|
73
76
|
|
|
74
77
|
trainer: Trainer = hydra.utils.instantiate(
|
|
75
78
|
config.trainer,
|
|
@@ -83,7 +86,7 @@ def train(config: DictConfig) -> Optional[float]:
|
|
|
83
86
|
datamodule.block_nwp_and_sat = True
|
|
84
87
|
trainer.fit(model=model, datamodule=datamodule)
|
|
85
88
|
|
|
86
|
-
|
|
89
|
+
_callbacks_to_phase(callbacks, "main")
|
|
87
90
|
|
|
88
91
|
datamodule.block_nwp_and_sat = False
|
|
89
92
|
trainer.should_stop = False
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
"""Utils"""
|
|
2
|
+
|
|
1
3
|
import logging
|
|
2
4
|
import os
|
|
3
5
|
import warnings
|
|
@@ -60,7 +62,9 @@ def get_logger(name=__name__, level=logging.INFO) -> logging.Logger:
|
|
|
60
62
|
|
|
61
63
|
|
|
62
64
|
def extras(config: DictConfig) -> None:
|
|
63
|
-
"""A couple of optional utilities
|
|
65
|
+
"""A couple of optional utilities.
|
|
66
|
+
|
|
67
|
+
Controlled by main config file:
|
|
64
68
|
- disabling warnings
|
|
65
69
|
- easier access to debug mode
|
|
66
70
|
- forcing debug friendly configuration
|
|
@@ -143,6 +147,7 @@ def print_config(
|
|
|
143
147
|
|
|
144
148
|
|
|
145
149
|
def empty(*args, **kwargs):
|
|
150
|
+
"""Returns nothing"""
|
|
146
151
|
pass
|
|
147
152
|
|
|
148
153
|
|
|
@@ -209,6 +214,8 @@ def finish(
|
|
|
209
214
|
|
|
210
215
|
|
|
211
216
|
def plot_batch_forecasts(batch, y_hat, batch_idx=None):
|
|
217
|
+
"""Plot a batch of data and the forecast from that batch"""
|
|
218
|
+
|
|
212
219
|
def _get_numpy(key):
|
|
213
220
|
return batch[key].cpu().numpy().squeeze()
|
|
214
221
|
|
|
@@ -254,6 +261,8 @@ def plot_batch_forecasts(batch, y_hat, batch_idx=None):
|
|
|
254
261
|
|
|
255
262
|
|
|
256
263
|
def construct_ocf_ml_metrics_batch_df(batch, y, y_hat):
|
|
264
|
+
"""Helper function tot construct DataFrame for ocf_ml_metrics"""
|
|
265
|
+
|
|
257
266
|
def _repeat(x):
|
|
258
267
|
return np.repeat(x.squeeze(), n_times)
|
|
259
268
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
nowcasting_utils
|
|
2
|
+
nowcasting_datamodel
|
|
2
3
|
ocf_datapipes
|
|
3
4
|
ocf_ml_metrics
|
|
4
5
|
numpy
|
|
@@ -9,12 +10,15 @@ ipykernel
|
|
|
9
10
|
h5netcdf
|
|
10
11
|
torch>=2.0
|
|
11
12
|
lightning>=2.0.1
|
|
13
|
+
torchdata
|
|
12
14
|
pytest
|
|
13
15
|
pytest-cov
|
|
14
|
-
|
|
16
|
+
typer
|
|
17
|
+
sqlalchemy
|
|
15
18
|
jedi
|
|
16
19
|
tables
|
|
17
20
|
tilemapbase
|
|
21
|
+
testcontainers
|
|
18
22
|
wandb
|
|
19
23
|
tensorboard
|
|
20
24
|
tqdm
|
|
@@ -23,4 +27,3 @@ omegaconf
|
|
|
23
27
|
hydra-core
|
|
24
28
|
python-dotenv
|
|
25
29
|
hydra-optuna-sweeper
|
|
26
|
-
black
|