solarian-diffusion 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_policy/__init__.py +0 -0
- diffusion_policy/codecs/__init__.py +0 -0
- diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
- diffusion_policy/common/__init__.py +0 -0
- diffusion_policy/common/checkpoint_util.py +64 -0
- diffusion_policy/common/json_logger.py +117 -0
- diffusion_policy/common/normalize_util.py +251 -0
- diffusion_policy/common/pose_repr_util.py +112 -0
- diffusion_policy/common/pytorch_util.py +82 -0
- diffusion_policy/common/q_critic_util.py +135 -0
- diffusion_policy/common/replay_buffer.py +672 -0
- diffusion_policy/common/sampler.py +204 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
- diffusion_policy/config/legacy/task/can_image.yaml +64 -0
- diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
- diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
- diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
- diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
- diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
- diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/square_image.yaml +64 -0
- diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
- diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
- diffusion_policy/config/legacy/task/umi.yaml +45 -0
- diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
- diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
- diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
- diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
- diffusion_policy/config/rl_annotate.yaml +265 -0
- diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
- diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
- diffusion_policy/config/rl_annotate_v2.yaml +141 -0
- diffusion_policy/config/rl_annotate_v3.yaml +165 -0
- diffusion_policy/config/task/lift.yaml +89 -0
- diffusion_policy/config/task/square.yaml +92 -0
- diffusion_policy/config/task/tool_hang.yaml +93 -0
- diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/task/umi.yaml +105 -0
- diffusion_policy/config/task/umi_bimanual.yaml +135 -0
- diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
- diffusion_policy/config/task/umi_image.yaml +36 -0
- diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
- diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
- diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
- diffusion_policy/config/train_iql_umi.yaml +186 -0
- diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
- diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
- diffusion_policy/dataset/__init__.py +0 -0
- diffusion_policy/dataset/base_dataset.py +76 -0
- diffusion_policy/dataset/umi_dataset.py +652 -0
- diffusion_policy/env_runner/__init__.py +0 -0
- diffusion_policy/env_runner/base_image_runner.py +9 -0
- diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
- diffusion_policy/model/__init__.py +0 -0
- diffusion_policy/model/common/__init__.py +0 -0
- diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
- diffusion_policy/model/common/lr_scheduler.py +46 -0
- diffusion_policy/model/common/module_attr_mixin.py +14 -0
- diffusion_policy/model/common/normalizer.py +353 -0
- diffusion_policy/model/common/tensor_util.py +960 -0
- diffusion_policy/model/diffusion/__init__.py +0 -0
- diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
- diffusion_policy/model/diffusion/conv1d_components.py +46 -0
- diffusion_policy/model/diffusion/ema_model.py +88 -0
- diffusion_policy/model/diffusion/mask_generator.py +221 -0
- diffusion_policy/model/diffusion/positional_embedding.py +17 -0
- diffusion_policy/model/vision/__init__.py +0 -0
- diffusion_policy/model/vision/crop_randomizer.py +288 -0
- diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
- diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
- diffusion_policy/policy/__init__.py +0 -0
- diffusion_policy/policy/base_image_policy.py +27 -0
- diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
- diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
- diffusion_policy/policy/parl_action_optimizer.py +675 -0
- diffusion_policy/workspace/__init__.py +0 -0
- diffusion_policy/workspace/base_workspace.py +146 -0
- diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
- solarian_diffusion/__init__.py +32 -0
- solarian_diffusion/config.py +31 -0
- solarian_diffusion/policy_server.py +316 -0
- solarian_diffusion/rtc.py +272 -0
- solarian_diffusion/service.py +16 -0
- solarian_diffusion/trainer.py +213 -0
- solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
- solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
- solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
- solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
- solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
- solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
- solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
- umi/__init__.py +0 -0
- umi/common/__init__.py +0 -0
- umi/common/pose_util.py +115 -0
|
@@ -0,0 +1,1585 @@
|
|
|
1
|
+
# Stringified annotations: lets us keep type hints like
|
|
2
|
+
# `self.model: DiffusionUnetImagePolicy` without paying the import cost of
|
|
3
|
+
# `DiffusionUnetImagePolicy` (and friends) at module load. They'll be
|
|
4
|
+
# imported lazily inside `run()` where the runtime values actually matter.
|
|
5
|
+
# Required for the lazy-import block below to be safe across the file.
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
if __name__ == "__main__":
|
|
9
|
+
import sys
|
|
10
|
+
import os
|
|
11
|
+
import pathlib
|
|
12
|
+
|
|
13
|
+
ROOT_DIR = str(pathlib.Path(__file__).parent.parent.parent)
|
|
14
|
+
sys.path.append(ROOT_DIR)
|
|
15
|
+
os.chdir(ROOT_DIR)
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
import hydra
|
|
19
|
+
import torch
|
|
20
|
+
from omegaconf import OmegaConf
|
|
21
|
+
import pathlib
|
|
22
|
+
import copy
|
|
23
|
+
import random
|
|
24
|
+
import numpy as np
|
|
25
|
+
from diffusion_policy.workspace.base_workspace import BaseWorkspace
|
|
26
|
+
|
|
27
|
+
# NOTE: the following imports were previously at module top-level. They
|
|
28
|
+
# are required only by `run()` (training), not by `__init__()` or the
|
|
29
|
+
# inference-time accessors. Importing them at module top adds several
|
|
30
|
+
# seconds to cold-start of the policy server (notably `wandb` and the
|
|
31
|
+
# transitive `accelerate` / dataset / env_runner stack), so we defer
|
|
32
|
+
# them. The `if False:` block is kept to make the migration discoverable
|
|
33
|
+
# to static analyzers / IDEs: it documents what used to be here but is
|
|
34
|
+
# never executed at runtime.
|
|
35
|
+
if False: # pragma: no cover - tooling hint only
|
|
36
|
+
from torch.utils.data import DataLoader # noqa: F401
|
|
37
|
+
import wandb # noqa: F401
|
|
38
|
+
import pickle # noqa: F401
|
|
39
|
+
import tqdm # noqa: F401
|
|
40
|
+
import shutil # noqa: F401
|
|
41
|
+
from diffusion_policy.policy.diffusion_unet_image_policy import DiffusionUnetImagePolicy # noqa: F401
|
|
42
|
+
from diffusion_policy.dataset.base_dataset import BaseImageDataset, BaseDataset # noqa: F401
|
|
43
|
+
from diffusion_policy.env_runner.base_image_runner import BaseImageRunner # noqa: F401
|
|
44
|
+
from diffusion_policy.common.checkpoint_util import TopKCheckpointManager # noqa: F401
|
|
45
|
+
from diffusion_policy.common.json_logger import JsonLogger # noqa: F401
|
|
46
|
+
from diffusion_policy.common.pytorch_util import dict_apply, optimizer_to # noqa: F401
|
|
47
|
+
from diffusion_policy.model.diffusion.ema_model import EMAModel # noqa: F401
|
|
48
|
+
from diffusion_policy.model.common.lr_scheduler import get_scheduler # noqa: F401
|
|
49
|
+
from accelerate import Accelerator # noqa: F401
|
|
50
|
+
|
|
51
|
+
OmegaConf.register_new_resolver("eval", eval, replace=True)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@torch.no_grad()
|
|
55
|
+
def _awr_metrics(policy) -> dict:
|
|
56
|
+
"""Pass-through of the policy's per-batch AWR diagnostics.
|
|
57
|
+
|
|
58
|
+
``DiffusionUnetTimmPolicy.compute_loss`` populates
|
|
59
|
+
``last_awr_info`` after every forward pass when AWR is active. We
|
|
60
|
+
just forward those scalars to wandb, prefixed under ``awr/``. When
|
|
61
|
+
AWR is off this returns an empty dict (so the metrics never appear
|
|
62
|
+
on the BC training run's wandb panel and don't pollute resumes).
|
|
63
|
+
|
|
64
|
+
Filters to ``awr/*`` keys only so PA-RL diagnostics
|
|
65
|
+
(``parl/*``) don't get double-emitted by both this and
|
|
66
|
+
``_parl_metrics`` -- they share ``last_awr_info`` since both
|
|
67
|
+
write into it from compute_loss.
|
|
68
|
+
"""
|
|
69
|
+
info = getattr(policy, 'last_awr_info', None)
|
|
70
|
+
if not info:
|
|
71
|
+
return {}
|
|
72
|
+
return {k: v for k, v in info.items() if k.startswith('awr/')}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@torch.no_grad()
|
|
76
|
+
def _parl_metrics(policy) -> dict:
|
|
77
|
+
"""Pass-through of the policy's per-batch PA-RL diagnostics.
|
|
78
|
+
|
|
79
|
+
``DiffusionUnetTimmPolicy.compute_loss`` populates
|
|
80
|
+
``last_awr_info`` with ``parl/*`` keys when the batch carries
|
|
81
|
+
optimized actions; this just forwards them to wandb. Empty dict
|
|
82
|
+
when PA-RL is off so the metrics never appear on AWR / BC runs.
|
|
83
|
+
"""
|
|
84
|
+
info = getattr(policy, 'last_awr_info', None)
|
|
85
|
+
if not info:
|
|
86
|
+
return {}
|
|
87
|
+
return {k: v for k, v in info.items() if k.startswith('parl/')}
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@torch.no_grad()
|
|
91
|
+
def _bbox_mask_weight_metrics(policy) -> dict:
|
|
92
|
+
"""Snapshot of patch_embed.proj weight norms for bbox-mask-channel rgb keys.
|
|
93
|
+
|
|
94
|
+
Returns one block of metrics per rgb key that has a bbox mask routed to
|
|
95
|
+
it (see TimmObsEncoder.bbox_mask_map). For each key we log:
|
|
96
|
+
|
|
97
|
+
* ``bbox_mask/<rgb_key>/w_norm_mask_ch``: L2 norm of the 4th input-
|
|
98
|
+
channel slice of patch_embed.proj. Exactly 0 at training step 0
|
|
99
|
+
(zero-init); growing > 0 proves the optimizer is using the mask.
|
|
100
|
+
* ``bbox_mask/<rgb_key>/w_norm_rgb_mean``: mean L2 norm of the three
|
|
101
|
+
RGB input-channel slices. Stable reference scale to compare against.
|
|
102
|
+
* ``bbox_mask/<rgb_key>/w_ratio``: ratio of the two (mask norm /
|
|
103
|
+
mean rgb norm). A "healthy" trained value is roughly 0.05 - 1.0;
|
|
104
|
+
values close to 0 mean the optimizer declined to use the mask
|
|
105
|
+
(almost certainly because the training data didn't reward it).
|
|
106
|
+
|
|
107
|
+
Returns an empty dict when no bbox is routed via the mask channel
|
|
108
|
+
(e.g. ``bbox_mode: low_dim`` runs), so it's safe to call unconditionally.
|
|
109
|
+
"""
|
|
110
|
+
encoder = getattr(policy, 'obs_encoder', None)
|
|
111
|
+
if encoder is None:
|
|
112
|
+
return {}
|
|
113
|
+
mask_map = getattr(encoder, 'bbox_mask_map', None)
|
|
114
|
+
if not mask_map:
|
|
115
|
+
return {}
|
|
116
|
+
|
|
117
|
+
metrics = {}
|
|
118
|
+
for rgb_key in mask_map.keys():
|
|
119
|
+
# nn.ModuleDict doesn't expose .get(); use __contains__ + []
|
|
120
|
+
if rgb_key not in encoder.key_model_map:
|
|
121
|
+
continue
|
|
122
|
+
model = encoder.key_model_map[rgb_key]
|
|
123
|
+
if not hasattr(model, 'patch_embed') \
|
|
124
|
+
or not hasattr(model.patch_embed, 'proj'):
|
|
125
|
+
continue
|
|
126
|
+
weight = model.patch_embed.proj.weight # (out_ch, in_ch=4, k, k)
|
|
127
|
+
if weight.shape[1] < 4:
|
|
128
|
+
continue
|
|
129
|
+
rgb_norms = [weight[:, c].norm().item() for c in range(3)]
|
|
130
|
+
mask_norm = weight[:, 3].norm().item()
|
|
131
|
+
rgb_mean = sum(rgb_norms) / 3.0
|
|
132
|
+
ratio = (mask_norm / rgb_mean) if rgb_mean > 0 else 0.0
|
|
133
|
+
prefix = f'bbox_mask/{rgb_key}'
|
|
134
|
+
metrics[f'{prefix}/w_norm_mask_ch'] = mask_norm
|
|
135
|
+
metrics[f'{prefix}/w_norm_rgb_mean'] = rgb_mean
|
|
136
|
+
metrics[f'{prefix}/w_ratio'] = ratio
|
|
137
|
+
return metrics
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class TrainDiffusionUnetImageWorkspace(BaseWorkspace):
|
|
141
|
+
include_keys = ['global_step', 'epoch']
|
|
142
|
+
exclude_keys = tuple()
|
|
143
|
+
|
|
144
|
+
def __init__(self, cfg: OmegaConf, output_dir=None):
|
|
145
|
+
super().__init__(cfg, output_dir=output_dir)
|
|
146
|
+
|
|
147
|
+
# set seed
|
|
148
|
+
seed = cfg.training.seed
|
|
149
|
+
torch.manual_seed(seed)
|
|
150
|
+
np.random.seed(seed)
|
|
151
|
+
random.seed(seed)
|
|
152
|
+
|
|
153
|
+
# configure model
|
|
154
|
+
self.model: DiffusionUnetImagePolicy = hydra.utils.instantiate(cfg.policy)
|
|
155
|
+
|
|
156
|
+
self.ema_model: DiffusionUnetImagePolicy = None
|
|
157
|
+
if cfg.training.use_ema:
|
|
158
|
+
self.ema_model = copy.deepcopy(self.model)
|
|
159
|
+
|
|
160
|
+
# configure training state
|
|
161
|
+
# self.optimizer = hydra.utils.instantiate(
|
|
162
|
+
# cfg.optimizer, params=self.model.parameters())
|
|
163
|
+
|
|
164
|
+
obs_encorder_lr = cfg.optimizer.lr
|
|
165
|
+
if cfg.policy.obs_encoder.pretrained:
|
|
166
|
+
obs_encorder_lr *= 0.1
|
|
167
|
+
print('==> reduce pretrained obs_encorder\'s lr')
|
|
168
|
+
obs_encorder_params = list()
|
|
169
|
+
for param in self.model.obs_encoder.parameters():
|
|
170
|
+
if param.requires_grad:
|
|
171
|
+
obs_encorder_params.append(param)
|
|
172
|
+
print(f'obs_encorder params: {len(obs_encorder_params)}')
|
|
173
|
+
param_groups = [
|
|
174
|
+
{'params': self.model.model.parameters()},
|
|
175
|
+
{'params': obs_encorder_params, 'lr': obs_encorder_lr}
|
|
176
|
+
]
|
|
177
|
+
# self.optimizer = hydra.utils.instantiate(
|
|
178
|
+
# cfg.optimizer, params=param_groups)
|
|
179
|
+
optimizer_cfg = OmegaConf.to_container(cfg.optimizer, resolve=True)
|
|
180
|
+
optimizer_cfg.pop('_target_')
|
|
181
|
+
self.optimizer = torch.optim.AdamW(
|
|
182
|
+
params=param_groups,
|
|
183
|
+
**optimizer_cfg
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
# configure training state
|
|
187
|
+
self.global_step = 0
|
|
188
|
+
self.epoch = 0
|
|
189
|
+
|
|
190
|
+
# PA-RL: frozen IQL critic is materialized lazily by run() once
|
|
191
|
+
# the device is settled. Kept as None on workspaces that don't
|
|
192
|
+
# opt in, so legacy AWR / BC runs pay nothing.
|
|
193
|
+
self.q_critic = None
|
|
194
|
+
|
|
195
|
+
# PA-RL: optional one-time cache of ``critic.state_embed`` for
|
|
196
|
+
# every dataset frame, built once after the (frozen) critic is
|
|
197
|
+
# loaded and reused across all relabel cycles. Stored as a
|
|
198
|
+
# CPU tensor of shape ``(N_frames, D_state)`` so per-batch
|
|
199
|
+
# lookup is a cheap index_select; ``valid`` flags rows whose
|
|
200
|
+
# anchor_idx couldn't be resolved during the build pass.
|
|
201
|
+
# ``None`` on workspaces that don't opt in (``parl.cache_state_embed=false``).
|
|
202
|
+
# Both fields are ``Optional[torch.Tensor]`` once populated by run().
|
|
203
|
+
self._parl_state_embed_cache = None
|
|
204
|
+
self._parl_state_embed_valid = None
|
|
205
|
+
|
|
206
|
+
# PA-RL: optional one-time cache of the proposer's
|
|
207
|
+
# ``obs_encoder(nobs)`` output for every dataset frame. Built
|
|
208
|
+
# ONCE after the (frozen) proposer encoder is loaded and reused
|
|
209
|
+
# across ALL relabel cycles. Eliminates the second ViT-L
|
|
210
|
+
# forward in propose_actions (the proposer's encoder is the
|
|
211
|
+
# SAME size as the critic's), so combined with the state_embed
|
|
212
|
+
# cache the relabel pass does ZERO ViT-L forwards per iter --
|
|
213
|
+
# all encoder work is amortized into the one-time precompute.
|
|
214
|
+
# Only valid when ``training.freeze_encoder=true``;
|
|
215
|
+
# automatically disabled otherwise.
|
|
216
|
+
self._parl_global_cond_cache = None
|
|
217
|
+
self._parl_global_cond_valid = None
|
|
218
|
+
|
|
219
|
+
# do not save optimizer if resume=False
|
|
220
|
+
if not cfg.training.resume:
|
|
221
|
+
self.exclude_keys = ['optimizer']
|
|
222
|
+
|
|
223
|
+
def _precompute_obs_feature_cache(
|
|
224
|
+
self,
|
|
225
|
+
*,
|
|
226
|
+
train_dataset,
|
|
227
|
+
val_dataset,
|
|
228
|
+
forward_fn,
|
|
229
|
+
cache_label: str,
|
|
230
|
+
n_frames: int,
|
|
231
|
+
device,
|
|
232
|
+
accelerator,
|
|
233
|
+
parl_cfg,
|
|
234
|
+
):
|
|
235
|
+
"""Generic helper: scan train + val once and cache an
|
|
236
|
+
``(N_frames, D)`` tensor produced by a frozen encoder via
|
|
237
|
+
``forward_fn(obs_dict_on_device) -> (B, D)``.
|
|
238
|
+
|
|
239
|
+
Both ``_precompute_state_embed_cache`` and
|
|
240
|
+
``_precompute_global_cond_cache`` use this skeleton; only the
|
|
241
|
+
forward function differs.
|
|
242
|
+
|
|
243
|
+
Notes
|
|
244
|
+
-----
|
|
245
|
+
- Augmentation realism: the dataset's ``__getitem__`` injects
|
|
246
|
+
random start-pose noise (``umi_dataset.py`` ~L538) and the
|
|
247
|
+
obs encoder may apply RandomCrop / ColorJitter. The cache
|
|
248
|
+
therefore stores ONE realization of those random augs per
|
|
249
|
+
frame. This matches the noise budget the encoder's owner
|
|
250
|
+
(IQL critic / BC policy) was trained against (each train
|
|
251
|
+
step also sees a single random aug per frame), so a single
|
|
252
|
+
realization is statistically equivalent to a fresh draw.
|
|
253
|
+
- Validity: any sample whose ``anchor_idx`` is ``-1`` (sampler
|
|
254
|
+
sentinel) leaves its cache row uninitialized; ``valid``
|
|
255
|
+
flags it false and the relabel pass falls back to
|
|
256
|
+
recomputing on the fly for that batch.
|
|
257
|
+
|
|
258
|
+
Args:
|
|
259
|
+
forward_fn: callable taking an ``obs_dict`` of tensors
|
|
260
|
+
on ``device`` and returning a single ``(B, D)``
|
|
261
|
+
tensor on the same device. Wrapped externally in
|
|
262
|
+
``torch.no_grad`` + ``accelerator.autocast``; do not
|
|
263
|
+
wrap inside the callable.
|
|
264
|
+
cache_label: short human label used in log lines and the
|
|
265
|
+
tqdm description (e.g. ``"state_embed"`` or
|
|
266
|
+
``"global_cond"``).
|
|
267
|
+
|
|
268
|
+
Returns:
|
|
269
|
+
``(cache, valid)`` -- pinned CPU ``(N_frames, D)`` tensor
|
|
270
|
+
and ``(N_frames,)`` bool mask.
|
|
271
|
+
"""
|
|
272
|
+
from torch.utils.data import DataLoader
|
|
273
|
+
import time as _time
|
|
274
|
+
import tqdm as _tqdm
|
|
275
|
+
from diffusion_policy.common.pytorch_util import dict_apply
|
|
276
|
+
|
|
277
|
+
precompute_bs = int(parl_cfg.get(
|
|
278
|
+
'precompute_batch_size',
|
|
279
|
+
parl_cfg.get('relabel_batch_size', 16),
|
|
280
|
+
))
|
|
281
|
+
precompute_workers = int(parl_cfg.get(
|
|
282
|
+
'precompute_num_workers',
|
|
283
|
+
parl_cfg.get('relabel_num_workers', 4),
|
|
284
|
+
))
|
|
285
|
+
|
|
286
|
+
# Discover the output dim by running ONE tiny batch. Avoids
|
|
287
|
+
# hardcoding the architecture's hidden dim in the workspace
|
|
288
|
+
# and keeps this method drop-in for any future encoder.
|
|
289
|
+
probe = train_dataset[0]
|
|
290
|
+
probe_obs = dict_apply(
|
|
291
|
+
probe['obs'],
|
|
292
|
+
lambda x: x.unsqueeze(0).to(device, non_blocking=True),
|
|
293
|
+
)
|
|
294
|
+
with torch.no_grad(), accelerator.autocast():
|
|
295
|
+
probe_emb = forward_fn(probe_obs)
|
|
296
|
+
D = int(probe_emb.shape[-1])
|
|
297
|
+
cache_dtype = probe_emb.dtype # bf16 under autocast on A100
|
|
298
|
+
del probe, probe_obs, probe_emb
|
|
299
|
+
|
|
300
|
+
# GPU-resident cache. We considered pinned CPU tensors here
|
|
301
|
+
# but ran into NaN values reading back from pinned bf16
|
|
302
|
+
# tensors on this stack -- pytorch's pin_memory has had
|
|
303
|
+
# subtle bf16 issues across versions. GPU storage fits
|
|
304
|
+
# comfortably (a 200k-frame, D=1024 bf16 cache is ~410 MB)
|
|
305
|
+
# AND eliminates the per-batch H2D copy, so this is the
|
|
306
|
+
# cleaner default. The relabel pass is single-rank in v1,
|
|
307
|
+
# so only the main-process GPU holds the cache.
|
|
308
|
+
cache = torch.zeros(
|
|
309
|
+
(n_frames, D), dtype=cache_dtype, device=device)
|
|
310
|
+
valid = torch.zeros(
|
|
311
|
+
(n_frames,), dtype=torch.bool, device=device)
|
|
312
|
+
|
|
313
|
+
splits = []
|
|
314
|
+
if len(train_dataset) > 0:
|
|
315
|
+
splits.append(('train', train_dataset))
|
|
316
|
+
if val_dataset is not None and len(val_dataset) > 0:
|
|
317
|
+
splits.append(('val', val_dataset))
|
|
318
|
+
|
|
319
|
+
n_seen = 0
|
|
320
|
+
t0_total = _time.perf_counter()
|
|
321
|
+
for split_name, ds in splits:
|
|
322
|
+
# Drop any in-memory action_optimized so the dataset
|
|
323
|
+
# serves the BASE obs (action_optimized doesn't actually
|
|
324
|
+
# change obs but we mirror the relabel pass's defensive
|
|
325
|
+
# save/restore for symmetry).
|
|
326
|
+
saved_actions = ds.action_optimized
|
|
327
|
+
saved_valid = ds.action_optimized_valid
|
|
328
|
+
ds.action_optimized = None
|
|
329
|
+
ds.action_optimized_valid = None
|
|
330
|
+
try:
|
|
331
|
+
loader = DataLoader(
|
|
332
|
+
ds,
|
|
333
|
+
batch_size=precompute_bs,
|
|
334
|
+
shuffle=False,
|
|
335
|
+
num_workers=precompute_workers,
|
|
336
|
+
pin_memory=True,
|
|
337
|
+
persistent_workers=False,
|
|
338
|
+
drop_last=False,
|
|
339
|
+
)
|
|
340
|
+
accelerator.print(
|
|
341
|
+
f"[PA-RL] precompute {cache_label} cache for {split_name}: "
|
|
342
|
+
f"{len(ds)} samples / {len(loader)} batches "
|
|
343
|
+
f"(D={D}, dtype={cache_dtype})"
|
|
344
|
+
)
|
|
345
|
+
t0 = _time.perf_counter()
|
|
346
|
+
pbar = _tqdm.tqdm(
|
|
347
|
+
loader,
|
|
348
|
+
total=len(loader),
|
|
349
|
+
desc=f"PA-RL {cache_label} {split_name}",
|
|
350
|
+
dynamic_ncols=True,
|
|
351
|
+
mininterval=1.0,
|
|
352
|
+
leave=False,
|
|
353
|
+
disable=not accelerator.is_main_process,
|
|
354
|
+
)
|
|
355
|
+
for batch in pbar:
|
|
356
|
+
obs = dict_apply(
|
|
357
|
+
batch['obs'],
|
|
358
|
+
lambda x: x.to(device, non_blocking=True),
|
|
359
|
+
)
|
|
360
|
+
anchor_idx_t = batch['anchor_idx'].to(
|
|
361
|
+
device, non_blocking=True).long()
|
|
362
|
+
with torch.no_grad(), accelerator.autocast():
|
|
363
|
+
emb = forward_fn(obs)
|
|
364
|
+
# Vectorized scatter on GPU: ``cache`` lives on
|
|
365
|
+
# ``device`` so we never round-trip through CPU.
|
|
366
|
+
# ``in_range_t`` filters out the sentinel
|
|
367
|
+
# anchor_idx == -1 rows (sampler emits this when
|
|
368
|
+
# the global frame can't be resolved); those rows
|
|
369
|
+
# stay zero in the cache and ``valid`` flags them
|
|
370
|
+
# false so the relabel pass falls back to fresh
|
|
371
|
+
# compute for that batch.
|
|
372
|
+
in_range_t = (anchor_idx_t >= 0) & (
|
|
373
|
+
anchor_idx_t < n_frames)
|
|
374
|
+
if bool(in_range_t.any().item()):
|
|
375
|
+
valid_anchors_t = anchor_idx_t[in_range_t]
|
|
376
|
+
cache[valid_anchors_t] = emb.detach()[in_range_t]
|
|
377
|
+
valid[valid_anchors_t] = True
|
|
378
|
+
n_seen += int(emb.shape[0])
|
|
379
|
+
pbar.close()
|
|
380
|
+
dt = _time.perf_counter() - t0
|
|
381
|
+
accelerator.print(
|
|
382
|
+
f"[PA-RL] {cache_label} {split_name} done in {dt:.1f}s "
|
|
383
|
+
f"({len(ds) / max(dt, 1e-3):.1f} samples/s)"
|
|
384
|
+
)
|
|
385
|
+
finally:
|
|
386
|
+
ds.action_optimized = saved_actions
|
|
387
|
+
ds.action_optimized_valid = saved_valid
|
|
388
|
+
|
|
389
|
+
n_valid = int(valid.sum().item())
|
|
390
|
+
total_dt = _time.perf_counter() - t0_total
|
|
391
|
+
# Sanity check: any non-finite values in the cache silently
|
|
392
|
+
# propagate to NaN Q-values during relabel and look like the
|
|
393
|
+
# critic has gone insane. Cheap to check once at build time.
|
|
394
|
+
# Only inspect the valid rows (zeros in unused rows are fine).
|
|
395
|
+
if n_valid > 0:
|
|
396
|
+
valid_rows = cache[valid]
|
|
397
|
+
n_bad = int((~torch.isfinite(valid_rows)).any(dim=-1).sum().item())
|
|
398
|
+
if n_bad > 0:
|
|
399
|
+
# Hard-fail rather than silently produce NaN Q-values:
|
|
400
|
+
# there is no recovery from a corrupt feature cache.
|
|
401
|
+
raise RuntimeError(
|
|
402
|
+
f"[PA-RL] {cache_label} cache has {n_bad}/{n_valid} "
|
|
403
|
+
f"valid rows with non-finite values (NaN/Inf). "
|
|
404
|
+
f"This usually means the encoder ran into bf16 "
|
|
405
|
+
f"overflow on this stack. Try setting "
|
|
406
|
+
f"parl.cache_{cache_label}=false to disable the "
|
|
407
|
+
f"cache, or upgrade pytorch."
|
|
408
|
+
)
|
|
409
|
+
accelerator.print(
|
|
410
|
+
f"[PA-RL] {cache_label} cache built in {total_dt:.1f}s: "
|
|
411
|
+
f"{n_valid}/{n_frames} frames valid "
|
|
412
|
+
f"({100.0 * n_valid / max(n_frames, 1):.1f}%), "
|
|
413
|
+
f"{cache.element_size() * cache.nelement() / 1e6:.1f} MB on "
|
|
414
|
+
f"{cache.device}"
|
|
415
|
+
)
|
|
416
|
+
return cache, valid
|
|
417
|
+
|
|
418
|
+
def _precompute_state_embed_cache(
|
|
419
|
+
self,
|
|
420
|
+
*,
|
|
421
|
+
train_dataset,
|
|
422
|
+
val_dataset,
|
|
423
|
+
critic,
|
|
424
|
+
n_frames: int,
|
|
425
|
+
device,
|
|
426
|
+
accelerator,
|
|
427
|
+
parl_cfg,
|
|
428
|
+
):
|
|
429
|
+
"""Precompute and cache ``critic.state_embed`` for every dataset frame.
|
|
430
|
+
|
|
431
|
+
The IQL critic's obs encoder is FROZEN throughout PA-RL fine-
|
|
432
|
+
tuning (``parl.freeze_critic=true``), so its state embedding
|
|
433
|
+
for any given frame is constant across the entire run. Each
|
|
434
|
+
relabel cycle replaces the per-batch
|
|
435
|
+
``critic.state_embed(obs_dict)`` call (a heavy ViT-L forward)
|
|
436
|
+
with an O(B) ``index_select``.
|
|
437
|
+
"""
|
|
438
|
+
critic.eval()
|
|
439
|
+
return self._precompute_obs_feature_cache(
|
|
440
|
+
train_dataset=train_dataset,
|
|
441
|
+
val_dataset=val_dataset,
|
|
442
|
+
forward_fn=lambda obs: critic.state_embed(obs, target=False),
|
|
443
|
+
cache_label="state_embed",
|
|
444
|
+
n_frames=n_frames,
|
|
445
|
+
device=device,
|
|
446
|
+
accelerator=accelerator,
|
|
447
|
+
parl_cfg=parl_cfg,
|
|
448
|
+
)
|
|
449
|
+
|
|
450
|
+
def _precompute_global_cond_cache(
|
|
451
|
+
self,
|
|
452
|
+
*,
|
|
453
|
+
train_dataset,
|
|
454
|
+
val_dataset,
|
|
455
|
+
proposer,
|
|
456
|
+
n_frames: int,
|
|
457
|
+
device,
|
|
458
|
+
accelerator,
|
|
459
|
+
parl_cfg,
|
|
460
|
+
):
|
|
461
|
+
"""Precompute and cache the proposer's ``obs_encoder(nobs)`` output.
|
|
462
|
+
|
|
463
|
+
When ``training.freeze_encoder=True`` the policy's (and EMA
|
|
464
|
+
proposer's) ViT-L visual encoder is frozen for the entire
|
|
465
|
+
PA-RL run. Its output for any given frame is therefore
|
|
466
|
+
constant, so we amortize one full pass over the dataset and
|
|
467
|
+
afterwards every relabel cycle skips the heavy
|
|
468
|
+
``proposer.obs_encoder(nobs)`` forward inside ``propose_actions``,
|
|
469
|
+
replacing it with an O(B) ``index_select``. This is the same
|
|
470
|
+
optimization as ``_precompute_state_embed_cache`` but for the
|
|
471
|
+
SECOND big ViT-L the relabel pass would otherwise run.
|
|
472
|
+
|
|
473
|
+
The dim D_cond depends on the policy's
|
|
474
|
+
``timm_obs_encoder.output_shape()`` and is discovered at
|
|
475
|
+
runtime by probing the first sample, so the helper works
|
|
476
|
+
unchanged for any future encoder shape.
|
|
477
|
+
"""
|
|
478
|
+
proposer.eval()
|
|
479
|
+
|
|
480
|
+
def _forward(obs):
|
|
481
|
+
nobs = proposer.normalizer.normalize(obs)
|
|
482
|
+
return proposer.obs_encoder(nobs)
|
|
483
|
+
|
|
484
|
+
return self._precompute_obs_feature_cache(
|
|
485
|
+
train_dataset=train_dataset,
|
|
486
|
+
val_dataset=val_dataset,
|
|
487
|
+
forward_fn=_forward,
|
|
488
|
+
cache_label="global_cond",
|
|
489
|
+
n_frames=n_frames,
|
|
490
|
+
device=device,
|
|
491
|
+
accelerator=accelerator,
|
|
492
|
+
parl_cfg=parl_cfg,
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
def _parl_relabel(
|
|
496
|
+
self,
|
|
497
|
+
*,
|
|
498
|
+
train_dataset,
|
|
499
|
+
val_dataset,
|
|
500
|
+
proposer,
|
|
501
|
+
critic,
|
|
502
|
+
n_frames: int,
|
|
503
|
+
action_horizon: int,
|
|
504
|
+
action_dim: int,
|
|
505
|
+
device,
|
|
506
|
+
accelerator,
|
|
507
|
+
parl_cfg,
|
|
508
|
+
state_embed_cache=None,
|
|
509
|
+
state_embed_valid=None,
|
|
510
|
+
global_cond_cache=None,
|
|
511
|
+
global_cond_valid=None,
|
|
512
|
+
):
|
|
513
|
+
"""Run ONE PA-RL relabel pass and return ``(actions, valid, info)``.
|
|
514
|
+
|
|
515
|
+
Iterates the train + val datasets sequentially (no shuffle, no
|
|
516
|
+
DDP sharding -- single-rank pass on accelerator's main
|
|
517
|
+
process), proposes K candidates per state from ``proposer``,
|
|
518
|
+
runs PA-RL action optimization against the frozen ``critic``,
|
|
519
|
+
and scatters the chosen actions into a per-frame
|
|
520
|
+
``(n_frames, H, A)`` array indexed by the global anchor
|
|
521
|
+
``anchor_idx`` emitted by ``UmiDataset.__getitem__``.
|
|
522
|
+
|
|
523
|
+
Algorithm 1 + Eq. 4.3 from Sobol Mark et al. 2024.
|
|
524
|
+
Implementation lives in
|
|
525
|
+
``diffusion_policy.policy.parl_action_optimizer``.
|
|
526
|
+
|
|
527
|
+
Returns:
|
|
528
|
+
actions: (n_frames, H, A) float32; rows where valid=False
|
|
529
|
+
are zeros (defensive default; the dataset emits
|
|
530
|
+
``has_action_optimized=0`` for those, so the loss
|
|
531
|
+
falls back to dataset action).
|
|
532
|
+
valid: (n_frames,) bool.
|
|
533
|
+
info: dict of aggregated diagnostics under the ``parl/``
|
|
534
|
+
prefix, suitable for ``accelerator.log``.
|
|
535
|
+
"""
|
|
536
|
+
from torch.utils.data import DataLoader
|
|
537
|
+
import time as _time
|
|
538
|
+
import tqdm as _tqdm
|
|
539
|
+
from diffusion_policy.policy.parl_action_optimizer import optimize_actions
|
|
540
|
+
from diffusion_policy.common.pytorch_util import dict_apply
|
|
541
|
+
|
|
542
|
+
K = int(parl_cfg.get('k_samples', 32))
|
|
543
|
+
m_top = int(parl_cfg.get('m_top', 10))
|
|
544
|
+
T = int(parl_cfg.get('T_grad_steps', 10))
|
|
545
|
+
alpha = float(parl_cfg.get('alpha_grad', 3.0e-4))
|
|
546
|
+
pick_mode = str(parl_cfg.get('distill_mode', 'argmax'))
|
|
547
|
+
softmax_temp = float(parl_cfg.get('softmax_temperature', 1.0))
|
|
548
|
+
include_data_action = bool(parl_cfg.get('include_dataset_action', True))
|
|
549
|
+
proposer_steps = parl_cfg.get('proposer_inference_steps', None)
|
|
550
|
+
if proposer_steps is not None:
|
|
551
|
+
proposer_steps = int(proposer_steps)
|
|
552
|
+
relabel_batch_size = int(parl_cfg.get('relabel_batch_size', 16))
|
|
553
|
+
relabel_workers = int(parl_cfg.get('relabel_num_workers', 4))
|
|
554
|
+
|
|
555
|
+
# Make sure the proposer / critic are in eval mode and on the
|
|
556
|
+
# right device. We never set requires_grad on either (proposer
|
|
557
|
+
# has frozen visual when freeze_encoder=True; critic is
|
|
558
|
+
# explicitly frozen at load time when freeze_critic=True).
|
|
559
|
+
proposer.eval()
|
|
560
|
+
critic.eval()
|
|
561
|
+
|
|
562
|
+
out_actions = np.zeros((n_frames, action_horizon, action_dim),
|
|
563
|
+
dtype=np.float32)
|
|
564
|
+
out_valid = np.zeros(n_frames, dtype=bool)
|
|
565
|
+
|
|
566
|
+
# Single aggregator across train + val for wandb (we don't
|
|
567
|
+
# need to break out per-split numbers in the diagnostics; the
|
|
568
|
+
# split breakdown is in the per-pass log line below).
|
|
569
|
+
agg = {}
|
|
570
|
+
|
|
571
|
+
splits = []
|
|
572
|
+
if len(train_dataset) > 0:
|
|
573
|
+
splits.append(('train', train_dataset))
|
|
574
|
+
if val_dataset is not None and len(val_dataset) > 0:
|
|
575
|
+
splits.append(('val', val_dataset))
|
|
576
|
+
|
|
577
|
+
n_seen = 0
|
|
578
|
+
t0_total = _time.perf_counter()
|
|
579
|
+
for split_name, ds in splits:
|
|
580
|
+
# Disable any cache that the dataset already has so this
|
|
581
|
+
# pass sees the BASE dataset action (not yesterday's
|
|
582
|
+
# optimized chunk masquerading as the dataset). Restored
|
|
583
|
+
# by the caller after the pass when it calls
|
|
584
|
+
# ``set_action_optimized`` with the new array.
|
|
585
|
+
saved_actions = ds.action_optimized
|
|
586
|
+
saved_valid = ds.action_optimized_valid
|
|
587
|
+
ds.action_optimized = None
|
|
588
|
+
ds.action_optimized_valid = None
|
|
589
|
+
try:
|
|
590
|
+
loader = DataLoader(
|
|
591
|
+
ds,
|
|
592
|
+
batch_size=relabel_batch_size,
|
|
593
|
+
shuffle=False,
|
|
594
|
+
num_workers=relabel_workers,
|
|
595
|
+
pin_memory=True,
|
|
596
|
+
persistent_workers=False,
|
|
597
|
+
drop_last=False,
|
|
598
|
+
)
|
|
599
|
+
accelerator.print(
|
|
600
|
+
f"[PA-RL] relabel {split_name}: "
|
|
601
|
+
f"{len(ds)} samples / {len(loader)} batches "
|
|
602
|
+
f"(K={K}, m={m_top}, T={T}, alpha={alpha:g}, "
|
|
603
|
+
f"DDIM_steps={proposer_steps if proposer_steps is not None else 'policy_default'})"
|
|
604
|
+
)
|
|
605
|
+
t0 = _time.perf_counter()
|
|
606
|
+
# Live tqdm bar over the relabel batches. Postfix shows
|
|
607
|
+
# the running mean Q before / after local optimization
|
|
608
|
+
# so the operator can confirm Q is improving (not just
|
|
609
|
+
# that the loop is moving) without needing to wait for
|
|
610
|
+
# the per-cycle wandb log at the end.
|
|
611
|
+
pbar = _tqdm.tqdm(
|
|
612
|
+
loader,
|
|
613
|
+
total=len(loader),
|
|
614
|
+
desc=f"PA-RL relabel {split_name}",
|
|
615
|
+
dynamic_ncols=True,
|
|
616
|
+
mininterval=1.0,
|
|
617
|
+
leave=False,
|
|
618
|
+
disable=not accelerator.is_main_process,
|
|
619
|
+
)
|
|
620
|
+
# Running Q sums for the postfix; intentionally weighted
|
|
621
|
+
# by batch size so partial batches don't bias the mean.
|
|
622
|
+
q_before_running = 0.0
|
|
623
|
+
q_after_running = 0.0
|
|
624
|
+
q_chosen_running = 0.0
|
|
625
|
+
q_n_running = 0
|
|
626
|
+
for batch_idx, batch in enumerate(pbar):
|
|
627
|
+
obs = dict_apply(
|
|
628
|
+
batch['obs'],
|
|
629
|
+
lambda x: x.to(device, non_blocking=True),
|
|
630
|
+
)
|
|
631
|
+
anchor_idx_t = batch['anchor_idx']
|
|
632
|
+
anchor_idx = anchor_idx_t.cpu().numpy()
|
|
633
|
+
dataset_action = None
|
|
634
|
+
if include_data_action:
|
|
635
|
+
dataset_action = batch['action'].to(
|
|
636
|
+
device, non_blocking=True)
|
|
637
|
+
|
|
638
|
+
# Cache lookup: when the frozen-critic state_embed
|
|
639
|
+
# cache is populated AND every row in this batch
|
|
640
|
+
# has a valid cached embedding, we hand it to
|
|
641
|
+
# optimize_actions and skip the heavy ViT-L
|
|
642
|
+
# forward inside ``critic.state_embed``. If ANY
|
|
643
|
+
# row is invalid (anchor_idx == -1, or never
|
|
644
|
+
# written), we fall back to recomputing on the
|
|
645
|
+
# fly so correctness is preserved at the cost of
|
|
646
|
+
# one extra forward for that batch.
|
|
647
|
+
# Cache lookups for the two big ViT-L outputs:
|
|
648
|
+
# - state_embed_cache -> critic.state_embed(obs)
|
|
649
|
+
# - global_cond_cache -> proposer.obs_encoder(nobs)
|
|
650
|
+
# Both are computed once at run() startup and
|
|
651
|
+
# constant for the rest of training (their owners
|
|
652
|
+
# are frozen). Caches live ON the GPU so per-batch
|
|
653
|
+
# we just do an O(B) GPU-side ``index_select``;
|
|
654
|
+
# no H2D copy. Falls back to recomputing on the
|
|
655
|
+
# fly if any row in this batch has an unresolved
|
|
656
|
+
# ``anchor_idx``.
|
|
657
|
+
ai_long = anchor_idx_t.to(
|
|
658
|
+
device, non_blocking=True).long()
|
|
659
|
+
state_embed_override = None
|
|
660
|
+
if state_embed_cache is not None:
|
|
661
|
+
in_range = (ai_long >= 0) & (
|
|
662
|
+
ai_long < state_embed_cache.shape[0])
|
|
663
|
+
if bool(in_range.all().item()):
|
|
664
|
+
sv = state_embed_valid.index_select(0, ai_long)
|
|
665
|
+
if bool(sv.all().item()):
|
|
666
|
+
state_embed_override = (
|
|
667
|
+
state_embed_cache.index_select(
|
|
668
|
+
0, ai_long))
|
|
669
|
+
global_cond_override = None
|
|
670
|
+
if global_cond_cache is not None:
|
|
671
|
+
in_range_g = (ai_long >= 0) & (
|
|
672
|
+
ai_long < global_cond_cache.shape[0])
|
|
673
|
+
if bool(in_range_g.all().item()):
|
|
674
|
+
gv = global_cond_valid.index_select(0, ai_long)
|
|
675
|
+
if bool(gv.all().item()):
|
|
676
|
+
global_cond_override = (
|
|
677
|
+
global_cond_cache.index_select(
|
|
678
|
+
0, ai_long))
|
|
679
|
+
|
|
680
|
+
# Honor the accelerator's mixed_precision setting
|
|
681
|
+
# for the relabel pass too (training already does
|
|
682
|
+
# via accelerator.prepare). The proposer's ViT-L
|
|
683
|
+
# and the critic's state_embed dominate runtime
|
|
684
|
+
# and benefit ~3-4x from bf16 on A100. The local
|
|
685
|
+
# gradient ascent on the action runs through the
|
|
686
|
+
# critic's q_heads inside the autocast region;
|
|
687
|
+
# autograd handles dtype promotion to fp32 for
|
|
688
|
+
# the leaf action grad correctly.
|
|
689
|
+
with accelerator.autocast():
|
|
690
|
+
action_opt, info = optimize_actions(
|
|
691
|
+
proposer=proposer,
|
|
692
|
+
critic=critic,
|
|
693
|
+
obs_dict=obs,
|
|
694
|
+
K=K, m=m_top, T=T, alpha=alpha,
|
|
695
|
+
pick_mode=pick_mode,
|
|
696
|
+
softmax_temperature=softmax_temp,
|
|
697
|
+
dataset_action=dataset_action,
|
|
698
|
+
proposer_inference_steps=proposer_steps,
|
|
699
|
+
state_embed_override=state_embed_override,
|
|
700
|
+
global_cond_override=global_cond_override,
|
|
701
|
+
)
|
|
702
|
+
a_np = action_opt.detach().float().cpu().numpy()
|
|
703
|
+
# Vectorized scatter: O(B) Python loops add tens
|
|
704
|
+
# of microseconds per row that don't scale with
|
|
705
|
+
# GPU work, so they punish us at large B. Build a
|
|
706
|
+
# boolean mask once and let numpy do the dispatch.
|
|
707
|
+
in_range = (anchor_idx >= 0) & (anchor_idx < n_frames)
|
|
708
|
+
if in_range.any():
|
|
709
|
+
valid_anchors = anchor_idx[in_range]
|
|
710
|
+
out_actions[valid_anchors] = a_np[in_range]
|
|
711
|
+
out_valid[valid_anchors] = True
|
|
712
|
+
n_seen += int(a_np.shape[0])
|
|
713
|
+
for k_, v_ in info.items():
|
|
714
|
+
if isinstance(v_, (bool,)):
|
|
715
|
+
v_ = float(v_)
|
|
716
|
+
if isinstance(v_, (int, float)):
|
|
717
|
+
agg.setdefault(k_, []).append(float(v_))
|
|
718
|
+
|
|
719
|
+
# Update tqdm postfix every batch from the same
|
|
720
|
+
# ``info`` dict optimize_actions just returned. The
|
|
721
|
+
# running averages here weight each batch by its
|
|
722
|
+
# size so the displayed numbers are unbiased even
|
|
723
|
+
# if the last batch is partial.
|
|
724
|
+
bsz = int(a_np.shape[0])
|
|
725
|
+
q_before_running += float(
|
|
726
|
+
info.get('parl/q_before_local_mean', 0.0)) * bsz
|
|
727
|
+
q_after_running += float(
|
|
728
|
+
info.get('parl/q_after_local_mean', 0.0)) * bsz
|
|
729
|
+
q_chosen_running += float(
|
|
730
|
+
info.get('parl/q_chosen_mean', 0.0)) * bsz
|
|
731
|
+
q_n_running += bsz
|
|
732
|
+
if q_n_running > 0:
|
|
733
|
+
pbar.set_postfix({
|
|
734
|
+
'Q_before': f"{q_before_running / q_n_running:+.3f}",
|
|
735
|
+
'Q_after': f"{q_after_running / q_n_running:+.3f}",
|
|
736
|
+
'Q_chosen': f"{q_chosen_running / q_n_running:+.3f}",
|
|
737
|
+
'rev': f"{float(info.get('parl/frac_local_reverted', 0.0)):.2f}",
|
|
738
|
+
'data_picked': f"{float(info.get('parl/frac_dataset_action_chosen', 0.0)):.2f}",
|
|
739
|
+
}, refresh=False)
|
|
740
|
+
pbar.close()
|
|
741
|
+
dt = _time.perf_counter() - t0
|
|
742
|
+
accelerator.print(
|
|
743
|
+
f"[PA-RL] relabel {split_name} done in {dt:.1f}s "
|
|
744
|
+
f"({len(ds) / max(dt, 1e-3):.1f} samples/s)"
|
|
745
|
+
)
|
|
746
|
+
finally:
|
|
747
|
+
ds.action_optimized = saved_actions
|
|
748
|
+
ds.action_optimized_valid = saved_valid
|
|
749
|
+
|
|
750
|
+
info_means = {k: float(np.mean(v)) for k, v in agg.items() if v}
|
|
751
|
+
info_means['parl/n_relabeled_frames'] = int(out_valid.sum())
|
|
752
|
+
info_means['parl/n_total_frames'] = int(n_frames)
|
|
753
|
+
info_means['parl/n_samples_seen'] = int(n_seen)
|
|
754
|
+
info_means['parl/relabel_seconds'] = float(
|
|
755
|
+
_time.perf_counter() - t0_total)
|
|
756
|
+
return out_actions, out_valid, info_means
|
|
757
|
+
|
|
758
|
+
def run(self):
|
|
759
|
+
# Lazy imports: these are training-only dependencies. Keeping them
|
|
760
|
+
# out of module-top imports saves several seconds of cold start when
|
|
761
|
+
# the workspace class is loaded purely for inference (e.g. by
|
|
762
|
+
# policy_server.py -> hydra.utils.get_class). The first call to
|
|
763
|
+
# run() pays the import cost; the imports are then cached in
|
|
764
|
+
# sys.modules so subsequent calls are free.
|
|
765
|
+
from torch.utils.data import DataLoader
|
|
766
|
+
from solarian.tracking.accelerate import accelerate_trackers
|
|
767
|
+
import pickle
|
|
768
|
+
import tqdm
|
|
769
|
+
import shutil
|
|
770
|
+
from diffusion_policy.policy.diffusion_unet_image_policy import DiffusionUnetImagePolicy
|
|
771
|
+
from diffusion_policy.dataset.base_dataset import BaseImageDataset, BaseDataset
|
|
772
|
+
from diffusion_policy.env_runner.base_image_runner import BaseImageRunner
|
|
773
|
+
from diffusion_policy.common.checkpoint_util import TopKCheckpointManager
|
|
774
|
+
from diffusion_policy.common.json_logger import JsonLogger
|
|
775
|
+
from diffusion_policy.common.pytorch_util import dict_apply, optimizer_to
|
|
776
|
+
from diffusion_policy.model.diffusion.ema_model import EMAModel
|
|
777
|
+
from diffusion_policy.model.common.lr_scheduler import get_scheduler
|
|
778
|
+
from accelerate import Accelerator, DistributedDataParallelKwargs
|
|
779
|
+
|
|
780
|
+
cfg = copy.deepcopy(self.cfg)
|
|
781
|
+
|
|
782
|
+
# When the visual encoder is frozen (typical for AWR finetuning),
|
|
783
|
+
# its parameters don't receive gradients. The default DDP wrap
|
|
784
|
+
# then fails with "Expected to have finished reduction in the
|
|
785
|
+
# prior iteration..." because DDP's reducer expects every
|
|
786
|
+
# registered parameter to participate in backward. The fix is
|
|
787
|
+
# to construct DDP with find_unused_parameters=True so it
|
|
788
|
+
# tolerates the no-grad params; we gate this on freeze_encoder
|
|
789
|
+
# to avoid the (small) per-step graph-traversal overhead in
|
|
790
|
+
# plain BC training where everything trains.
|
|
791
|
+
ddp_kwargs = DistributedDataParallelKwargs(
|
|
792
|
+
find_unused_parameters=bool(
|
|
793
|
+
cfg.training.get('freeze_encoder', False)
|
|
794
|
+
)
|
|
795
|
+
)
|
|
796
|
+
# logging.backend (wandb, solarian_base, none, or a list) picks where
|
|
797
|
+
# the run goes; tracking drops empty tags and clamps long ones.
|
|
798
|
+
logging_cfg = OmegaConf.to_container(cfg.logging, resolve=True)
|
|
799
|
+
log_with, init_kwargs = accelerate_trackers(logging_cfg)
|
|
800
|
+
accelerator = Accelerator(
|
|
801
|
+
log_with=log_with,
|
|
802
|
+
kwargs_handlers=[ddp_kwargs],
|
|
803
|
+
)
|
|
804
|
+
accelerator.init_trackers(
|
|
805
|
+
project_name=cfg.logging.project,
|
|
806
|
+
config=OmegaConf.to_container(cfg, resolve=True),
|
|
807
|
+
init_kwargs=init_kwargs,
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
# resume training
|
|
811
|
+
if cfg.training.resume:
|
|
812
|
+
# Support custom checkpoint path via config, or use latest from output_dir
|
|
813
|
+
if hasattr(cfg.training, 'checkpoint_path') and cfg.training.checkpoint_path:
|
|
814
|
+
ckpt_path = pathlib.Path(cfg.training.checkpoint_path)
|
|
815
|
+
if ckpt_path.is_file():
|
|
816
|
+
accelerator.print(f"Resuming from checkpoint {ckpt_path}")
|
|
817
|
+
self.load_checkpoint(path=ckpt_path)
|
|
818
|
+
else:
|
|
819
|
+
accelerator.print(f"Warning: Checkpoint path {ckpt_path} not found, skipping resume")
|
|
820
|
+
else:
|
|
821
|
+
lastest_ckpt_path = self.get_checkpoint_path()
|
|
822
|
+
if lastest_ckpt_path.is_file():
|
|
823
|
+
accelerator.print(f"Resuming from checkpoint {lastest_ckpt_path}")
|
|
824
|
+
self.load_checkpoint(path=lastest_ckpt_path)
|
|
825
|
+
elif (
|
|
826
|
+
hasattr(cfg.training, 'init_from_ckpt')
|
|
827
|
+
and cfg.training.init_from_ckpt
|
|
828
|
+
):
|
|
829
|
+
# Fresh run that *initializes* model weights from another
|
|
830
|
+
# checkpoint without resuming optimizer / scheduler state.
|
|
831
|
+
# Used for AWR finetuning of a pre-trained BC policy:
|
|
832
|
+
# we want the BC weights as a starting point but a fresh
|
|
833
|
+
# optimizer (so the LR schedule restarts cleanly), fresh
|
|
834
|
+
# global_step / epoch counters, and (importantly) the *new*
|
|
835
|
+
# cfg's policy hyperparameters in self.model -- including
|
|
836
|
+
# awr_beta etc.
|
|
837
|
+
init_path = pathlib.Path(cfg.training.init_from_ckpt)
|
|
838
|
+
if init_path.is_file():
|
|
839
|
+
accelerator.print(
|
|
840
|
+
f"Initializing model weights from {init_path} "
|
|
841
|
+
f"(optimizer + step counters NOT loaded)"
|
|
842
|
+
)
|
|
843
|
+
# Skip optimizer state -- the new run uses a fresh
|
|
844
|
+
# optimizer with possibly-different LR. include_keys=()
|
|
845
|
+
# skips global_step / epoch pickles too. The AWR
|
|
846
|
+
# config knobs are plain Python attributes (set in
|
|
847
|
+
# __init__ from the new cfg, not stored in state_dict),
|
|
848
|
+
# so a strict load of the BC ckpt's `model` and
|
|
849
|
+
# `ema_model` state_dicts produces no key mismatch.
|
|
850
|
+
self.load_checkpoint(
|
|
851
|
+
path=init_path,
|
|
852
|
+
exclude_keys=['optimizer'],
|
|
853
|
+
include_keys=(),
|
|
854
|
+
)
|
|
855
|
+
# Force a clean restart of the schedule even though
|
|
856
|
+
# global_step/epoch were already 0 from __init__.
|
|
857
|
+
self.global_step = 0
|
|
858
|
+
self.epoch = 0
|
|
859
|
+
|
|
860
|
+
# BC-anchored AWR: snapshot the freshly-loaded BC
|
|
861
|
+
# weights into a frozen reference U-Net on the policy.
|
|
862
|
+
# No-op when ``policy.awr_anchor_lambda <= 0``, so
|
|
863
|
+
# legacy AWR / BC runs pay nothing extra. We do this
|
|
864
|
+
# here (not in policy.__init__) because the policy is
|
|
865
|
+
# built from the new AWR cfg with random weights; the
|
|
866
|
+
# anchor must be the BC ckpt's weights, not the random
|
|
867
|
+
# init.
|
|
868
|
+
init_fn = getattr(self.model, 'init_frozen_ref', None)
|
|
869
|
+
if callable(init_fn):
|
|
870
|
+
accelerator.print(
|
|
871
|
+
"Calling policy.init_frozen_ref() to snapshot "
|
|
872
|
+
"BC weights for anchor loss (no-op when "
|
|
873
|
+
"awr_anchor_lambda <= 0).")
|
|
874
|
+
init_fn()
|
|
875
|
+
# Mirror onto the EMA shadow so its forward pass
|
|
876
|
+
# under sample_every / validation can also access
|
|
877
|
+
# the anchor reference. The deep-copied ref itself
|
|
878
|
+
# is shared by attribute reference -- it's frozen
|
|
879
|
+
# and never updated, so sharing is safe.
|
|
880
|
+
if (
|
|
881
|
+
cfg.training.use_ema
|
|
882
|
+
and getattr(self, 'ema_model', None) is not None
|
|
883
|
+
):
|
|
884
|
+
object.__setattr__(
|
|
885
|
+
self.ema_model, 'frozen_ref_model',
|
|
886
|
+
getattr(self.model, 'frozen_ref_model', None),
|
|
887
|
+
)
|
|
888
|
+
else:
|
|
889
|
+
accelerator.print(
|
|
890
|
+
f"Warning: init_from_ckpt={init_path} not found; "
|
|
891
|
+
f"training will start from random / pretrained weights."
|
|
892
|
+
)
|
|
893
|
+
|
|
894
|
+
# ============== PA-RL setup ==============
|
|
895
|
+
# Resolve PA-RL config block (may be absent). Fields:
|
|
896
|
+
# parl.enabled bool master switch
|
|
897
|
+
# parl.relabel_every int epochs between relabels
|
|
898
|
+
# parl.k_samples / m_top / T_grad_steps / alpha_grad
|
|
899
|
+
# parl.distill_mode 'argmax' | 'softmax'
|
|
900
|
+
# parl.softmax_temperature float
|
|
901
|
+
# parl.proposer 'ema' | 'main'
|
|
902
|
+
# parl.include_dataset_action bool
|
|
903
|
+
# parl.proposer_inference_steps int (override DDIM steps)
|
|
904
|
+
# parl.relabel_batch_size / num_workers int
|
|
905
|
+
# parl.freeze_critic bool (always True for v1)
|
|
906
|
+
# Q-critic checkpoint path lives at ``training.q_critic_ckpt``
|
|
907
|
+
# for symmetry with ``training.init_from_ckpt`` (BC ckpt).
|
|
908
|
+
parl_cfg_node = None
|
|
909
|
+
try:
|
|
910
|
+
if OmegaConf.select(cfg, 'parl') is not None:
|
|
911
|
+
parl_cfg_node = cfg.parl
|
|
912
|
+
except Exception:
|
|
913
|
+
parl_cfg_node = None
|
|
914
|
+
parl_enabled = bool(
|
|
915
|
+
parl_cfg_node is not None
|
|
916
|
+
and OmegaConf.select(parl_cfg_node, 'enabled', default=False)
|
|
917
|
+
)
|
|
918
|
+
q_ckpt_path = OmegaConf.select(
|
|
919
|
+
cfg, 'training.q_critic_ckpt', default=None)
|
|
920
|
+
|
|
921
|
+
if parl_enabled and q_ckpt_path:
|
|
922
|
+
from diffusion_policy.common.q_critic_util import load_iql_critic
|
|
923
|
+
init_path = pathlib.Path(os.path.expanduser(str(q_ckpt_path)))
|
|
924
|
+
if not init_path.is_file():
|
|
925
|
+
raise FileNotFoundError(
|
|
926
|
+
f"PA-RL: training.q_critic_ckpt={init_path} not found."
|
|
927
|
+
)
|
|
928
|
+
# Load critic to CPU first; we'll move it to device once
|
|
929
|
+
# `accelerator.prepare()` has resolved the actual device
|
|
930
|
+
# below. This also matches policy_server.load_q_critic's
|
|
931
|
+
# contract (CPU -> device after build).
|
|
932
|
+
accelerator.print(
|
|
933
|
+
f"[PA-RL] loading frozen Q critic from {init_path}")
|
|
934
|
+
q_critic, q_cfg = load_iql_critic(
|
|
935
|
+
str(init_path), torch.device('cpu'),
|
|
936
|
+
)
|
|
937
|
+
# Sanity-check action shape vs the diffusion policy.
|
|
938
|
+
policy_h = int(self.model.action_horizon)
|
|
939
|
+
policy_a = int(self.model.action_dim)
|
|
940
|
+
critic_h = int(q_critic.action_horizon)
|
|
941
|
+
critic_a = int(q_critic.action_dim)
|
|
942
|
+
if (critic_h, critic_a) != (policy_h, policy_a):
|
|
943
|
+
raise RuntimeError(
|
|
944
|
+
f"PA-RL: Q-critic action shape ({critic_h}, {critic_a}) "
|
|
945
|
+
f"!= diffusion policy ({policy_h}, {policy_a}); "
|
|
946
|
+
f"the relabel pass would produce nonsense Q values."
|
|
947
|
+
)
|
|
948
|
+
# Freeze critic params (default; the freeze_critic flag is
|
|
949
|
+
# plumbed for forward-compat with future Cal-QL co-training).
|
|
950
|
+
freeze_critic = bool(OmegaConf.select(
|
|
951
|
+
parl_cfg_node, 'freeze_critic', default=True))
|
|
952
|
+
if freeze_critic:
|
|
953
|
+
for p in q_critic.parameters():
|
|
954
|
+
p.requires_grad_(False)
|
|
955
|
+
self.q_critic = q_critic
|
|
956
|
+
accelerator.print(
|
|
957
|
+
f"[PA-RL] q_critic ready (frozen={freeze_critic}); "
|
|
958
|
+
f"action=({critic_h}, {critic_a})"
|
|
959
|
+
)
|
|
960
|
+
elif parl_enabled and not q_ckpt_path:
|
|
961
|
+
accelerator.print(
|
|
962
|
+
"[PA-RL] parl.enabled=true but training.q_critic_ckpt is "
|
|
963
|
+
"unset; PA-RL will be DISABLED (relabel pass skipped, "
|
|
964
|
+
"dataset emits no action_optimized)."
|
|
965
|
+
)
|
|
966
|
+
parl_enabled = False
|
|
967
|
+
|
|
968
|
+
# configure dataset
|
|
969
|
+
dataset: BaseImageDataset
|
|
970
|
+
dataset = hydra.utils.instantiate(cfg.task.dataset)
|
|
971
|
+
assert isinstance(dataset, BaseImageDataset) or isinstance(dataset, BaseDataset)
|
|
972
|
+
train_dataloader = DataLoader(dataset, **cfg.dataloader)
|
|
973
|
+
|
|
974
|
+
# compute normalizer on the main process and save to disk
|
|
975
|
+
normalizer_path = os.path.join(self.output_dir, 'normalizer.pkl')
|
|
976
|
+
if accelerator.is_main_process:
|
|
977
|
+
normalizer = dataset.get_normalizer()
|
|
978
|
+
pickle.dump(normalizer, open(normalizer_path, 'wb'))
|
|
979
|
+
|
|
980
|
+
# load normalizer on all processes
|
|
981
|
+
accelerator.wait_for_everyone()
|
|
982
|
+
normalizer = pickle.load(open(normalizer_path, 'rb'))
|
|
983
|
+
|
|
984
|
+
# configure validation dataset
|
|
985
|
+
val_dataset = dataset.get_validation_dataset()
|
|
986
|
+
val_dataloader = DataLoader(val_dataset, **cfg.val_dataloader)
|
|
987
|
+
print('train dataset:', len(dataset), 'train dataloader:', len(train_dataloader))
|
|
988
|
+
print('val dataset:', len(val_dataset), 'val dataloader:', len(val_dataloader))
|
|
989
|
+
|
|
990
|
+
self.model.set_normalizer(normalizer)
|
|
991
|
+
if cfg.training.use_ema:
|
|
992
|
+
self.ema_model.set_normalizer(normalizer)
|
|
993
|
+
|
|
994
|
+
# Set initial_lr for each param group if resuming (required by LR scheduler)
|
|
995
|
+
if cfg.training.resume and self.global_step > 0:
|
|
996
|
+
for param_group in self.optimizer.param_groups:
|
|
997
|
+
if 'initial_lr' not in param_group:
|
|
998
|
+
param_group['initial_lr'] = param_group['lr']
|
|
999
|
+
|
|
1000
|
+
# configure lr scheduler
|
|
1001
|
+
lr_scheduler = get_scheduler(
|
|
1002
|
+
cfg.training.lr_scheduler,
|
|
1003
|
+
optimizer=self.optimizer,
|
|
1004
|
+
num_warmup_steps=cfg.training.lr_warmup_steps,
|
|
1005
|
+
num_training_steps=(
|
|
1006
|
+
len(train_dataloader) * cfg.training.num_epochs) \
|
|
1007
|
+
// cfg.training.gradient_accumulate_every,
|
|
1008
|
+
# pytorch assumes stepping LRScheduler every epoch
|
|
1009
|
+
# however huggingface diffusers steps it every batch
|
|
1010
|
+
last_epoch=self.global_step-1
|
|
1011
|
+
)
|
|
1012
|
+
|
|
1013
|
+
# configure ema
|
|
1014
|
+
ema: EMAModel = None
|
|
1015
|
+
if cfg.training.use_ema:
|
|
1016
|
+
ema = hydra.utils.instantiate(
|
|
1017
|
+
cfg.ema,
|
|
1018
|
+
model=self.ema_model)
|
|
1019
|
+
|
|
1020
|
+
# configure env
|
|
1021
|
+
env_runner: BaseImageRunner
|
|
1022
|
+
env_runner = hydra.utils.instantiate(
|
|
1023
|
+
cfg.task.env_runner,
|
|
1024
|
+
output_dir=self.output_dir)
|
|
1025
|
+
assert isinstance(env_runner, BaseImageRunner)
|
|
1026
|
+
|
|
1027
|
+
# # configure logging
|
|
1028
|
+
# wandb_run = wandb.init(
|
|
1029
|
+
# dir=str(self.output_dir),
|
|
1030
|
+
# config=OmegaConf.to_container(cfg, resolve=True),
|
|
1031
|
+
# **cfg.logging
|
|
1032
|
+
# )
|
|
1033
|
+
# wandb.config.update(
|
|
1034
|
+
# {
|
|
1035
|
+
# "output_dir": self.output_dir,
|
|
1036
|
+
# }
|
|
1037
|
+
# )
|
|
1038
|
+
|
|
1039
|
+
# configure checkpoint
|
|
1040
|
+
# Build constant fields used in checkpoint filenames from the experiment
|
|
1041
|
+
# config (task name, model type, visual encoder, total batch size,
|
|
1042
|
+
# dataset file name without extension). These are optional in cfg, so
|
|
1043
|
+
# missing values fall back to 'na' to avoid breaking other workspaces
|
|
1044
|
+
# that share this manager.
|
|
1045
|
+
dataset_path = cfg.task.get('dataset_path', '') if hasattr(cfg, 'task') else ''
|
|
1046
|
+
dataset_file_name = os.path.basename(str(dataset_path))
|
|
1047
|
+
# Strip common archive/zarr extensions (e.g. .zarr.zip, .zarr, .zip).
|
|
1048
|
+
while True:
|
|
1049
|
+
stem, ext = os.path.splitext(dataset_file_name)
|
|
1050
|
+
if ext.lower() in ('.zip', '.zarr', '.gz', '.tar'):
|
|
1051
|
+
dataset_file_name = stem
|
|
1052
|
+
else:
|
|
1053
|
+
break
|
|
1054
|
+
if not dataset_file_name:
|
|
1055
|
+
dataset_file_name = 'na'
|
|
1056
|
+
|
|
1057
|
+
total_batch = int(cfg.get('batch_size', 0)) * int(cfg.get('num_gpus', 1))
|
|
1058
|
+
|
|
1059
|
+
static_format_kwargs = {
|
|
1060
|
+
'task_name': cfg.get('task_name', 'na'),
|
|
1061
|
+
'model_type': cfg.get('model_type', 'na'),
|
|
1062
|
+
'visual_encoder': cfg.get('visual_encoder', 'na'),
|
|
1063
|
+
'total_batch': total_batch,
|
|
1064
|
+
'dataset_file_name': dataset_file_name,
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
topk_manager = TopKCheckpointManager(
|
|
1068
|
+
save_dir=os.path.join(self.output_dir, 'checkpoints'),
|
|
1069
|
+
static_format_kwargs=static_format_kwargs,
|
|
1070
|
+
**cfg.checkpoint.topk
|
|
1071
|
+
)
|
|
1072
|
+
|
|
1073
|
+
# device transfer
|
|
1074
|
+
# device = torch.device(cfg.training.device)
|
|
1075
|
+
# self.model.to(device)
|
|
1076
|
+
# if self.ema_model is not None:
|
|
1077
|
+
# self.ema_model.to(device)
|
|
1078
|
+
# optimizer_to(self.optimizer, device)
|
|
1079
|
+
|
|
1080
|
+
# accelerator
|
|
1081
|
+
train_dataloader, val_dataloader, self.model, self.optimizer, lr_scheduler = accelerator.prepare(
|
|
1082
|
+
train_dataloader, val_dataloader, self.model, self.optimizer, lr_scheduler
|
|
1083
|
+
)
|
|
1084
|
+
device = self.model.device
|
|
1085
|
+
if self.ema_model is not None:
|
|
1086
|
+
self.ema_model.to(device)
|
|
1087
|
+
|
|
1088
|
+
# PA-RL: move the (already-frozen) Q critic to the same device.
|
|
1089
|
+
# Done AFTER accelerator.prepare so single- and multi-GPU runs
|
|
1090
|
+
# both pick up the right device id without re-reading config.
|
|
1091
|
+
# We never DDP-wrap the critic: it's frozen in v1, used only in
|
|
1092
|
+
# eval(), and the relabel pass runs single-rank on the main
|
|
1093
|
+
# process anyway.
|
|
1094
|
+
if self.q_critic is not None:
|
|
1095
|
+
self.q_critic.to(device)
|
|
1096
|
+
self.q_critic.eval()
|
|
1097
|
+
|
|
1098
|
+
# PA-RL: optional one-time precompute of ``critic.state_embed``
|
|
1099
|
+
# for every dataset frame. Saves the heavy ViT-L state_embed
|
|
1100
|
+
# forward on EVERY relabel cycle thereafter; replaced by an
|
|
1101
|
+
# O(B) ``index_select`` from a pinned CPU tensor and a single
|
|
1102
|
+
# H2D copy. Only runs when ``parl.cache_state_embed=true`` AND
|
|
1103
|
+
# the critic is frozen (the cache is invalid if the critic's
|
|
1104
|
+
# encoder changes mid-run). Single-rank: main process owns
|
|
1105
|
+
# the build; other ranks wait at the barrier and currently
|
|
1106
|
+
# don't share the cache (the relabel pass is also single-rank
|
|
1107
|
+
# in v1, so rank>0 never hits the lookup site).
|
|
1108
|
+
cache_state_embed = (
|
|
1109
|
+
parl_enabled
|
|
1110
|
+
and self.q_critic is not None
|
|
1111
|
+
and bool(OmegaConf.select(
|
|
1112
|
+
cfg, 'parl.cache_state_embed', default=True))
|
|
1113
|
+
and bool(OmegaConf.select(
|
|
1114
|
+
cfg, 'parl.freeze_critic', default=True))
|
|
1115
|
+
)
|
|
1116
|
+
if cache_state_embed and accelerator.is_main_process:
|
|
1117
|
+
cache_n_frames = int(getattr(dataset, 'n_frames_total', 0))
|
|
1118
|
+
if cache_n_frames > 0:
|
|
1119
|
+
cache_t, cache_v = self._precompute_state_embed_cache(
|
|
1120
|
+
train_dataset=dataset,
|
|
1121
|
+
val_dataset=val_dataset,
|
|
1122
|
+
critic=self.q_critic,
|
|
1123
|
+
n_frames=cache_n_frames,
|
|
1124
|
+
device=device,
|
|
1125
|
+
accelerator=accelerator,
|
|
1126
|
+
parl_cfg=cfg.parl,
|
|
1127
|
+
)
|
|
1128
|
+
self._parl_state_embed_cache = cache_t
|
|
1129
|
+
self._parl_state_embed_valid = cache_v
|
|
1130
|
+
accelerator.wait_for_everyone()
|
|
1131
|
+
|
|
1132
|
+
# PA-RL: optional one-time precompute of the proposer's
|
|
1133
|
+
# ``obs_encoder`` output for every dataset frame. Same idea
|
|
1134
|
+
# as the state_embed cache above but for the SECOND big
|
|
1135
|
+
# ViT-L the relabel pass would otherwise run (the proposer's
|
|
1136
|
+
# visual encoder inside ``propose_actions``). Combined, the
|
|
1137
|
+
# two caches eliminate ALL ViT-L forwards from the per-iter
|
|
1138
|
+
# relabel cost; the only remaining heavy compute is the
|
|
1139
|
+
# U-Net's K candidate diffusion samples plus the tiny
|
|
1140
|
+
# action/q-head MLPs.
|
|
1141
|
+
# Gated on (a) the proposer encoder actually being frozen
|
|
1142
|
+
# (``training.freeze_encoder=true``) and (b)
|
|
1143
|
+
# ``parl.cache_global_cond=true`` (default true). The proposer
|
|
1144
|
+
# we cache against is the EMA snapshot when available --
|
|
1145
|
+
# matches the proposer the relabel pass uses; the encoder
|
|
1146
|
+
# weights are identical to the underlying model's because
|
|
1147
|
+
# EMA only steps params that move and the encoder doesn't.
|
|
1148
|
+
cache_global_cond = (
|
|
1149
|
+
parl_enabled
|
|
1150
|
+
and bool(OmegaConf.select(
|
|
1151
|
+
cfg, 'parl.cache_global_cond', default=True))
|
|
1152
|
+
and bool(OmegaConf.select(
|
|
1153
|
+
cfg, 'training.freeze_encoder', default=False))
|
|
1154
|
+
)
|
|
1155
|
+
if cache_global_cond and accelerator.is_main_process:
|
|
1156
|
+
cache_n_frames = int(getattr(dataset, 'n_frames_total', 0))
|
|
1157
|
+
if cache_n_frames > 0:
|
|
1158
|
+
cache_proposer = (
|
|
1159
|
+
self.ema_model if self.ema_model is not None
|
|
1160
|
+
else accelerator.unwrap_model(self.model)
|
|
1161
|
+
)
|
|
1162
|
+
cache_proposer.eval()
|
|
1163
|
+
# Encoder must be in eval mode for deterministic
|
|
1164
|
+
# transforms, in particular to ensure dropout (if
|
|
1165
|
+
# any) is disabled. requires_grad is already false
|
|
1166
|
+
# via the freeze_encoder enforcement at the top of
|
|
1167
|
+
# the training loop, but eval() also matters for
|
|
1168
|
+
# batchnorm / dropout layers a future encoder might
|
|
1169
|
+
# carry.
|
|
1170
|
+
cache_proposer.obs_encoder.eval()
|
|
1171
|
+
cache_t, cache_v = self._precompute_global_cond_cache(
|
|
1172
|
+
train_dataset=dataset,
|
|
1173
|
+
val_dataset=val_dataset,
|
|
1174
|
+
proposer=cache_proposer,
|
|
1175
|
+
n_frames=cache_n_frames,
|
|
1176
|
+
device=device,
|
|
1177
|
+
accelerator=accelerator,
|
|
1178
|
+
parl_cfg=cfg.parl,
|
|
1179
|
+
)
|
|
1180
|
+
self._parl_global_cond_cache = cache_t
|
|
1181
|
+
self._parl_global_cond_valid = cache_v
|
|
1182
|
+
accelerator.wait_for_everyone()
|
|
1183
|
+
|
|
1184
|
+
# save batch for sampling
|
|
1185
|
+
train_sampling_batch = None
|
|
1186
|
+
|
|
1187
|
+
# PA-RL relabel cache lives on disk for resumability + sanity
|
|
1188
|
+
# inspection ("did the optimized chunks actually move away from
|
|
1189
|
+
# the BC chunks?"). The in-memory swap into the train / val
|
|
1190
|
+
# datasets happens immediately afterwards so dataloader workers
|
|
1191
|
+
# spawned for the next epoch see the new data via the dataset
|
|
1192
|
+
# they hold a reference to.
|
|
1193
|
+
parl_relabel_every = int(OmegaConf.select(
|
|
1194
|
+
cfg, 'parl.relabel_every', default=1)) if parl_enabled else 0
|
|
1195
|
+
parl_save_cache = bool(OmegaConf.select(
|
|
1196
|
+
cfg, 'parl.save_cache_to_disk', default=True)) if parl_enabled else False
|
|
1197
|
+
# Pre-compute per-frame array shape so the relabel helper can
|
|
1198
|
+
# allocate without re-reading the replay buffer. ``dataset``
|
|
1199
|
+
# is the train split here; both train + val share the same
|
|
1200
|
+
# n_frames since they share the underlying ReplayBuffer.
|
|
1201
|
+
parl_n_frames = int(getattr(dataset, 'n_frames_total', 0))
|
|
1202
|
+
parl_action_horizon = int(getattr(dataset, 'action_horizon', 0))
|
|
1203
|
+
parl_action_dim = int(getattr(dataset, 'action_dim', 0))
|
|
1204
|
+
|
|
1205
|
+
if cfg.training.debug:
|
|
1206
|
+
cfg.training.num_epochs = 2
|
|
1207
|
+
cfg.training.max_train_steps = 3
|
|
1208
|
+
cfg.training.max_val_steps = 3
|
|
1209
|
+
cfg.training.rollout_every = 1
|
|
1210
|
+
cfg.training.checkpoint_every = 1
|
|
1211
|
+
cfg.training.val_every = 1
|
|
1212
|
+
cfg.training.sample_every = 1
|
|
1213
|
+
|
|
1214
|
+
# training loop
|
|
1215
|
+
log_path = os.path.join(self.output_dir, 'logs.json.txt')
|
|
1216
|
+
with JsonLogger(log_path) as json_logger:
|
|
1217
|
+
for local_epoch_idx in range(cfg.training.num_epochs):
|
|
1218
|
+
# ========= PA-RL: optional relabel pass ==========
|
|
1219
|
+
# Runs at the start of each epoch where the cycle hits
|
|
1220
|
+
# (epoch 0 included, so we bootstrap from the BC ckpt's
|
|
1221
|
+
# weights). Single-rank: only the main process samples
|
|
1222
|
+
# / scores; other ranks wait at the barrier and then
|
|
1223
|
+
# all ranks load the freshly-written cache file.
|
|
1224
|
+
if (
|
|
1225
|
+
parl_enabled
|
|
1226
|
+
and self.q_critic is not None
|
|
1227
|
+
and parl_relabel_every > 0
|
|
1228
|
+
and (self.epoch % parl_relabel_every) == 0
|
|
1229
|
+
):
|
|
1230
|
+
parl_actions = None
|
|
1231
|
+
parl_valid = None
|
|
1232
|
+
parl_info = {}
|
|
1233
|
+
if accelerator.is_main_process:
|
|
1234
|
+
# Use EMA shadow as the proposer when available
|
|
1235
|
+
# (more stable and matches inference); fall
|
|
1236
|
+
# back to the unwrapped main model otherwise.
|
|
1237
|
+
proposer_name = str(OmegaConf.select(
|
|
1238
|
+
cfg, 'parl.proposer', default='ema'))
|
|
1239
|
+
if proposer_name == 'ema' and self.ema_model is not None:
|
|
1240
|
+
proposer = self.ema_model
|
|
1241
|
+
else:
|
|
1242
|
+
proposer = accelerator.unwrap_model(self.model)
|
|
1243
|
+
accelerator.print(
|
|
1244
|
+
f"[PA-RL] starting relabel for epoch={self.epoch} "
|
|
1245
|
+
f"(proposer={proposer_name})"
|
|
1246
|
+
)
|
|
1247
|
+
parl_actions, parl_valid, parl_info = self._parl_relabel(
|
|
1248
|
+
train_dataset=dataset,
|
|
1249
|
+
val_dataset=val_dataset,
|
|
1250
|
+
proposer=proposer,
|
|
1251
|
+
critic=self.q_critic,
|
|
1252
|
+
n_frames=parl_n_frames,
|
|
1253
|
+
action_horizon=parl_action_horizon,
|
|
1254
|
+
action_dim=parl_action_dim,
|
|
1255
|
+
device=device,
|
|
1256
|
+
accelerator=accelerator,
|
|
1257
|
+
parl_cfg=cfg.parl,
|
|
1258
|
+
state_embed_cache=self._parl_state_embed_cache,
|
|
1259
|
+
state_embed_valid=self._parl_state_embed_valid,
|
|
1260
|
+
global_cond_cache=self._parl_global_cond_cache,
|
|
1261
|
+
global_cond_valid=self._parl_global_cond_valid,
|
|
1262
|
+
)
|
|
1263
|
+
if parl_save_cache:
|
|
1264
|
+
cache_path = os.path.join(
|
|
1265
|
+
self.output_dir,
|
|
1266
|
+
f'relabel_cache_epoch{self.epoch:04d}.npy',
|
|
1267
|
+
)
|
|
1268
|
+
valid_path = os.path.join(
|
|
1269
|
+
self.output_dir,
|
|
1270
|
+
f'relabel_valid_epoch{self.epoch:04d}.npy',
|
|
1271
|
+
)
|
|
1272
|
+
np.save(cache_path, parl_actions)
|
|
1273
|
+
np.save(valid_path, parl_valid)
|
|
1274
|
+
accelerator.print(
|
|
1275
|
+
f"[PA-RL] wrote cache: {cache_path} "
|
|
1276
|
+
f"({parl_actions.nbytes / 1e6:.1f} MB) "
|
|
1277
|
+
f"+ valid mask"
|
|
1278
|
+
)
|
|
1279
|
+
accelerator.wait_for_everyone()
|
|
1280
|
+
# Multi-rank: ranks > 0 mmap the freshly-written
|
|
1281
|
+
# cache from disk; rank 0 keeps its in-memory
|
|
1282
|
+
# array (cheaper, no roundtrip). For the v1
|
|
1283
|
+
# single-GPU path the rank > 0 branch is dead
|
|
1284
|
+
# code -- but it costs nothing and future-proofs
|
|
1285
|
+
# the multi-GPU path.
|
|
1286
|
+
if accelerator.is_main_process:
|
|
1287
|
+
dataset.set_action_optimized(parl_actions, parl_valid)
|
|
1288
|
+
val_dataset.set_action_optimized(parl_actions, parl_valid)
|
|
1289
|
+
elif parl_save_cache:
|
|
1290
|
+
cache_path = os.path.join(
|
|
1291
|
+
self.output_dir,
|
|
1292
|
+
f'relabel_cache_epoch{self.epoch:04d}.npy',
|
|
1293
|
+
)
|
|
1294
|
+
valid_path = os.path.join(
|
|
1295
|
+
self.output_dir,
|
|
1296
|
+
f'relabel_valid_epoch{self.epoch:04d}.npy',
|
|
1297
|
+
)
|
|
1298
|
+
actions_mm = np.load(cache_path, mmap_mode='r')
|
|
1299
|
+
valid_mm = np.load(valid_path, mmap_mode='r').astype(bool)
|
|
1300
|
+
dataset.set_action_optimized(actions_mm, valid_mm)
|
|
1301
|
+
val_dataset.set_action_optimized(actions_mm, valid_mm)
|
|
1302
|
+
accelerator.wait_for_everyone()
|
|
1303
|
+
|
|
1304
|
+
# Rebuild dataloaders so any persistent_workers
|
|
1305
|
+
# spawn fresh and pick up the swapped cache via
|
|
1306
|
+
# the dataset reference. Re-prepare with
|
|
1307
|
+
# accelerator so DistributedSampler etc. is
|
|
1308
|
+
# re-applied for multi-GPU.
|
|
1309
|
+
fresh_train_loader = DataLoader(dataset, **cfg.dataloader)
|
|
1310
|
+
fresh_val_loader = DataLoader(val_dataset, **cfg.val_dataloader)
|
|
1311
|
+
train_dataloader, val_dataloader = accelerator.prepare(
|
|
1312
|
+
fresh_train_loader, fresh_val_loader,
|
|
1313
|
+
)
|
|
1314
|
+
if accelerator.is_main_process and parl_info:
|
|
1315
|
+
accelerator.log(
|
|
1316
|
+
{**parl_info, 'epoch': self.epoch},
|
|
1317
|
+
step=self.global_step,
|
|
1318
|
+
)
|
|
1319
|
+
|
|
1320
|
+
self.model.train()
|
|
1321
|
+
|
|
1322
|
+
step_log = dict()
|
|
1323
|
+
# ========= train for this epoch ==========
|
|
1324
|
+
if cfg.training.freeze_encoder:
|
|
1325
|
+
# ``self.model`` is a DDP wrapper after
|
|
1326
|
+
# accelerator.prepare(), so ``obs_encoder`` lives on
|
|
1327
|
+
# the inner module. Use accelerator.unwrap_model so
|
|
1328
|
+
# this works in both single-GPU (no wrapper) and
|
|
1329
|
+
# multi-GPU DDP. eval() + requires_grad_(False)
|
|
1330
|
+
# mutate state in place; DDP forwards the
|
|
1331
|
+
# frozen-grad signal through the wrapper.
|
|
1332
|
+
_unwrapped = accelerator.unwrap_model(self.model)
|
|
1333
|
+
_unwrapped.obs_encoder.eval()
|
|
1334
|
+
_unwrapped.obs_encoder.requires_grad_(False)
|
|
1335
|
+
|
|
1336
|
+
train_losses = list()
|
|
1337
|
+
with tqdm.tqdm(train_dataloader, desc=f"Training epoch {self.epoch}",
|
|
1338
|
+
leave=False, mininterval=cfg.training.tqdm_interval_sec) as tepoch:
|
|
1339
|
+
for batch_idx, batch in enumerate(tepoch):
|
|
1340
|
+
# device transfer
|
|
1341
|
+
batch = dict_apply(batch, lambda x: x.to(device, non_blocking=True))
|
|
1342
|
+
|
|
1343
|
+
# always use the latest batch
|
|
1344
|
+
train_sampling_batch = batch
|
|
1345
|
+
|
|
1346
|
+
# compute loss
|
|
1347
|
+
raw_loss = self.model(batch)
|
|
1348
|
+
loss = raw_loss / cfg.training.gradient_accumulate_every
|
|
1349
|
+
loss.backward()
|
|
1350
|
+
|
|
1351
|
+
# step optimizer
|
|
1352
|
+
if self.global_step % cfg.training.gradient_accumulate_every == 0:
|
|
1353
|
+
self.optimizer.step()
|
|
1354
|
+
self.optimizer.zero_grad()
|
|
1355
|
+
lr_scheduler.step()
|
|
1356
|
+
|
|
1357
|
+
# update ema
|
|
1358
|
+
if cfg.training.use_ema:
|
|
1359
|
+
ema.step(accelerator.unwrap_model(self.model))
|
|
1360
|
+
|
|
1361
|
+
# logging
|
|
1362
|
+
raw_loss_cpu = raw_loss.item()
|
|
1363
|
+
tepoch.set_postfix(loss=raw_loss_cpu, refresh=False)
|
|
1364
|
+
train_losses.append(raw_loss_cpu)
|
|
1365
|
+
step_log = {
|
|
1366
|
+
'train_loss': raw_loss_cpu,
|
|
1367
|
+
'global_step': self.global_step,
|
|
1368
|
+
'epoch': self.epoch,
|
|
1369
|
+
'lr': lr_scheduler.get_last_lr()[0]
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
# Log bbox mask channel weight norms. Sample frequently
|
|
1373
|
+
# at the start so the early growth (from zero) is
|
|
1374
|
+
# visible on the wandb curve, then less often once
|
|
1375
|
+
# training is settled. Cheap (a few tensor norms) but
|
|
1376
|
+
# we still gate by step to avoid spamming.
|
|
1377
|
+
if accelerator.is_main_process:
|
|
1378
|
+
log_every = (10 if self.global_step < 500 else 50)
|
|
1379
|
+
if (self.global_step % log_every) == 0:
|
|
1380
|
+
_unwrapped = accelerator.unwrap_model(self.model)
|
|
1381
|
+
step_log.update(_bbox_mask_weight_metrics(_unwrapped))
|
|
1382
|
+
step_log.update(_awr_metrics(_unwrapped))
|
|
1383
|
+
step_log.update(_parl_metrics(_unwrapped))
|
|
1384
|
+
|
|
1385
|
+
is_last_batch = (batch_idx == (len(train_dataloader)-1))
|
|
1386
|
+
if not is_last_batch:
|
|
1387
|
+
# log of last step is combined with validation and rollout
|
|
1388
|
+
accelerator.log(step_log, step=self.global_step)
|
|
1389
|
+
json_logger.log(step_log)
|
|
1390
|
+
self.global_step += 1
|
|
1391
|
+
|
|
1392
|
+
if (cfg.training.max_train_steps is not None) \
|
|
1393
|
+
and batch_idx >= (cfg.training.max_train_steps-1):
|
|
1394
|
+
break
|
|
1395
|
+
|
|
1396
|
+
# at the end of each epoch
|
|
1397
|
+
# replace train_loss with epoch average
|
|
1398
|
+
train_loss = np.mean(train_losses)
|
|
1399
|
+
step_log['train_loss'] = train_loss
|
|
1400
|
+
|
|
1401
|
+
# End-of-epoch snapshot of the mask channel weights. Always
|
|
1402
|
+
# emitted (no throttling) so the wandb x-axis "epoch" has a
|
|
1403
|
+
# clean per-epoch curve to chart alongside train_loss.
|
|
1404
|
+
if accelerator.is_main_process:
|
|
1405
|
+
_unwrapped = accelerator.unwrap_model(self.model)
|
|
1406
|
+
step_log.update(_bbox_mask_weight_metrics(_unwrapped))
|
|
1407
|
+
step_log.update(_awr_metrics(_unwrapped))
|
|
1408
|
+
step_log.update(_parl_metrics(_unwrapped))
|
|
1409
|
+
|
|
1410
|
+
# ========= eval for this epoch ==========
|
|
1411
|
+
policy = accelerator.unwrap_model(self.model)
|
|
1412
|
+
if cfg.training.use_ema:
|
|
1413
|
+
policy = self.ema_model
|
|
1414
|
+
policy.eval()
|
|
1415
|
+
|
|
1416
|
+
# run rollout
|
|
1417
|
+
if (self.epoch % cfg.training.rollout_every) == 0:
|
|
1418
|
+
runner_log = env_runner.run(policy)
|
|
1419
|
+
# log all
|
|
1420
|
+
step_log.update(runner_log)
|
|
1421
|
+
|
|
1422
|
+
# run validation
|
|
1423
|
+
# Runs on main rank only (other ranks idle through here, then
|
|
1424
|
+
# rejoin at accelerator.log below). Uses the unwrapped policy
|
|
1425
|
+
# (EMA if enabled) so it reflects what we'd actually deploy.
|
|
1426
|
+
if (self.epoch % cfg.training.val_every) == 0 and len(val_dataloader) > 0 and accelerator.is_main_process:
|
|
1427
|
+
with torch.no_grad():
|
|
1428
|
+
val_losses = list()
|
|
1429
|
+
with tqdm.tqdm(val_dataloader, desc=f"Validation epoch {self.epoch}",
|
|
1430
|
+
leave=False, mininterval=cfg.training.tqdm_interval_sec) as tepoch:
|
|
1431
|
+
for batch_idx, batch in enumerate(tepoch):
|
|
1432
|
+
batch = dict_apply(batch, lambda x: x.to(device, non_blocking=True))
|
|
1433
|
+
# policy is already the unwrapped (EMA if
|
|
1434
|
+
# enabled) module — its forward() returns the
|
|
1435
|
+
# same diffusion loss the train step uses.
|
|
1436
|
+
loss = policy(batch)
|
|
1437
|
+
val_losses.append(loss.item())
|
|
1438
|
+
if (cfg.training.max_val_steps is not None) \
|
|
1439
|
+
and batch_idx >= (cfg.training.max_val_steps-1):
|
|
1440
|
+
break
|
|
1441
|
+
if len(val_losses) > 0:
|
|
1442
|
+
step_log['val_loss'] = float(np.mean(val_losses))
|
|
1443
|
+
|
|
1444
|
+
def _action_mse_components(pred, gt):
|
|
1445
|
+
"""Compute (full, pos, rot, width) MSE components from
|
|
1446
|
+
(B, T, A) tensors viewed as (B, T, num_robot, 10).
|
|
1447
|
+
"""
|
|
1448
|
+
B, T, _ = pred.shape
|
|
1449
|
+
pred_v = pred.view(B, T, -1, 10)
|
|
1450
|
+
gt_v = gt.view(B, T, -1, 10)
|
|
1451
|
+
return (
|
|
1452
|
+
torch.nn.functional.mse_loss(pred_v, gt_v),
|
|
1453
|
+
torch.nn.functional.mse_loss(pred_v[..., :3], gt_v[..., :3]),
|
|
1454
|
+
torch.nn.functional.mse_loss(pred_v[..., 3:9], gt_v[..., 3:9]),
|
|
1455
|
+
torch.nn.functional.mse_loss(pred_v[..., 9], gt_v[..., 9]),
|
|
1456
|
+
)
|
|
1457
|
+
|
|
1458
|
+
def log_action_mse(step_log, category, pred_action, batch):
|
|
1459
|
+
"""Log action MSE components.
|
|
1460
|
+
|
|
1461
|
+
PA-RL changes this from the original "vs dataset action"
|
|
1462
|
+
behavior. The dataset for fine-tuning contains failure
|
|
1463
|
+
rollouts (misses, corrections) we explicitly do NOT want
|
|
1464
|
+
to imitate; we are training the policy to imitate the
|
|
1465
|
+
OPTIMIZED action targets emitted by the relabel pass.
|
|
1466
|
+
Therefore the primary action_mse metric is computed
|
|
1467
|
+
against ``batch['action_optimized']`` masked to samples
|
|
1468
|
+
where ``has_action_optimized`` is set. The "vs dataset"
|
|
1469
|
+
version is still logged under ``*_dataset`` because the
|
|
1470
|
+
GAP between the two is the cleanest drift signal we
|
|
1471
|
+
have without an expert-only val split:
|
|
1472
|
+
* primary low + dataset stable: good progress.
|
|
1473
|
+
* primary low + dataset rising: optimized targets
|
|
1474
|
+
drifting from BC (turn anchor up, or critic has
|
|
1475
|
+
blind spots).
|
|
1476
|
+
* primary rising: distillation failing.
|
|
1477
|
+
|
|
1478
|
+
For categories without an optimized cache yet (e.g. on
|
|
1479
|
+
the very first epoch before relabel runs, or with PA-RL
|
|
1480
|
+
disabled), the primary metric falls back to the dataset
|
|
1481
|
+
action so the metric remains continuous across modes.
|
|
1482
|
+
"""
|
|
1483
|
+
gt_dataset = batch['action']
|
|
1484
|
+
full_d, pos_d, rot_d, w_d = _action_mse_components(
|
|
1485
|
+
pred_action, gt_dataset)
|
|
1486
|
+
step_log[f'{category}_action_mse_error_dataset'] = full_d
|
|
1487
|
+
step_log[f'{category}_action_mse_error_pos_dataset'] = pos_d
|
|
1488
|
+
step_log[f'{category}_action_mse_error_rot_dataset'] = rot_d
|
|
1489
|
+
step_log[f'{category}_action_mse_error_width_dataset'] = w_d
|
|
1490
|
+
|
|
1491
|
+
gt_opt = batch.get('action_optimized', None)
|
|
1492
|
+
has_opt = batch.get('has_action_optimized', None)
|
|
1493
|
+
use_opt = (
|
|
1494
|
+
gt_opt is not None
|
|
1495
|
+
and has_opt is not None
|
|
1496
|
+
and bool((has_opt > 0.5).any().item())
|
|
1497
|
+
)
|
|
1498
|
+
if use_opt:
|
|
1499
|
+
mask = (has_opt > 0.5).view(-1)
|
|
1500
|
+
pred_m = pred_action[mask]
|
|
1501
|
+
gt_m = gt_opt[mask]
|
|
1502
|
+
full_o, pos_o, rot_o, w_o = _action_mse_components(
|
|
1503
|
+
pred_m, gt_m)
|
|
1504
|
+
# Primary monitor target: how well the deployed
|
|
1505
|
+
# (full DDIM rollout) policy matches the actual
|
|
1506
|
+
# training target, on the subset that has one.
|
|
1507
|
+
step_log[f'{category}_action_mse_error'] = full_o
|
|
1508
|
+
step_log[f'{category}_action_mse_error_pos'] = pos_o
|
|
1509
|
+
step_log[f'{category}_action_mse_error_rot'] = rot_o
|
|
1510
|
+
step_log[f'{category}_action_mse_error_width'] = w_o
|
|
1511
|
+
else:
|
|
1512
|
+
# Pre-relabel or non-PARL run: optimized cache not
|
|
1513
|
+
# populated for this category. Use the dataset
|
|
1514
|
+
# comparison as the primary metric so monitor_key
|
|
1515
|
+
# stays well-defined and the wandb plot is
|
|
1516
|
+
# continuous.
|
|
1517
|
+
step_log[f'{category}_action_mse_error'] = full_d
|
|
1518
|
+
step_log[f'{category}_action_mse_error_pos'] = pos_d
|
|
1519
|
+
step_log[f'{category}_action_mse_error_rot'] = rot_d
|
|
1520
|
+
step_log[f'{category}_action_mse_error_width'] = w_d
|
|
1521
|
+
# run diffusion sampling on a training batch
|
|
1522
|
+
if (self.epoch % cfg.training.sample_every) == 0 and accelerator.is_main_process:
|
|
1523
|
+
with torch.no_grad():
|
|
1524
|
+
# sample trajectory from training set, and evaluate difference
|
|
1525
|
+
batch = dict_apply(train_sampling_batch, lambda x: x.to(device, non_blocking=True))
|
|
1526
|
+
pred_action = policy.predict_action(batch['obs'], None)['action_pred']
|
|
1527
|
+
log_action_mse(step_log, 'train', pred_action, batch)
|
|
1528
|
+
|
|
1529
|
+
if len(val_dataloader) > 0:
|
|
1530
|
+
val_sampling_batch = next(iter(val_dataloader))
|
|
1531
|
+
batch = dict_apply(val_sampling_batch, lambda x: x.to(device, non_blocking=True))
|
|
1532
|
+
pred_action = policy.predict_action(batch['obs'], None)['action_pred']
|
|
1533
|
+
log_action_mse(step_log, 'val', pred_action, batch)
|
|
1534
|
+
|
|
1535
|
+
del batch
|
|
1536
|
+
del pred_action
|
|
1537
|
+
|
|
1538
|
+
# checkpoint
|
|
1539
|
+
if (self.epoch % cfg.training.checkpoint_every) == 0 and accelerator.is_main_process:
|
|
1540
|
+
# unwrap the model to save ckpt
|
|
1541
|
+
model_ddp = self.model
|
|
1542
|
+
self.model = accelerator.unwrap_model(self.model)
|
|
1543
|
+
|
|
1544
|
+
# checkpointing
|
|
1545
|
+
if cfg.checkpoint.save_last_ckpt:
|
|
1546
|
+
self.save_checkpoint()
|
|
1547
|
+
if cfg.checkpoint.save_last_snapshot:
|
|
1548
|
+
self.save_snapshot()
|
|
1549
|
+
|
|
1550
|
+
# sanitize metric names
|
|
1551
|
+
metric_dict = dict()
|
|
1552
|
+
for key, value in step_log.items():
|
|
1553
|
+
new_key = key.replace('/', '_')
|
|
1554
|
+
metric_dict[new_key] = value
|
|
1555
|
+
|
|
1556
|
+
# We can't copy the last checkpoint here
|
|
1557
|
+
# since save_checkpoint uses threads.
|
|
1558
|
+
# therefore at this point the file might have been empty!
|
|
1559
|
+
topk_ckpt_path = topk_manager.get_ckpt_path(metric_dict)
|
|
1560
|
+
|
|
1561
|
+
if topk_ckpt_path is not None:
|
|
1562
|
+
self.save_checkpoint(path=topk_ckpt_path)
|
|
1563
|
+
|
|
1564
|
+
# recover the DDP model
|
|
1565
|
+
self.model = model_ddp
|
|
1566
|
+
# ========= eval end for this epoch ==========
|
|
1567
|
+
# end of epoch
|
|
1568
|
+
# log of last step is combined with validation and rollout
|
|
1569
|
+
accelerator.log(step_log, step=self.global_step)
|
|
1570
|
+
json_logger.log(step_log)
|
|
1571
|
+
self.global_step += 1
|
|
1572
|
+
self.epoch += 1
|
|
1573
|
+
|
|
1574
|
+
accelerator.end_training()
|
|
1575
|
+
|
|
1576
|
+
@hydra.main(
|
|
1577
|
+
version_base=None,
|
|
1578
|
+
config_path=str(pathlib.Path(__file__).parent.parent.joinpath("config")),
|
|
1579
|
+
config_name=pathlib.Path(__file__).stem)
|
|
1580
|
+
def main(cfg):
|
|
1581
|
+
workspace = TrainDiffusionUnetImageWorkspace(cfg)
|
|
1582
|
+
workspace.run()
|
|
1583
|
+
|
|
1584
|
+
if __name__ == "__main__":
|
|
1585
|
+
main()
|