solarian-diffusion 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_policy/__init__.py +0 -0
- diffusion_policy/codecs/__init__.py +0 -0
- diffusion_policy/codecs/imagecodecs_numcodecs.py +1408 -0
- diffusion_policy/common/__init__.py +0 -0
- diffusion_policy/common/checkpoint_util.py +64 -0
- diffusion_policy/common/json_logger.py +117 -0
- diffusion_policy/common/normalize_util.py +251 -0
- diffusion_policy/common/pose_repr_util.py +112 -0
- diffusion_policy/common/pytorch_util.py +82 -0
- diffusion_policy/common/q_critic_util.py +135 -0
- diffusion_policy/common/replay_buffer.py +672 -0
- diffusion_policy/common/sampler.py +204 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed.yaml +34 -0
- diffusion_policy/config/legacy/task/blockpush_lowdim_seed_abs.yaml +34 -0
- diffusion_policy/config/legacy/task/can_image.yaml +64 -0
- diffusion_policy/config/legacy/task/can_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/can_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/can_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim.yaml +33 -0
- diffusion_policy/config/legacy/task/kitchen_lowdim_abs.yaml +38 -0
- diffusion_policy/config/legacy/task/lift_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/lift_image.yaml +64 -0
- diffusion_policy/config/legacy/task/lift_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/lift_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/lift_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/pusht_image.yaml +40 -0
- diffusion_policy/config/legacy/task/pusht_lowdim.yaml +34 -0
- diffusion_policy/config/legacy/task/real_pusht_image.yaml +47 -0
- diffusion_policy/config/legacy/task/square_abs.yaml +72 -0
- diffusion_policy/config/legacy/task/square_image.yaml +64 -0
- diffusion_policy/config/legacy/task/square_image_abs.yaml +64 -0
- diffusion_policy/config/legacy/task/square_lowdim.yaml +46 -0
- diffusion_policy/config/legacy/task/square_lowdim_abs.yaml +47 -0
- diffusion_policy/config/legacy/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/legacy/task/tool_hang_image.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_image_abs.yaml +63 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim.yaml +45 -0
- diffusion_policy/config/legacy/task/tool_hang_lowdim_abs.yaml +46 -0
- diffusion_policy/config/legacy/task/transport_image.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_image_abs.yaml +75 -0
- diffusion_policy/config/legacy/task/transport_lowdim.yaml +49 -0
- diffusion_policy/config/legacy/task/transport_lowdim_abs.yaml +49 -0
- diffusion_policy/config/legacy/task/umi.yaml +45 -0
- diffusion_policy/config/legacy/task/umi_image.yaml +36 -0
- diffusion_policy/config/legacy/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/legacy/train_bet_lowdim_workspace.yaml +132 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_kitchen_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_pusht_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_real_hybrid_workspace.yaml +144 -0
- diffusion_policy/config/legacy/train_diffusion_transformer_umi_teleop_hybrid_workspace.yaml +156 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_hybrid_workspace.yaml +140 -0
- diffusion_policy/config/legacy/train_diffusion_unet_ddim_lowdim_workspace.yaml +146 -0
- diffusion_policy/config/legacy/train_diffusion_unet_hybrid_workspace.yaml +137 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_pretrained_workspace.yaml +149 -0
- diffusion_policy/config/legacy/train_diffusion_unet_image_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_lowdim_workspace.yaml +143 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_image_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_real_pretrained_workspace.yaml +152 -0
- diffusion_policy/config/legacy/train_diffusion_unet_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_hybrid_workspace.yaml +141 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_teleop_hybrid_workspace.yaml +153 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_timm_workspace.yaml +145 -0
- diffusion_policy/config/legacy/train_diffusion_unet_umi_workspace.yaml +150 -0
- diffusion_policy/config/legacy/train_diffusion_unet_video_workspace.yaml +128 -0
- diffusion_policy/config/legacy/train_ibc_dfo_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_ibc_dfo_lowdim_workspace.yaml +109 -0
- diffusion_policy/config/legacy/train_ibc_dfo_real_hybrid_workspace.yaml +111 -0
- diffusion_policy/config/legacy/train_robomimic_image_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_lowdim_workspace.yaml +92 -0
- diffusion_policy/config/legacy/train_robomimic_real_image_workspace.yaml +92 -0
- diffusion_policy/config/rl_annotate.yaml +265 -0
- diffusion_policy/config/rl_annotate_per_frame.yaml +162 -0
- diffusion_policy/config/rl_annotate_per_frame_v4.yaml +162 -0
- diffusion_policy/config/rl_annotate_v2.yaml +141 -0
- diffusion_policy/config/rl_annotate_v3.yaml +165 -0
- diffusion_policy/config/task/lift.yaml +89 -0
- diffusion_policy/config/task/square.yaml +92 -0
- diffusion_policy/config/task/tool_hang.yaml +93 -0
- diffusion_policy/config/task/tool_hang_abs.yaml +71 -0
- diffusion_policy/config/task/umi.yaml +105 -0
- diffusion_policy/config/task/umi_bimanual.yaml +135 -0
- diffusion_policy/config/task/umi_cup_wild.yaml +106 -0
- diffusion_policy/config/task/umi_image.yaml +36 -0
- diffusion_policy/config/task/umi_pick_lego.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_175.yaml +106 -0
- diffusion_policy/config/task/umi_pick_single_lego_bbox.yaml +135 -0
- diffusion_policy/config/task/umi_teleop_image.yaml +41 -0
- diffusion_policy/config/train_diffusion_transformer_umi_bimanual_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_transformer_umi_workspace.yaml +185 -0
- diffusion_policy/config/train_diffusion_unet_image_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_robomimic_workspace.yaml +152 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_cup_wild.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_awr.yaml +324 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_lego_pile_bbox_parl.yaml +399 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego.yaml +258 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_125.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_175.yaml +260 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_single_lego_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_pick_two_legos_bbox.yaml +265 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace.yaml +224 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_lego_resume.yaml +213 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_pick_lego.yaml +254 -0
- diffusion_policy/config/train_diffusion_unet_timm_umi_workspace_red_cup.yaml +219 -0
- diffusion_policy/config/train_diffusion_unet_umi_bimanual_workspace.yaml +203 -0
- diffusion_policy/config/train_iql_umi.yaml +186 -0
- diffusion_policy/config/train_iql_umi_v2.yaml +212 -0
- diffusion_policy/config/train_iql_umi_v3.yaml +212 -0
- diffusion_policy/dataset/__init__.py +0 -0
- diffusion_policy/dataset/base_dataset.py +76 -0
- diffusion_policy/dataset/umi_dataset.py +652 -0
- diffusion_policy/env_runner/__init__.py +0 -0
- diffusion_policy/env_runner/base_image_runner.py +9 -0
- diffusion_policy/env_runner/real_pusht_image_runner.py +10 -0
- diffusion_policy/model/__init__.py +0 -0
- diffusion_policy/model/common/__init__.py +0 -0
- diffusion_policy/model/common/dict_of_tensor_mixin.py +38 -0
- diffusion_policy/model/common/lr_scheduler.py +46 -0
- diffusion_policy/model/common/module_attr_mixin.py +14 -0
- diffusion_policy/model/common/normalizer.py +353 -0
- diffusion_policy/model/common/tensor_util.py +960 -0
- diffusion_policy/model/diffusion/__init__.py +0 -0
- diffusion_policy/model/diffusion/conditional_unet1d.py +238 -0
- diffusion_policy/model/diffusion/conv1d_components.py +46 -0
- diffusion_policy/model/diffusion/ema_model.py +88 -0
- diffusion_policy/model/diffusion/mask_generator.py +221 -0
- diffusion_policy/model/diffusion/positional_embedding.py +17 -0
- diffusion_policy/model/vision/__init__.py +0 -0
- diffusion_policy/model/vision/crop_randomizer.py +288 -0
- diffusion_policy/model/vision/multi_image_obs_encoder.py +195 -0
- diffusion_policy/model/vision/timm_obs_encoder.py +525 -0
- diffusion_policy/policy/__init__.py +0 -0
- diffusion_policy/policy/base_image_policy.py +27 -0
- diffusion_policy/policy/diffusion_unet_image_policy.py +264 -0
- diffusion_policy/policy/diffusion_unet_timm_policy.py +607 -0
- diffusion_policy/policy/parl_action_optimizer.py +675 -0
- diffusion_policy/workspace/__init__.py +0 -0
- diffusion_policy/workspace/base_workspace.py +146 -0
- diffusion_policy/workspace/train_diffusion_unet_image_workspace.py +1585 -0
- solarian_diffusion/__init__.py +32 -0
- solarian_diffusion/config.py +31 -0
- solarian_diffusion/policy_server.py +316 -0
- solarian_diffusion/rtc.py +272 -0
- solarian_diffusion/service.py +16 -0
- solarian_diffusion/trainer.py +213 -0
- solarian_diffusion-0.1.0.dist-info/METADATA +126 -0
- solarian_diffusion-0.1.0.dist-info/RECORD +159 -0
- solarian_diffusion-0.1.0.dist-info/WHEEL +5 -0
- solarian_diffusion-0.1.0.dist-info/entry_points.txt +8 -0
- solarian_diffusion-0.1.0.dist-info/licenses/LICENSE +201 -0
- solarian_diffusion-0.1.0.dist-info/licenses/NOTICE +45 -0
- solarian_diffusion-0.1.0.dist-info/top_level.txt +3 -0
- umi/__init__.py +0 -0
- umi/common/__init__.py +0 -0
- umi/common/pose_util.py +115 -0
|
@@ -0,0 +1,607 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
from typing import Dict, Optional
|
|
3
|
+
import torch
|
|
4
|
+
import torch.nn as nn
|
|
5
|
+
import torch.nn.functional as F
|
|
6
|
+
import numpy as np
|
|
7
|
+
from einops import rearrange, reduce
|
|
8
|
+
from diffusers.schedulers.scheduling_ddpm import DDPMScheduler
|
|
9
|
+
|
|
10
|
+
from diffusion_policy.model.common.normalizer import LinearNormalizer
|
|
11
|
+
from diffusion_policy.policy.base_image_policy import BaseImagePolicy
|
|
12
|
+
from diffusion_policy.model.diffusion.conditional_unet1d import ConditionalUnet1D
|
|
13
|
+
from diffusion_policy.model.diffusion.mask_generator import LowdimMaskGenerator
|
|
14
|
+
from diffusion_policy.model.vision.timm_obs_encoder import TimmObsEncoder
|
|
15
|
+
from diffusion_policy.common.pytorch_util import dict_apply
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DiffusionUnetTimmPolicy(BaseImagePolicy):
|
|
19
|
+
def __init__(self,
|
|
20
|
+
shape_meta: dict,
|
|
21
|
+
noise_scheduler: DDPMScheduler,
|
|
22
|
+
obs_encoder: TimmObsEncoder,
|
|
23
|
+
num_inference_steps=None,
|
|
24
|
+
obs_as_global_cond=True,
|
|
25
|
+
diffusion_step_embed_dim=256,
|
|
26
|
+
down_dims=(256,512,1024),
|
|
27
|
+
kernel_size=5,
|
|
28
|
+
n_groups=8,
|
|
29
|
+
cond_predict_scale=True,
|
|
30
|
+
input_pertub=0.1,
|
|
31
|
+
inpaint_fixed_action_prefix=False,
|
|
32
|
+
train_diffusion_n_samples=1,
|
|
33
|
+
# ============== AWR finetuning knobs ==============
|
|
34
|
+
# Advantage-Weighted Regression: weight each sample's
|
|
35
|
+
# denoising MSE by ``exp(advantage / awr_beta)``. Disabled
|
|
36
|
+
# (legacy BC) when awr_beta <= 0 OR the batch lacks
|
|
37
|
+
# ``advantage`` / ``has_advantage`` keys (which UmiDataset
|
|
38
|
+
# adds when rl/q + rl/v are present in the source zarr).
|
|
39
|
+
#
|
|
40
|
+
# awr_beta: temperature on the advantage. Lower beta
|
|
41
|
+
# sharpens the weighting (more like Q-max but
|
|
42
|
+
# unstable); higher beta softens it toward BC.
|
|
43
|
+
# A sensible default is roughly ``std(A)`` on
|
|
44
|
+
# your dataset (your IQL ckpt name encodes
|
|
45
|
+
# ``val_adv_std``, so use that as a starting
|
|
46
|
+
# point). 0.0 disables AWR entirely.
|
|
47
|
+
# awr_w_max: per-sample weight clip BEFORE batch norm.
|
|
48
|
+
# Caps the influence of a single outlier with
|
|
49
|
+
# very large advantage. Typical 10..100.
|
|
50
|
+
# awr_lambda: blend between BC weight (1.0) and AWR weight
|
|
51
|
+
# ``w``. 1.0 is pure AWR; 0.0 collapses back
|
|
52
|
+
# to BC; values in (0, 1) act as a "safety
|
|
53
|
+
# belt" toward the BC checkpoint's behavior.
|
|
54
|
+
# awr_normalize_weights: if True, divide weights by their
|
|
55
|
+
# batch mean before applying. Makes the
|
|
56
|
+
# effective LR roughly invariant to beta and
|
|
57
|
+
# to which subset of the data has high
|
|
58
|
+
# advantage. Strongly recommended.
|
|
59
|
+
# awr_use_ema_weights: deprecated knob slot kept here as a
|
|
60
|
+
# future hook (e.g. for a slow-moving running
|
|
61
|
+
# mean of the batch weight to stabilize early
|
|
62
|
+
# training); currently unused.
|
|
63
|
+
awr_beta=0.0,
|
|
64
|
+
awr_w_max=20.0,
|
|
65
|
+
awr_lambda=1.0,
|
|
66
|
+
awr_normalize_weights=True,
|
|
67
|
+
# ============== BC-anchored AWR knobs ==============
|
|
68
|
+
# The plain AWR loss above can overfit to high-advantage
|
|
69
|
+
# outliers and drift the policy off the BC manifold,
|
|
70
|
+
# producing twitchy / unstable rollouts. The three extra
|
|
71
|
+
# knobs below let us anchor the fine-tune back toward BC:
|
|
72
|
+
#
|
|
73
|
+
# awr_w_min: lower clamp on per-sample weights
|
|
74
|
+
# AFTER the log-cap. Prevents the
|
|
75
|
+
# optimizer from completely ignoring
|
|
76
|
+
# low-advantage but on-distribution
|
|
77
|
+
# samples (the prior says "still do
|
|
78
|
+
# what BC said to do"). Typical
|
|
79
|
+
# 0.3..0.5; 0 reproduces legacy AWR.
|
|
80
|
+
# awr_bc_lambda: coefficient on the UNWEIGHTED BC
|
|
81
|
+
# loss term (plain denoising MSE,
|
|
82
|
+
# SAME pred/target as the AWR term so
|
|
83
|
+
# there's no extra forward). Pulls
|
|
84
|
+
# the policy back toward "imitate
|
|
85
|
+
# every demo equally". Typical
|
|
86
|
+
# 0.5..5.0; 0 disables the term.
|
|
87
|
+
# awr_anchor_lambda: coefficient on the FROZEN-REFERENCE
|
|
88
|
+
# anchor: MSE between the current
|
|
89
|
+
# policy's eps-prediction and a
|
|
90
|
+
# frozen copy of the BC policy's
|
|
91
|
+
# eps-prediction at the same
|
|
92
|
+
# (x_t, t, obs). Acts as a soft KL
|
|
93
|
+
# regularizer toward the BC policy
|
|
94
|
+
# in eps-space. Requires the
|
|
95
|
+
# workspace to call
|
|
96
|
+
# ``init_frozen_ref()`` once after
|
|
97
|
+
# loading ``init_from_ckpt``.
|
|
98
|
+
# Typical 0.1..1.0; 0 disables.
|
|
99
|
+
awr_w_min=0.0,
|
|
100
|
+
awr_bc_lambda=0.0,
|
|
101
|
+
awr_anchor_lambda=0.0,
|
|
102
|
+
# parameters passed to step
|
|
103
|
+
**kwargs
|
|
104
|
+
):
|
|
105
|
+
super().__init__()
|
|
106
|
+
|
|
107
|
+
# parse shapes
|
|
108
|
+
action_shape = shape_meta['action']['shape']
|
|
109
|
+
assert len(action_shape) == 1
|
|
110
|
+
action_dim = action_shape[0]
|
|
111
|
+
action_horizon = shape_meta['action']['horizon']
|
|
112
|
+
# get feature dim
|
|
113
|
+
obs_feature_dim = np.prod(obs_encoder.output_shape())
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# create diffusion model
|
|
117
|
+
assert obs_as_global_cond
|
|
118
|
+
input_dim = action_dim
|
|
119
|
+
global_cond_dim = obs_feature_dim
|
|
120
|
+
|
|
121
|
+
model = ConditionalUnet1D(
|
|
122
|
+
input_dim=input_dim,
|
|
123
|
+
local_cond_dim=None,
|
|
124
|
+
global_cond_dim=global_cond_dim,
|
|
125
|
+
diffusion_step_embed_dim=diffusion_step_embed_dim,
|
|
126
|
+
down_dims=down_dims,
|
|
127
|
+
kernel_size=kernel_size,
|
|
128
|
+
n_groups=n_groups,
|
|
129
|
+
cond_predict_scale=cond_predict_scale
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
self.obs_encoder = obs_encoder
|
|
133
|
+
self.model = model
|
|
134
|
+
self.noise_scheduler = noise_scheduler
|
|
135
|
+
self.normalizer = LinearNormalizer()
|
|
136
|
+
self.obs_feature_dim = obs_feature_dim
|
|
137
|
+
self.action_dim = action_dim
|
|
138
|
+
self.action_horizon = action_horizon # used for training
|
|
139
|
+
self.obs_as_global_cond = obs_as_global_cond
|
|
140
|
+
self.input_pertub = input_pertub
|
|
141
|
+
self.inpaint_fixed_action_prefix = inpaint_fixed_action_prefix
|
|
142
|
+
self.train_diffusion_n_samples = int(train_diffusion_n_samples)
|
|
143
|
+
self.kwargs = kwargs
|
|
144
|
+
|
|
145
|
+
if num_inference_steps is None:
|
|
146
|
+
num_inference_steps = noise_scheduler.config.num_train_timesteps
|
|
147
|
+
self.num_inference_steps = num_inference_steps
|
|
148
|
+
|
|
149
|
+
# AWR finetuning state. ``last_awr_info`` carries per-batch
|
|
150
|
+
# diagnostic scalars (weight stats, fraction of zero-advantage
|
|
151
|
+
# samples, etc.) that the workspace pulls into wandb. Set to
|
|
152
|
+
# None when AWR is disabled or the batch had no advantage info.
|
|
153
|
+
self.awr_beta = float(awr_beta)
|
|
154
|
+
self.awr_w_max = float(awr_w_max)
|
|
155
|
+
self.awr_lambda = float(awr_lambda)
|
|
156
|
+
self.awr_normalize_weights = bool(awr_normalize_weights)
|
|
157
|
+
self.awr_w_min = float(awr_w_min)
|
|
158
|
+
self.awr_bc_lambda = float(awr_bc_lambda)
|
|
159
|
+
self.awr_anchor_lambda = float(awr_anchor_lambda)
|
|
160
|
+
# Frozen reference U-Net for the anchor term. Materialized
|
|
161
|
+
# lazily by `init_frozen_ref()` (called by the workspace AFTER
|
|
162
|
+
# `init_from_ckpt` has loaded the BC weights into self.model).
|
|
163
|
+
# Excluded from `state_dict()` so AWR checkpoints don't double
|
|
164
|
+
# in size (the ref is re-built from `init_from_ckpt` on the
|
|
165
|
+
# next finetune; for inference loads it's never needed).
|
|
166
|
+
self.frozen_ref_model: Optional[nn.Module] = None
|
|
167
|
+
self.last_awr_info: Dict[str, float] = {}
|
|
168
|
+
|
|
169
|
+
def init_frozen_ref(self) -> None:
|
|
170
|
+
"""Snapshot the current self.model as a frozen anchor.
|
|
171
|
+
|
|
172
|
+
Called by the workspace right after ``init_from_ckpt`` has
|
|
173
|
+
loaded the BC weights, so the frozen reference IS the BC
|
|
174
|
+
policy at finetune-step-0. No-op when the anchor term is
|
|
175
|
+
disabled (``awr_anchor_lambda <= 0``) so the legacy AWR run
|
|
176
|
+
doesn't pay the extra ~600 MB of GPU memory for a copy of
|
|
177
|
+
the U-Net it'll never use.
|
|
178
|
+
"""
|
|
179
|
+
if self.awr_anchor_lambda <= 0.0:
|
|
180
|
+
return
|
|
181
|
+
if self.frozen_ref_model is not None:
|
|
182
|
+
return # idempotent
|
|
183
|
+
# Deep-copy ensures parameter tensors don't share storage with
|
|
184
|
+
# self.model -- otherwise an in-place op on self.model (or
|
|
185
|
+
# accelerator-wrapping) could mutate the anchor.
|
|
186
|
+
ref = copy.deepcopy(self.model).eval()
|
|
187
|
+
for p in ref.parameters():
|
|
188
|
+
p.requires_grad_(False)
|
|
189
|
+
# We register as an attribute (not a submodule) so it is NOT
|
|
190
|
+
# part of `self.parameters()` and therefore not picked up by
|
|
191
|
+
# the optimizer / DDP / EMA. The eval-mode forward is invoked
|
|
192
|
+
# explicitly from compute_loss.
|
|
193
|
+
object.__setattr__(self, 'frozen_ref_model', ref)
|
|
194
|
+
|
|
195
|
+
def state_dict(self, *args, **kwargs):
|
|
196
|
+
"""Strip the frozen reference from saved checkpoints.
|
|
197
|
+
|
|
198
|
+
Including it would double the on-disk size of every AWR
|
|
199
|
+
checkpoint for zero benefit (the reference is rebuilt from
|
|
200
|
+
``init_from_ckpt`` on the next finetune run, and inference
|
|
201
|
+
loads don't need it at all).
|
|
202
|
+
"""
|
|
203
|
+
sd = super().state_dict(*args, **kwargs)
|
|
204
|
+
keys_to_drop = [k for k in sd.keys() if k.startswith('frozen_ref_model.')]
|
|
205
|
+
for k in keys_to_drop:
|
|
206
|
+
del sd[k]
|
|
207
|
+
return sd
|
|
208
|
+
|
|
209
|
+
def load_state_dict(self, state_dict, strict: bool = True, *args, **kwargs):
|
|
210
|
+
"""Tolerate the absence of frozen_ref_model.* keys on load.
|
|
211
|
+
|
|
212
|
+
AWR checkpoints (which we save via the overridden
|
|
213
|
+
``state_dict`` above) never contain those keys, but a
|
|
214
|
+
downstream ``init_frozen_ref()`` re-materializes the buffer
|
|
215
|
+
from current weights so nothing is lost.
|
|
216
|
+
"""
|
|
217
|
+
if strict:
|
|
218
|
+
ref_keys = [k for k in self.state_dict().keys()
|
|
219
|
+
if k.startswith('frozen_ref_model.')]
|
|
220
|
+
if ref_keys and not any(
|
|
221
|
+
k.startswith('frozen_ref_model.') for k in state_dict.keys()):
|
|
222
|
+
# Demote strict load for this specific case.
|
|
223
|
+
strict = False
|
|
224
|
+
return super().load_state_dict(state_dict, strict=strict, *args, **kwargs)
|
|
225
|
+
|
|
226
|
+
# ========= inference ============
|
|
227
|
+
def conditional_sample(self,
|
|
228
|
+
condition_data,
|
|
229
|
+
condition_mask,
|
|
230
|
+
local_cond=None,
|
|
231
|
+
global_cond=None,
|
|
232
|
+
generator=None,
|
|
233
|
+
# keyword arguments to scheduler.step
|
|
234
|
+
**kwargs
|
|
235
|
+
):
|
|
236
|
+
model = self.model
|
|
237
|
+
scheduler = self.noise_scheduler
|
|
238
|
+
|
|
239
|
+
trajectory = torch.randn(
|
|
240
|
+
size=condition_data.shape,
|
|
241
|
+
dtype=condition_data.dtype,
|
|
242
|
+
device=condition_data.device,
|
|
243
|
+
generator=generator)
|
|
244
|
+
|
|
245
|
+
# set step values
|
|
246
|
+
scheduler.set_timesteps(self.num_inference_steps)
|
|
247
|
+
|
|
248
|
+
for t in scheduler.timesteps:
|
|
249
|
+
# 1. apply conditioning
|
|
250
|
+
trajectory[condition_mask] = condition_data[condition_mask]
|
|
251
|
+
|
|
252
|
+
# 2. predict model output
|
|
253
|
+
model_output = model(trajectory, t,
|
|
254
|
+
local_cond=local_cond, global_cond=global_cond)
|
|
255
|
+
|
|
256
|
+
# 3. compute previous image: x_t -> x_t-1
|
|
257
|
+
trajectory = scheduler.step(
|
|
258
|
+
model_output, t, trajectory,
|
|
259
|
+
generator=generator,
|
|
260
|
+
**kwargs
|
|
261
|
+
).prev_sample
|
|
262
|
+
|
|
263
|
+
# finally make sure conditioning is enforced
|
|
264
|
+
trajectory[condition_mask] = condition_data[condition_mask]
|
|
265
|
+
|
|
266
|
+
return trajectory
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def predict_action(self, obs_dict: Dict[str, torch.Tensor], fixed_action_prefix: torch.Tensor=None) -> Dict[str, torch.Tensor]:
|
|
270
|
+
"""
|
|
271
|
+
obs_dict: must include "obs" key
|
|
272
|
+
fixed_action_prefix: unnormalized action prefix
|
|
273
|
+
result: must include "action" key
|
|
274
|
+
"""
|
|
275
|
+
assert 'past_action' not in obs_dict # not implemented yet
|
|
276
|
+
# normalize input
|
|
277
|
+
nobs = self.normalizer.normalize(obs_dict)
|
|
278
|
+
B = next(iter(nobs.values())).shape[0]
|
|
279
|
+
|
|
280
|
+
# condition through global feature
|
|
281
|
+
global_cond = self.obs_encoder(nobs)
|
|
282
|
+
|
|
283
|
+
# empty data for action
|
|
284
|
+
cond_data = torch.zeros(size=(B, self.action_horizon, self.action_dim), device=self.device, dtype=self.dtype)
|
|
285
|
+
cond_mask = torch.zeros_like(cond_data, dtype=torch.bool)
|
|
286
|
+
|
|
287
|
+
if fixed_action_prefix is not None and self.inpaint_fixed_action_prefix:
|
|
288
|
+
n_fixed_steps = fixed_action_prefix.shape[1]
|
|
289
|
+
cond_data[:, :n_fixed_steps] = fixed_action_prefix
|
|
290
|
+
cond_mask[:, :n_fixed_steps] = True
|
|
291
|
+
cond_data = self.normalizer['action'].normalize(cond_data)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
# run sampling
|
|
295
|
+
nsample = self.conditional_sample(
|
|
296
|
+
condition_data=cond_data,
|
|
297
|
+
condition_mask=cond_mask,
|
|
298
|
+
local_cond=None,
|
|
299
|
+
global_cond=global_cond,
|
|
300
|
+
**self.kwargs)
|
|
301
|
+
|
|
302
|
+
# unnormalize prediction
|
|
303
|
+
assert nsample.shape == (B, self.action_horizon, self.action_dim)
|
|
304
|
+
action_pred = self.normalizer['action'].unnormalize(nsample)
|
|
305
|
+
|
|
306
|
+
result = {
|
|
307
|
+
'action': action_pred,
|
|
308
|
+
'action_pred': action_pred
|
|
309
|
+
}
|
|
310
|
+
return result
|
|
311
|
+
|
|
312
|
+
# ========= training ============
|
|
313
|
+
def set_normalizer(self, normalizer: LinearNormalizer):
|
|
314
|
+
self.normalizer.load_state_dict(normalizer.state_dict())
|
|
315
|
+
|
|
316
|
+
def compute_loss(self, batch):
|
|
317
|
+
# normalize input
|
|
318
|
+
assert 'valid_mask' not in batch
|
|
319
|
+
nobs = self.normalizer.normalize(batch['obs'])
|
|
320
|
+
nactions_dataset = self.normalizer['action'].normalize(batch['action'])
|
|
321
|
+
|
|
322
|
+
# ---------------- PA-RL action-optimization target ----------------
|
|
323
|
+
# When the dataloader emits ``action_optimized`` (UmiDataset's
|
|
324
|
+
# cache populated by the workspace's relabel pass), swap the
|
|
325
|
+
# DDPM target from the dataset action to the optimized action
|
|
326
|
+
# on a PER-SAMPLE basis. ``has_action_optimized`` is the gate
|
|
327
|
+
# (1.0 => optimized; 0.0 => dataset fallback), mirroring the
|
|
328
|
+
# ``has_advantage`` convention so partially-cached batches don't
|
|
329
|
+
# silently zero out a fraction of samples.
|
|
330
|
+
#
|
|
331
|
+
# PA-RL is mutually exclusive with AWR within a single batch:
|
|
332
|
+
# we don't want to "Q-optimize the action then re-weight the
|
|
333
|
+
# loss on it by stale dataset advantages". When PA-RL fires,
|
|
334
|
+
# the AWR knobs are forced inactive and the loss reduces to
|
|
335
|
+
# the standard DDPM denoising loss against the optimized chunk
|
|
336
|
+
# plus any optional anchor terms. The PA-RL config sets
|
|
337
|
+
# ``awr_beta=0`` for the same reason; this gate is just defense
|
|
338
|
+
# in depth.
|
|
339
|
+
action_optimized = batch.get('action_optimized', None)
|
|
340
|
+
has_action_optimized = batch.get('has_action_optimized', None)
|
|
341
|
+
parl_active = (
|
|
342
|
+
action_optimized is not None
|
|
343
|
+
and has_action_optimized is not None
|
|
344
|
+
and bool((has_action_optimized > 0.5).any().item())
|
|
345
|
+
)
|
|
346
|
+
if parl_active:
|
|
347
|
+
nactions_opt = self.normalizer['action'].normalize(action_optimized)
|
|
348
|
+
has_opt = has_action_optimized.view(-1, 1, 1).to(
|
|
349
|
+
dtype=torch.bool, device=nactions_opt.device)
|
|
350
|
+
nactions = torch.where(has_opt, nactions_opt, nactions_dataset)
|
|
351
|
+
else:
|
|
352
|
+
nactions = nactions_dataset
|
|
353
|
+
|
|
354
|
+
# AWR per-sample inputs. Pulled here (BEFORE the optional
|
|
355
|
+
# train_diffusion_n_samples expansion below) so we can repeat
|
|
356
|
+
# them in lockstep with the action / obs duplication. Both
|
|
357
|
+
# default to None when the batch doesn't carry advantage info,
|
|
358
|
+
# which is also the path taken when awr_beta <= 0.
|
|
359
|
+
# When PA-RL is active we force these to None so the AWR
|
|
360
|
+
# weighting path is fully off (see comment block above).
|
|
361
|
+
advantage = batch.get('advantage', None) if not parl_active else None
|
|
362
|
+
has_advantage = batch.get('has_advantage', None) if not parl_active else None
|
|
363
|
+
awr_active = (
|
|
364
|
+
self.awr_beta > 0.0
|
|
365
|
+
and advantage is not None
|
|
366
|
+
and has_advantage is not None
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
assert self.obs_as_global_cond
|
|
370
|
+
global_cond = self.obs_encoder(nobs)
|
|
371
|
+
|
|
372
|
+
# train on multiple diffusion samples per obs
|
|
373
|
+
if self.train_diffusion_n_samples != 1:
|
|
374
|
+
# repeat obs features and actions multiple times along the batch dimension
|
|
375
|
+
# each sample will later have a different noise sample, effecty training
|
|
376
|
+
# more diffusion steps per each obs encoder forward pass
|
|
377
|
+
global_cond = torch.repeat_interleave(global_cond,
|
|
378
|
+
repeats=self.train_diffusion_n_samples, dim=0)
|
|
379
|
+
nactions = torch.repeat_interleave(nactions,
|
|
380
|
+
repeats=self.train_diffusion_n_samples, dim=0)
|
|
381
|
+
if awr_active:
|
|
382
|
+
advantage = torch.repeat_interleave(
|
|
383
|
+
advantage, repeats=self.train_diffusion_n_samples, dim=0)
|
|
384
|
+
has_advantage = torch.repeat_interleave(
|
|
385
|
+
has_advantage, repeats=self.train_diffusion_n_samples, dim=0)
|
|
386
|
+
|
|
387
|
+
trajectory = nactions
|
|
388
|
+
# Sample noise that we'll add to the images
|
|
389
|
+
noise = torch.randn(trajectory.shape, device=trajectory.device)
|
|
390
|
+
# input perturbation by adding additonal noise to alleviate exposure bias
|
|
391
|
+
# reference: https://github.com/forever208/DDPM-IP
|
|
392
|
+
noise_new = noise + self.input_pertub * torch.randn(trajectory.shape, device=trajectory.device)
|
|
393
|
+
|
|
394
|
+
# Sample a random timestep for each image
|
|
395
|
+
timesteps = torch.randint(
|
|
396
|
+
0, self.noise_scheduler.config.num_train_timesteps,
|
|
397
|
+
(nactions.shape[0],), device=trajectory.device
|
|
398
|
+
).long()
|
|
399
|
+
|
|
400
|
+
# Add noise to the clean images according to the noise magnitude at each timestep
|
|
401
|
+
# (this is the forward diffusion process)
|
|
402
|
+
noisy_trajectory = self.noise_scheduler.add_noise(
|
|
403
|
+
trajectory, noise_new, timesteps)
|
|
404
|
+
|
|
405
|
+
# Predict the noise residual
|
|
406
|
+
pred = self.model(
|
|
407
|
+
noisy_trajectory,
|
|
408
|
+
timesteps,
|
|
409
|
+
local_cond=None,
|
|
410
|
+
global_cond=global_cond
|
|
411
|
+
)
|
|
412
|
+
|
|
413
|
+
pred_type = self.noise_scheduler.config.prediction_type
|
|
414
|
+
if pred_type == 'epsilon':
|
|
415
|
+
target = noise
|
|
416
|
+
elif pred_type == 'sample':
|
|
417
|
+
target = trajectory
|
|
418
|
+
else:
|
|
419
|
+
raise ValueError(f"Unsupported prediction type {pred_type}")
|
|
420
|
+
|
|
421
|
+
loss = F.mse_loss(pred, target, reduction='none')
|
|
422
|
+
loss = loss.type(loss.dtype)
|
|
423
|
+
# Reduce to a single scalar per sample (B,). We keep this
|
|
424
|
+
# un-meaned so AWR can apply its per-sample weight cleanly.
|
|
425
|
+
loss_per_sample = reduce(loss, 'b ... -> b', 'mean')
|
|
426
|
+
|
|
427
|
+
# Compute the anchor loss once (independent of AWR weighting)
|
|
428
|
+
# so we can add it to either the BC-only or the AWR path.
|
|
429
|
+
# Skipped when no frozen reference is attached, i.e. when
|
|
430
|
+
# `awr_anchor_lambda` is 0 or `init_frozen_ref()` was never
|
|
431
|
+
# called -- preserving byte-identical legacy BC behavior.
|
|
432
|
+
loss_anchor = None
|
|
433
|
+
if (
|
|
434
|
+
self.awr_anchor_lambda > 0.0
|
|
435
|
+
and self.frozen_ref_model is not None
|
|
436
|
+
):
|
|
437
|
+
# The frozen ref is intentionally registered via
|
|
438
|
+
# `object.__setattr__` (NOT as an nn.Module submodule) so
|
|
439
|
+
# it's invisible to DDP / EMA / the optimizer. The
|
|
440
|
+
# downside: `accelerator.prepare(self.model)` / `.to()`
|
|
441
|
+
# don't recurse into it, so on the first forward it may
|
|
442
|
+
# still be on CPU while the rest of the model is on the
|
|
443
|
+
# current process's CUDA device. Move it ONCE here --
|
|
444
|
+
# idempotent on subsequent steps and free when the device
|
|
445
|
+
# already matches.
|
|
446
|
+
ref_device = next(
|
|
447
|
+
self.frozen_ref_model.parameters()).device
|
|
448
|
+
if ref_device != noisy_trajectory.device:
|
|
449
|
+
self.frozen_ref_model.to(noisy_trajectory.device)
|
|
450
|
+
with torch.no_grad():
|
|
451
|
+
pred_ref = self.frozen_ref_model(
|
|
452
|
+
noisy_trajectory, timesteps,
|
|
453
|
+
local_cond=None, global_cond=global_cond,
|
|
454
|
+
)
|
|
455
|
+
loss_anchor = F.mse_loss(pred, pred_ref)
|
|
456
|
+
|
|
457
|
+
if not awr_active:
|
|
458
|
+
# Legacy BC path: byte-identical to the pre-AWR loss
|
|
459
|
+
# (mean over the per-sample MSE scalar is mathematically
|
|
460
|
+
# equivalent to mean over the full (b, h, a) tensor).
|
|
461
|
+
# Still permit the anchor term to fire here so
|
|
462
|
+
# `awr_beta=0, awr_anchor_lambda>0` gives plain BC-with-
|
|
463
|
+
# anchor (useful as a sanity baseline).
|
|
464
|
+
self.last_awr_info = {}
|
|
465
|
+
total = loss_per_sample.mean()
|
|
466
|
+
if loss_anchor is not None:
|
|
467
|
+
total = total + self.awr_anchor_lambda * loss_anchor
|
|
468
|
+
self.last_awr_info = {
|
|
469
|
+
'awr/loss_bc': float(loss_per_sample.mean()),
|
|
470
|
+
'awr/loss_anchor': float(loss_anchor),
|
|
471
|
+
}
|
|
472
|
+
# PA-RL diagnostics: track what fraction of the batch was
|
|
473
|
+
# sourced from the optimized cache vs fell back to dataset
|
|
474
|
+
# actions. Mean L2 between optimized and dataset target
|
|
475
|
+
# tells us whether the PA-RL relabel pass actually moved
|
|
476
|
+
# actions (vs collapsed back to the BC chunk).
|
|
477
|
+
if parl_active:
|
|
478
|
+
with torch.no_grad():
|
|
479
|
+
has_opt_b = has_action_optimized.float().view(-1)
|
|
480
|
+
n_total = int(has_opt_b.numel())
|
|
481
|
+
frac_opt = float(has_opt_b.mean().item()) \
|
|
482
|
+
if n_total > 0 else 0.0
|
|
483
|
+
if has_opt_b.bool().any():
|
|
484
|
+
delta = (
|
|
485
|
+
nactions_opt[has_opt_b.bool()]
|
|
486
|
+
- nactions_dataset[has_opt_b.bool()]
|
|
487
|
+
).flatten(start_dim=1)
|
|
488
|
+
l2 = float(delta.norm(dim=-1).mean().item())
|
|
489
|
+
else:
|
|
490
|
+
l2 = 0.0
|
|
491
|
+
self.last_awr_info.update({
|
|
492
|
+
'parl/active': 1.0,
|
|
493
|
+
'parl/frac_optimized': frac_opt,
|
|
494
|
+
'parl/target_l2_change_norm_mean': l2,
|
|
495
|
+
'parl/loss_ddpm': float(loss_per_sample.mean().item()),
|
|
496
|
+
'parl/loss_anchor': (
|
|
497
|
+
float(loss_anchor.item())
|
|
498
|
+
if loss_anchor is not None else 0.0
|
|
499
|
+
),
|
|
500
|
+
})
|
|
501
|
+
return total
|
|
502
|
+
|
|
503
|
+
# ===== AWR weighting =====
|
|
504
|
+
# advantage / has_advantage are detached float tensors from the
|
|
505
|
+
# data loader, but be paranoid about dtype/device alignment in
|
|
506
|
+
# case a future loader change supplies them in fp16 / cpu.
|
|
507
|
+
adv = advantage.detach().to(
|
|
508
|
+
device=loss_per_sample.device, dtype=loss_per_sample.dtype)
|
|
509
|
+
m = has_advantage.detach().to(
|
|
510
|
+
device=loss_per_sample.device, dtype=loss_per_sample.dtype)
|
|
511
|
+
|
|
512
|
+
# Numerically safe exp: clamp the exponent so a single sample
|
|
513
|
+
# with a very large advantage can't blow up to inf. We clamp
|
|
514
|
+
# in *log space* against awr_w_max so the post-clamp weight is
|
|
515
|
+
# exactly the same as `clamp(exp(...), max=awr_w_max)` but we
|
|
516
|
+
# never materialize the inf intermediate.
|
|
517
|
+
log_w_cap = float(np.log(self.awr_w_max))
|
|
518
|
+
exponent = (adv / self.awr_beta).clamp(max=log_w_cap)
|
|
519
|
+
w_awr = torch.exp(exponent) # (B,), in (0, awr_w_max]
|
|
520
|
+
|
|
521
|
+
# Optional lower clamp on the per-sample weight, applied BEFORE
|
|
522
|
+
# the "no advantage info -> w=1" fallback so low-advantage
|
|
523
|
+
# but on-distribution samples can't be silently zeroed by a
|
|
524
|
+
# very low (or very negative) `adv / beta`. Acts as a soft
|
|
525
|
+
# "always respect at least this much of BC" prior.
|
|
526
|
+
if self.awr_w_min > 0.0:
|
|
527
|
+
w_awr = w_awr.clamp(min=self.awr_w_min)
|
|
528
|
+
|
|
529
|
+
# Replace AWR weight with 1.0 (i.e. plain BC) for samples that
|
|
530
|
+
# were flagged as not having a valid IQL anchor by the dataset.
|
|
531
|
+
# This keeps gradient information flowing through those samples
|
|
532
|
+
# rather than zeroing them out, which would shrink the
|
|
533
|
+
# effective batch size on partially-annotated datasets.
|
|
534
|
+
w = torch.where(m > 0.5, w_awr, torch.ones_like(w_awr))
|
|
535
|
+
|
|
536
|
+
# Optional per-batch normalization. Without it, the absolute
|
|
537
|
+
# magnitude of the loss (and therefore the effective learning
|
|
538
|
+
# rate) drifts with awr_beta and with the empirical advantage
|
|
539
|
+
# distribution of the current batch. We intentionally normalize
|
|
540
|
+
# over the FULL batch (including the BC-fallback samples) so
|
|
541
|
+
# the relative weighting between AWR-eligible and ineligible
|
|
542
|
+
# samples is preserved.
|
|
543
|
+
if self.awr_normalize_weights:
|
|
544
|
+
w = w / (w.mean() + 1e-8)
|
|
545
|
+
|
|
546
|
+
# BC↔AWR linear blend. lambda=1 = pure AWR; lambda=0 collapses
|
|
547
|
+
# to plain BC (useful as a sanity baseline at config flip).
|
|
548
|
+
eff_w = (1.0 - self.awr_lambda) * torch.ones_like(w) \
|
|
549
|
+
+ self.awr_lambda * w
|
|
550
|
+
|
|
551
|
+
# The "AWR" term in our 3-term BC-anchored loss.
|
|
552
|
+
loss_awr = (loss_per_sample * eff_w).mean()
|
|
553
|
+
|
|
554
|
+
# Optional UNWEIGHTED BC term ("pull back toward plain
|
|
555
|
+
# imitation of every demo equally"). Uses the SAME per-sample
|
|
556
|
+
# MSE we already computed -- no extra forward pass.
|
|
557
|
+
loss_bc = loss_per_sample.mean()
|
|
558
|
+
|
|
559
|
+
total = loss_awr
|
|
560
|
+
if self.awr_bc_lambda > 0.0:
|
|
561
|
+
total = total + self.awr_bc_lambda * loss_bc
|
|
562
|
+
if loss_anchor is not None:
|
|
563
|
+
total = total + self.awr_anchor_lambda * loss_anchor
|
|
564
|
+
|
|
565
|
+
# Per-batch diagnostics. Stashed on the module so the workspace
|
|
566
|
+
# can pull them into wandb without changing compute_loss's
|
|
567
|
+
# contract (still returns a single scalar).
|
|
568
|
+
with torch.no_grad():
|
|
569
|
+
valid = m > 0.5
|
|
570
|
+
n_valid = valid.sum().clamp_min(1)
|
|
571
|
+
adv_valid_mean = (adv * valid).sum() / n_valid
|
|
572
|
+
adv_valid_max = adv[valid].max() if valid.any() else torch.tensor(
|
|
573
|
+
float('nan'), device=adv.device)
|
|
574
|
+
adv_valid_min = adv[valid].min() if valid.any() else torch.tensor(
|
|
575
|
+
float('nan'), device=adv.device)
|
|
576
|
+
self.last_awr_info = {
|
|
577
|
+
'awr/active': 1.0,
|
|
578
|
+
'awr/beta': float(self.awr_beta),
|
|
579
|
+
'awr/lambda': float(self.awr_lambda),
|
|
580
|
+
'awr/bc_lambda': float(self.awr_bc_lambda),
|
|
581
|
+
'awr/anchor_lambda': float(self.awr_anchor_lambda),
|
|
582
|
+
'awr/w_min_floor': float(self.awr_w_min),
|
|
583
|
+
'awr/w_mean': float(w.mean()),
|
|
584
|
+
'awr/w_std': float(w.std()),
|
|
585
|
+
'awr/w_min': float(w.min()),
|
|
586
|
+
'awr/w_max_seen': float(w.max()),
|
|
587
|
+
'awr/eff_w_mean': float(eff_w.mean()),
|
|
588
|
+
'awr/eff_w_std': float(eff_w.std()),
|
|
589
|
+
'awr/frac_clipped': float(
|
|
590
|
+
(exponent >= log_w_cap - 1e-6).float().mean()),
|
|
591
|
+
'awr/frac_floored': (
|
|
592
|
+
float((w_awr <= self.awr_w_min + 1e-6).float().mean())
|
|
593
|
+
if self.awr_w_min > 0.0 else 0.0),
|
|
594
|
+
'awr/frac_has_adv': float(valid.float().mean()),
|
|
595
|
+
'awr/adv_valid_mean': float(adv_valid_mean),
|
|
596
|
+
'awr/adv_valid_min': float(adv_valid_min),
|
|
597
|
+
'awr/adv_valid_max': float(adv_valid_max),
|
|
598
|
+
'awr/loss_awr': float(loss_awr),
|
|
599
|
+
'awr/loss_bc': float(loss_bc),
|
|
600
|
+
'awr/loss_anchor': (
|
|
601
|
+
float(loss_anchor) if loss_anchor is not None else 0.0),
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
return total
|
|
605
|
+
|
|
606
|
+
def forward(self, batch):
|
|
607
|
+
return self.compute_loss(batch)
|