pixr 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pixr/__init__.py +3 -0
- pixr/app/__init__.py +0 -0
- pixr/app/eval.py +95 -0
- pixr/app/train.py +192 -0
- pixr/data/__init__.py +2 -0
- pixr/data/loader.py +26 -0
- pixr/data/transforms.py +52 -0
- pixr/framework/__init__.py +7 -0
- pixr/framework/config.py +79 -0
- pixr/framework/device.py +176 -0
- pixr/framework/ema.py +85 -0
- pixr/framework/logger.py +32 -0
- pixr/framework/monitor.py +276 -0
- pixr/framework/random.py +15 -0
- pixr/framework/task.py +369 -0
- pixr/framework/trainer.py +397 -0
- pixr/generation.py +242 -0
- pixr/loss.py +128 -0
- pixr/models/__init__.py +11 -0
- pixr/models/attention.py +178 -0
- pixr/models/config.py +239 -0
- pixr/models/configs/cronut_b16_tiktoken.json +34 -0
- pixr/models/configs/cronut_s16_tiktoken.json +34 -0
- pixr/models/configs/cruller_hybrid_r50_base_1280.json +17 -0
- pixr/models/configs/cruller_naflex_clip_base.json +30 -0
- pixr/models/configs/cruller_naflex_clip_base_8k.json +30 -0
- pixr/models/configs/cruller_naflex_clip_base_ettin150m.json +31 -0
- pixr/models/configs/cruller_naflex_clip_large.json +30 -0
- pixr/models/configs/cruller_naflex_clip_large_ettin400m.json +31 -0
- pixr/models/configs/cruller_naflex_siglip2_base.json +14 -0
- pixr/models/configs/cruller_naflex_siglip2_base_ettin150m.json +15 -0
- pixr/models/configs/cruller_naflex_siglip2_base_qwen3_0.6b.json +16 -0
- pixr/models/configs/cruller_naflex_siglip2_so400m.json +14 -0
- pixr/models/configs/cruller_vit_clip_base_1280.json +20 -0
- pixr/models/configs/fritter_naflex_siglip2_base_ettin150m.json +18 -0
- pixr/models/configs/fritter_naflex_siglip2_base_ettin400m.json +18 -0
- pixr/models/configs/fritter_naflex_siglip2_base_qwen3_0.6b.json +18 -0
- pixr/models/configs/fritter_naflex_siglip2_so400m_ettin1b.json +18 -0
- pixr/models/cronut.py +389 -0
- pixr/models/cruller.py +124 -0
- pixr/models/factory.py +79 -0
- pixr/models/fritter.py +198 -0
- pixr/models/image_encoder_timm.py +181 -0
- pixr/models/kv_cache.py +112 -0
- pixr/models/param_groups.py +76 -0
- pixr/models/text_decoder.py +433 -0
- pixr/models/text_decoder_ettin.py +436 -0
- pixr/models/text_decoder_qwen3.py +352 -0
- pixr/task/__init__.py +4 -0
- pixr/task/common.py +212 -0
- pixr/task/docvqa.py +94 -0
- pixr/task/read.py +140 -0
- pixr/task/task_registry.py +19 -0
- pixr/tokenizers/__init__.py +3 -0
- pixr/tokenizers/config.py +8 -0
- pixr/tokenizers/tiktoken_tokenizer.py +152 -0
- pixr/tokenizers/tokenizer.py +45 -0
- pixr/utils/__init__.py +3 -0
- pixr/utils/cli.py +110 -0
- pixr/utils/file_utils.py +36 -0
- pixr/utils/json_utils.py +317 -0
- pixr/utils/metrics.py +65 -0
- pixr/utils/name_utils.py +11 -0
- pixr/version.py +1 -0
- pixr-0.1.0.dev0.dist-info/METADATA +349 -0
- pixr-0.1.0.dev0.dist-info/RECORD +69 -0
- pixr-0.1.0.dev0.dist-info/WHEEL +4 -0
- pixr-0.1.0.dev0.dist-info/entry_points.txt +6 -0
- pixr-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
pixr/framework/task.py
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
""" Training task base.
|
|
2
|
+
|
|
3
|
+
A task wraps a model w/ its objective, batch contract, pre / post-processing, data pipeline configs, evaluation,
|
|
4
|
+
and task specific checkpoint state. It is an nn.Module whose forward runs the training forward (incl. loss). The
|
|
5
|
+
optimization loop (optimizer, schedule, accumulation, logging, checkpoint I/O) lives in the Trainer.
|
|
6
|
+
|
|
7
|
+
Distributed / compile handling follows the OpenCLIP TrainingTask: the task owns DDP (`prepare_distributed`) or FSDP2
|
|
8
|
+
(`prepare_fsdp`) wrapping of `trainable_module` and torch.compile (`compile`) of the model, its blocks, or the task
|
|
9
|
+
forward. Task code calls `self.trainable_module(...)` for the (wrapped) training forward and
|
|
10
|
+
`self.get_trainable_module()` for the unwrapped model (attribute / method access, generation).
|
|
11
|
+
"""
|
|
12
|
+
import abc
|
|
13
|
+
import logging
|
|
14
|
+
from contextlib import contextmanager, nullcontext
|
|
15
|
+
from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
import torch.nn as nn
|
|
19
|
+
|
|
20
|
+
from pixr.utils import clean_state_dict
|
|
21
|
+
from .config import TaskCfg
|
|
22
|
+
from .device import DeviceEnv
|
|
23
|
+
from .ema import ModelEma
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from torch.distributed.device_mesh import DeviceMesh
|
|
27
|
+
from torch.distributed.fsdp import CPUOffloadPolicy, MixedPrecisionPolicy
|
|
28
|
+
|
|
29
|
+
_logger = logging.getLogger(__name__)
|
|
30
|
+
|
|
31
|
+
# model methods run outside forward() that must trigger FSDP all-gather / reshard like a forward
|
|
32
|
+
_FSDP_FORWARD_METHODS = ('encode_image', 'prefill', 'step')
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _amp_dtype(cfg: TaskCfg):
|
|
36
|
+
if not cfg.amp:
|
|
37
|
+
return None
|
|
38
|
+
if cfg.dtype in ('float16', 'fp16', 'half'):
|
|
39
|
+
return torch.float16
|
|
40
|
+
return torch.bfloat16
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def move_to_device(x, device, non_blocking: bool = True):
|
|
44
|
+
if isinstance(x, torch.Tensor):
|
|
45
|
+
return x.to(device, non_blocking=non_blocking)
|
|
46
|
+
if isinstance(x, dict):
|
|
47
|
+
return {k: move_to_device(v, device, non_blocking) for k, v in x.items()}
|
|
48
|
+
if isinstance(x, (list, tuple)) and x and isinstance(x[0], torch.Tensor):
|
|
49
|
+
return type(x)(move_to_device(v, device, non_blocking) for v in x)
|
|
50
|
+
return x
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def unwrap_model(model: nn.Module) -> nn.Module:
|
|
54
|
+
""" Strip DDP and torch.compile wrappers (FSDP2 shards in place, so needs no unwrapping). """
|
|
55
|
+
while True:
|
|
56
|
+
if isinstance(model, torch.nn.parallel.DistributedDataParallel):
|
|
57
|
+
model = model.module
|
|
58
|
+
elif isinstance(model, torch._dynamo.eval_frame.OptimizedModule):
|
|
59
|
+
model = model._orig_mod
|
|
60
|
+
else:
|
|
61
|
+
return model
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def strip_compiled_keys(sd: Dict[str, Any]) -> Dict[str, Any]:
|
|
65
|
+
""" Remove the `_orig_mod.` segments torch.compile wrappers (model or per block) insert into state dict keys. """
|
|
66
|
+
return {k.replace('._orig_mod.', '.').removeprefix('_orig_mod.'): v for k, v in sd.items()}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class TrainingTask(nn.Module, abc.ABC):
|
|
70
|
+
""" Model + objective. Subclasses implement training_forward() and optionally evaluate() / train_metrics(). """
|
|
71
|
+
task_name: str = ''
|
|
72
|
+
|
|
73
|
+
def __init__(self, cfg: TaskCfg, model: nn.Module, device_env: DeviceEnv):
|
|
74
|
+
super().__init__()
|
|
75
|
+
self.cfg = cfg
|
|
76
|
+
self.trainable_module = model
|
|
77
|
+
self.trainable_module_ema: Optional[ModelEma] = None
|
|
78
|
+
self.device_env = device_env
|
|
79
|
+
self.amp_dtype = _amp_dtype(cfg)
|
|
80
|
+
self._fsdp_enabled = False
|
|
81
|
+
self._fsdp_reshard_after_forward = True
|
|
82
|
+
self._full_params_roots = [] # modules currently held unsharded by full_params()
|
|
83
|
+
self._compiled_training_forward = None
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def device(self) -> torch.device:
|
|
87
|
+
return self.device_env.device
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def fsdp_enabled(self) -> bool:
|
|
91
|
+
return self._fsdp_enabled
|
|
92
|
+
|
|
93
|
+
def autocast(self):
|
|
94
|
+
# w/ FSDP2 the MixedPrecisionPolicy casts params / inputs, autocast is suppressed (as per OpenCLIP)
|
|
95
|
+
if self.amp_dtype is None or self._fsdp_enabled:
|
|
96
|
+
return nullcontext()
|
|
97
|
+
return torch.autocast(device_type=self.device.type, dtype=self.amp_dtype)
|
|
98
|
+
|
|
99
|
+
# --- batch / forward contract ---
|
|
100
|
+
def prepare_batch(self, batch: Dict[str, Any]) -> Dict[str, Any]:
|
|
101
|
+
return {k: move_to_device(v, self.device) for k, v in batch.items()}
|
|
102
|
+
|
|
103
|
+
def batch_size(self, batch: Dict[str, Any]) -> int:
|
|
104
|
+
x = batch['image_input']
|
|
105
|
+
return (x['patches'] if isinstance(x, dict) else x).shape[0]
|
|
106
|
+
|
|
107
|
+
@abc.abstractmethod
|
|
108
|
+
def training_forward(self, batch: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, torch.Tensor]]:
|
|
109
|
+
""" Returns (outputs, losses), losses['loss'] is the combined training loss. """
|
|
110
|
+
|
|
111
|
+
def forward(self, batch: Dict[str, Any]):
|
|
112
|
+
forward_fn = self._compiled_training_forward or self.training_forward
|
|
113
|
+
return forward_fn(batch)
|
|
114
|
+
|
|
115
|
+
# --- optional hooks ---
|
|
116
|
+
def train_metrics(self, batch: Dict[str, Any], outputs: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
117
|
+
""" Expensive train metrics on the primary process every `metrics_frequency` steps. """
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
def evaluate(self, loader, use_ema: bool = False) -> Dict[str, Any]:
|
|
121
|
+
raise NotImplementedError(f'Task {self.task_name} does not support evaluation.')
|
|
122
|
+
|
|
123
|
+
def train_data_cfg(self):
|
|
124
|
+
raise NotImplementedError
|
|
125
|
+
|
|
126
|
+
def eval_data_cfg(self):
|
|
127
|
+
raise NotImplementedError
|
|
128
|
+
|
|
129
|
+
def extra_state(self) -> Dict[str, Any]:
|
|
130
|
+
""" Extra (non-tensor) checkpoint state, e.g. model config, tokenizer tokens. """
|
|
131
|
+
return {}
|
|
132
|
+
|
|
133
|
+
# --- model access / EMA ---
|
|
134
|
+
def get_trainable_module(self, use_ema: bool = False) -> nn.Module:
|
|
135
|
+
""" The unwrapped model (or its EMA), use `trainable_module` directly for the wrapped training forward. """
|
|
136
|
+
if use_ema and self.trainable_module_ema is not None:
|
|
137
|
+
return self.trainable_module_ema.module
|
|
138
|
+
return unwrap_model(self.trainable_module)
|
|
139
|
+
|
|
140
|
+
def setup_ema(self, decay: float):
|
|
141
|
+
""" EMA copy of the model, kept out of the module tree (not wrapped by DDP / compile, not in parameters()).
|
|
142
|
+
|
|
143
|
+
Must be called before `prepare_fsdp()` (FSDP2 modules can't be deep-copied), which shards the copy too.
|
|
144
|
+
"""
|
|
145
|
+
assert not self._fsdp_enabled, 'EMA must be set up before prepare_fsdp().'
|
|
146
|
+
import copy
|
|
147
|
+
ema = ModelEma(copy.deepcopy(unwrap_model(self.trainable_module)), decay=decay)
|
|
148
|
+
object.__setattr__(self, 'trainable_module_ema', ema)
|
|
149
|
+
|
|
150
|
+
@torch.no_grad()
|
|
151
|
+
def update_ema(self):
|
|
152
|
+
if self.trainable_module_ema is not None:
|
|
153
|
+
self.trainable_module_ema.update(self.get_trainable_module())
|
|
154
|
+
|
|
155
|
+
@property
|
|
156
|
+
def has_ema(self) -> bool:
|
|
157
|
+
return self.trainable_module_ema is not None
|
|
158
|
+
|
|
159
|
+
# --- compile / distributed ---
|
|
160
|
+
def compile(self, target: str = 'task', **compile_kwargs) -> 'TrainingTask':
|
|
161
|
+
""" torch.compile w/o replacing the task object.
|
|
162
|
+
|
|
163
|
+
Targets:
|
|
164
|
+
'model': compile trainable_module (before DDP wrapping)
|
|
165
|
+
'blocks': compile the model's blocks in place (before DDP wrapping), surrounding code (incl. inline grad
|
|
166
|
+
checkpointing) stays eager, so checkpoint recompute is sequenced block by block by eager autograd
|
|
167
|
+
instead of traced into the compiled backward (bounds compiled backward memory)
|
|
168
|
+
'task': compile the task training forward (model + loss), after distributed wrapping
|
|
169
|
+
"""
|
|
170
|
+
if target == 'model':
|
|
171
|
+
self.trainable_module = torch.compile(self.trainable_module, **compile_kwargs)
|
|
172
|
+
elif target == 'blocks':
|
|
173
|
+
blocks = self._get_fsdp_shard_modules()
|
|
174
|
+
if not blocks:
|
|
175
|
+
raise ValueError("compile target 'blocks' found no blocks (model needs fsdp_shard_modules())")
|
|
176
|
+
model = unwrap_model(self.trainable_module)
|
|
177
|
+
for name, mod in blocks:
|
|
178
|
+
parent_name, _, attr = name.rpartition('.')
|
|
179
|
+
parent = model.get_submodule(parent_name) if parent_name else model
|
|
180
|
+
setattr(parent, attr, torch.compile(mod, **compile_kwargs))
|
|
181
|
+
elif target == 'task':
|
|
182
|
+
self._compiled_training_forward = torch.compile(self.training_forward, **compile_kwargs)
|
|
183
|
+
else:
|
|
184
|
+
raise ValueError(f'Unsupported task compile target: {target}')
|
|
185
|
+
return self
|
|
186
|
+
|
|
187
|
+
def prepare_distributed(self, device_ids: Optional[List] = None, **ddp_kwargs) -> 'TrainingTask':
|
|
188
|
+
""" Wrap trainable_module w/ DistributedDataParallel. """
|
|
189
|
+
assert not self._fsdp_enabled, 'Cannot wrap an FSDP2 sharded module w/ DDP, use FSDP or DDP, not both.'
|
|
190
|
+
self.trainable_module = torch.nn.parallel.DistributedDataParallel(
|
|
191
|
+
self.trainable_module, device_ids=device_ids, **ddp_kwargs)
|
|
192
|
+
if self.trainable_module_ema is not None:
|
|
193
|
+
# DDP broadcasts rank 0's params / buffers on wrap, an EMA copied before then holds this rank's (possibly
|
|
194
|
+
# different, rank seeded) init, re-copy so the EMA starts from the synchronized model on every rank
|
|
195
|
+
self.trainable_module_ema.set(self.get_trainable_module())
|
|
196
|
+
return self
|
|
197
|
+
|
|
198
|
+
def _get_fsdp_shard_modules(self) -> List[Tuple[str, nn.Module]]:
|
|
199
|
+
""" (name, module) blocks to wrap individually for FSDP2, composable checkpointing and 'blocks' compile. """
|
|
200
|
+
model = unwrap_model(self.trainable_module)
|
|
201
|
+
if hasattr(model, 'fsdp_shard_modules'):
|
|
202
|
+
return model.fsdp_shard_modules()
|
|
203
|
+
return []
|
|
204
|
+
|
|
205
|
+
def prepare_fsdp(
|
|
206
|
+
self,
|
|
207
|
+
mesh: Optional['DeviceMesh'] = None,
|
|
208
|
+
reshard_after_forward: bool = True,
|
|
209
|
+
mp_policy: Optional['MixedPrecisionPolicy'] = None,
|
|
210
|
+
offload_policy: Optional['CPUOffloadPolicy'] = None,
|
|
211
|
+
compile_blocks: bool = False,
|
|
212
|
+
compile_kwargs: Optional[Dict[str, Any]] = None,
|
|
213
|
+
grad_checkpointing: bool = False,
|
|
214
|
+
) -> 'TrainingTask':
|
|
215
|
+
""" Apply FSDP2 sharding to the trainable module: each shard block, then the root.
|
|
216
|
+
|
|
217
|
+
Activation checkpointing is applied here (composable, per shard block) instead of the models' inline
|
|
218
|
+
checkpointing. An EMA from setup_ema() is sharded the same way.
|
|
219
|
+
"""
|
|
220
|
+
assert not isinstance(self.trainable_module, torch.nn.parallel.DistributedDataParallel), \
|
|
221
|
+
'Cannot apply FSDP2 to a DDP wrapped module, use FSDP or DDP, not both.'
|
|
222
|
+
model = unwrap_model(self.trainable_module)
|
|
223
|
+
scalars = [n for n, p in model.named_parameters() if p.ndim == 0]
|
|
224
|
+
assert not scalars, f'FSDP2 needs params w/ a dim to shard, found 0-D params {scalars}.'
|
|
225
|
+
|
|
226
|
+
fsdp_kwargs = dict(mesh=mesh, reshard_after_forward=reshard_after_forward)
|
|
227
|
+
if mp_policy is not None:
|
|
228
|
+
fsdp_kwargs['mp_policy'] = mp_policy
|
|
229
|
+
if offload_policy is not None:
|
|
230
|
+
fsdp_kwargs['offload_policy'] = offload_policy
|
|
231
|
+
|
|
232
|
+
shard_modules = self._get_fsdp_shard_modules()
|
|
233
|
+
_logger.info(f'FSDP2: sharding {len(shard_modules)} blocks + root')
|
|
234
|
+
if not shard_modules:
|
|
235
|
+
_logger.warning('FSDP2: model defines no fsdp_shard_modules(), only the root will be sharded.')
|
|
236
|
+
|
|
237
|
+
if grad_checkpointing:
|
|
238
|
+
model.set_grad_checkpointing(False) # composable AC per block replaces the inline checkpointing
|
|
239
|
+
from torch.distributed._composable import checkpoint as composable_checkpoint
|
|
240
|
+
|
|
241
|
+
# Required ordering: compile -> composable AC -> FSDP.
|
|
242
|
+
#
|
|
243
|
+
# Composable AC hooks must be registered on the OptimizedModule wrapper, NOT on the inner block. If hooks are
|
|
244
|
+
# on the inner block, AC recomputation replays inside the compiled graph where FSDP DTensor metadata has
|
|
245
|
+
# changed, causing "Recomputed values have different metadata" errors. With hooks on the OptimizedModule, AC
|
|
246
|
+
# recomputation triggers at the module boundary (outside the compiled graph), which also triggers FSDP
|
|
247
|
+
# all-gather hooks, so parameters are correctly gathered during recompute.
|
|
248
|
+
wrapped_modules = []
|
|
249
|
+
for name, mod in shard_modules:
|
|
250
|
+
if compile_blocks:
|
|
251
|
+
mod = torch.compile(mod, **(compile_kwargs or {}))
|
|
252
|
+
parent_name, _, attr = name.rpartition('.')
|
|
253
|
+
parent = model.get_submodule(parent_name) if parent_name else model
|
|
254
|
+
parent.register_module(attr, mod) # re-register so FSDP sees the compiled wrapper
|
|
255
|
+
if grad_checkpointing:
|
|
256
|
+
composable_checkpoint(mod)
|
|
257
|
+
wrapped_modules.append((name, mod))
|
|
258
|
+
if compile_blocks:
|
|
259
|
+
_logger.info(f'FSDP2: compiled {len(wrapped_modules)} blocks w/ torch.compile')
|
|
260
|
+
|
|
261
|
+
self._fully_shard_model(model, wrapped_modules, fsdp_kwargs)
|
|
262
|
+
|
|
263
|
+
if self.trainable_module_ema is not None:
|
|
264
|
+
# The EMA only runs eval forwards, so it always reshards and isn't compiled or checkpointed.
|
|
265
|
+
ema_module = self.trainable_module_ema.module
|
|
266
|
+
self._fully_shard_model(
|
|
267
|
+
ema_module,
|
|
268
|
+
[(name, ema_module.get_submodule(name)) for name, _ in shard_modules],
|
|
269
|
+
{**fsdp_kwargs, 'reshard_after_forward': True},
|
|
270
|
+
)
|
|
271
|
+
self.trainable_module_ema.init_fsdp()
|
|
272
|
+
|
|
273
|
+
self._fsdp_enabled = True
|
|
274
|
+
self._fsdp_reshard_after_forward = reshard_after_forward
|
|
275
|
+
return self
|
|
276
|
+
|
|
277
|
+
@staticmethod
|
|
278
|
+
def _fully_shard_model(root: nn.Module, shard_modules: List[Tuple[str, nn.Module]], fsdp_kwargs: dict):
|
|
279
|
+
from torch.distributed.fsdp import fully_shard, register_fsdp_forward_method
|
|
280
|
+
for _, mod in shard_modules:
|
|
281
|
+
fully_shard(mod, **fsdp_kwargs)
|
|
282
|
+
# Root: remaining params (embeddings, norms, projections, heads). Left at FSDP's default (no reshard after
|
|
283
|
+
# forward for the root), the fused LM loss uses the root's (tied) output projection after the model forward.
|
|
284
|
+
root_kwargs = {k: v for k, v in fsdp_kwargs.items() if k != 'reshard_after_forward'}
|
|
285
|
+
fully_shard(root, **root_kwargs)
|
|
286
|
+
# generation entry points must all-gather / reshard like __call__
|
|
287
|
+
for method_name in _FSDP_FORWARD_METHODS:
|
|
288
|
+
if hasattr(root, method_name):
|
|
289
|
+
register_fsdp_forward_method(root, method_name)
|
|
290
|
+
|
|
291
|
+
def set_requires_gradient_sync(self, requires_sync: bool):
|
|
292
|
+
""" FSDP2 equivalent of DDP no_sync() for gradient accumulation. """
|
|
293
|
+
if self._fsdp_enabled:
|
|
294
|
+
self.get_trainable_module().set_requires_gradient_sync(requires_sync)
|
|
295
|
+
|
|
296
|
+
@contextmanager
|
|
297
|
+
def full_params(self, use_ema: bool = False):
|
|
298
|
+
""" Hold the full (unsharded) params of the model (or EMA) for the duration (FSDP2, collective on entry).
|
|
299
|
+
|
|
300
|
+
Within, forwards issue no collectives, so ranks can run eval / generation independently (different numbers
|
|
301
|
+
of batches or decode steps, or primary only). No-op w/o FSDP.
|
|
302
|
+
"""
|
|
303
|
+
if not self._fsdp_enabled:
|
|
304
|
+
yield
|
|
305
|
+
return
|
|
306
|
+
from torch.distributed.fsdp import FSDPModule
|
|
307
|
+
root = self.get_trainable_module(use_ema)
|
|
308
|
+
# Scopes for the same root nest, only the outermost unshards / reshards. An inner exit resharding would
|
|
309
|
+
# leave the outer scope (and has_full_params) claiming full params over sharded ones.
|
|
310
|
+
outermost = not any(m is root for m in self._full_params_roots)
|
|
311
|
+
fsdp_modules = []
|
|
312
|
+
if outermost:
|
|
313
|
+
root._get_fsdp_state()._lazy_init()
|
|
314
|
+
fsdp_modules = [m for m in root.modules() if isinstance(m, FSDPModule)]
|
|
315
|
+
for m in fsdp_modules:
|
|
316
|
+
m.set_reshard_after_forward(False, recurse=False)
|
|
317
|
+
m.unshard()
|
|
318
|
+
self._full_params_roots.append(root)
|
|
319
|
+
try:
|
|
320
|
+
yield
|
|
321
|
+
finally:
|
|
322
|
+
self._full_params_roots.remove(root) # one entry per scope
|
|
323
|
+
restore = self._fsdp_reshard_after_forward if root is self.get_trainable_module() else True
|
|
324
|
+
for m in fsdp_modules:
|
|
325
|
+
m.reshard()
|
|
326
|
+
if m is not root: # the root keeps FSDP's default (no reshard after forward)
|
|
327
|
+
m.set_reshard_after_forward(restore, recurse=False)
|
|
328
|
+
|
|
329
|
+
def has_full_params(self, module: nn.Module) -> bool:
|
|
330
|
+
""" True if `module` (the model or its EMA) can run w/o collectives: no FSDP, or held by full_params(). """
|
|
331
|
+
return not self._fsdp_enabled or any(m is module for m in self._full_params_roots)
|
|
332
|
+
|
|
333
|
+
# --- checkpoint state ---
|
|
334
|
+
def _module_state_dict(self, module: nn.Module) -> Dict[str, Any]:
|
|
335
|
+
""" Full state dict for a module, gathering shards under FSDP (collective, populated on rank 0 only). """
|
|
336
|
+
if not self._fsdp_enabled:
|
|
337
|
+
return strip_compiled_keys(module.state_dict())
|
|
338
|
+
from torch.distributed.checkpoint.state_dict import get_model_state_dict, StateDictOptions
|
|
339
|
+
options = StateDictOptions(full_state_dict=True, cpu_offload=True)
|
|
340
|
+
return strip_compiled_keys(get_model_state_dict(module, options=options))
|
|
341
|
+
|
|
342
|
+
def _load_module_state_dict(self, module: nn.Module, state_dict: Dict[str, Any]):
|
|
343
|
+
sd = strip_compiled_keys(clean_state_dict(state_dict))
|
|
344
|
+
if self._fsdp_enabled:
|
|
345
|
+
# DCP state dict FQNs are already free of torch.compile `_orig_mod` segments, load the clean names
|
|
346
|
+
from torch.distributed.checkpoint.state_dict import set_model_state_dict, StateDictOptions
|
|
347
|
+
set_model_state_dict(module, sd, options=StateDictOptions(full_state_dict=True))
|
|
348
|
+
return
|
|
349
|
+
model_keys = list(module.state_dict().keys())
|
|
350
|
+
if any('_orig_mod' in k for k in model_keys):
|
|
351
|
+
# remap clean names onto the live model's keys when its blocks are compiled in place
|
|
352
|
+
clean_to_model = {k.replace('._orig_mod.', '.').removeprefix('_orig_mod.'): k for k in model_keys}
|
|
353
|
+
sd = {clean_to_model.get(k, k): v for k, v in sd.items()}
|
|
354
|
+
module.load_state_dict(sd)
|
|
355
|
+
|
|
356
|
+
def model_state_dict(self) -> Dict[str, Any]:
|
|
357
|
+
state = {'model': self._module_state_dict(self.get_trainable_module())}
|
|
358
|
+
if self.trainable_module_ema is not None:
|
|
359
|
+
state['model_ema'] = self._module_state_dict(self.trainable_module_ema.module)
|
|
360
|
+
return state
|
|
361
|
+
|
|
362
|
+
def load_model_state_dict(self, state: Dict[str, Any]):
|
|
363
|
+
self._load_module_state_dict(self.get_trainable_module(), state.get('model', state))
|
|
364
|
+
if self.trainable_module_ema is not None:
|
|
365
|
+
if 'model_ema' in state:
|
|
366
|
+
self._load_module_state_dict(self.trainable_module_ema.module, state['model_ema'])
|
|
367
|
+
else:
|
|
368
|
+
_logger.warning('Checkpoint has no EMA weights, initializing the EMA from the loaded model weights.')
|
|
369
|
+
self.trainable_module_ema.set(self.get_trainable_module())
|