pixr 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. pixr/__init__.py +3 -0
  2. pixr/app/__init__.py +0 -0
  3. pixr/app/eval.py +95 -0
  4. pixr/app/train.py +192 -0
  5. pixr/data/__init__.py +2 -0
  6. pixr/data/loader.py +26 -0
  7. pixr/data/transforms.py +52 -0
  8. pixr/framework/__init__.py +7 -0
  9. pixr/framework/config.py +79 -0
  10. pixr/framework/device.py +176 -0
  11. pixr/framework/ema.py +85 -0
  12. pixr/framework/logger.py +32 -0
  13. pixr/framework/monitor.py +276 -0
  14. pixr/framework/random.py +15 -0
  15. pixr/framework/task.py +369 -0
  16. pixr/framework/trainer.py +397 -0
  17. pixr/generation.py +242 -0
  18. pixr/loss.py +128 -0
  19. pixr/models/__init__.py +11 -0
  20. pixr/models/attention.py +178 -0
  21. pixr/models/config.py +239 -0
  22. pixr/models/configs/cronut_b16_tiktoken.json +34 -0
  23. pixr/models/configs/cronut_s16_tiktoken.json +34 -0
  24. pixr/models/configs/cruller_hybrid_r50_base_1280.json +17 -0
  25. pixr/models/configs/cruller_naflex_clip_base.json +30 -0
  26. pixr/models/configs/cruller_naflex_clip_base_8k.json +30 -0
  27. pixr/models/configs/cruller_naflex_clip_base_ettin150m.json +31 -0
  28. pixr/models/configs/cruller_naflex_clip_large.json +30 -0
  29. pixr/models/configs/cruller_naflex_clip_large_ettin400m.json +31 -0
  30. pixr/models/configs/cruller_naflex_siglip2_base.json +14 -0
  31. pixr/models/configs/cruller_naflex_siglip2_base_ettin150m.json +15 -0
  32. pixr/models/configs/cruller_naflex_siglip2_base_qwen3_0.6b.json +16 -0
  33. pixr/models/configs/cruller_naflex_siglip2_so400m.json +14 -0
  34. pixr/models/configs/cruller_vit_clip_base_1280.json +20 -0
  35. pixr/models/configs/fritter_naflex_siglip2_base_ettin150m.json +18 -0
  36. pixr/models/configs/fritter_naflex_siglip2_base_ettin400m.json +18 -0
  37. pixr/models/configs/fritter_naflex_siglip2_base_qwen3_0.6b.json +18 -0
  38. pixr/models/configs/fritter_naflex_siglip2_so400m_ettin1b.json +18 -0
  39. pixr/models/cronut.py +389 -0
  40. pixr/models/cruller.py +124 -0
  41. pixr/models/factory.py +79 -0
  42. pixr/models/fritter.py +198 -0
  43. pixr/models/image_encoder_timm.py +181 -0
  44. pixr/models/kv_cache.py +112 -0
  45. pixr/models/param_groups.py +76 -0
  46. pixr/models/text_decoder.py +433 -0
  47. pixr/models/text_decoder_ettin.py +436 -0
  48. pixr/models/text_decoder_qwen3.py +352 -0
  49. pixr/task/__init__.py +4 -0
  50. pixr/task/common.py +212 -0
  51. pixr/task/docvqa.py +94 -0
  52. pixr/task/read.py +140 -0
  53. pixr/task/task_registry.py +19 -0
  54. pixr/tokenizers/__init__.py +3 -0
  55. pixr/tokenizers/config.py +8 -0
  56. pixr/tokenizers/tiktoken_tokenizer.py +152 -0
  57. pixr/tokenizers/tokenizer.py +45 -0
  58. pixr/utils/__init__.py +3 -0
  59. pixr/utils/cli.py +110 -0
  60. pixr/utils/file_utils.py +36 -0
  61. pixr/utils/json_utils.py +317 -0
  62. pixr/utils/metrics.py +65 -0
  63. pixr/utils/name_utils.py +11 -0
  64. pixr/version.py +1 -0
  65. pixr-0.1.0.dev0.dist-info/METADATA +349 -0
  66. pixr-0.1.0.dev0.dist-info/RECORD +69 -0
  67. pixr-0.1.0.dev0.dist-info/WHEEL +4 -0
  68. pixr-0.1.0.dev0.dist-info/entry_points.txt +6 -0
  69. pixr-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
pixr/framework/task.py ADDED
@@ -0,0 +1,369 @@
1
+ """ Training task base.
2
+
3
+ A task wraps a model w/ its objective, batch contract, pre / post-processing, data pipeline configs, evaluation,
4
+ and task specific checkpoint state. It is an nn.Module whose forward runs the training forward (incl. loss). The
5
+ optimization loop (optimizer, schedule, accumulation, logging, checkpoint I/O) lives in the Trainer.
6
+
7
+ Distributed / compile handling follows the OpenCLIP TrainingTask: the task owns DDP (`prepare_distributed`) or FSDP2
8
+ (`prepare_fsdp`) wrapping of `trainable_module` and torch.compile (`compile`) of the model, its blocks, or the task
9
+ forward. Task code calls `self.trainable_module(...)` for the (wrapped) training forward and
10
+ `self.get_trainable_module()` for the unwrapped model (attribute / method access, generation).
11
+ """
12
+ import abc
13
+ import logging
14
+ from contextlib import contextmanager, nullcontext
15
+ from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING
16
+
17
+ import torch
18
+ import torch.nn as nn
19
+
20
+ from pixr.utils import clean_state_dict
21
+ from .config import TaskCfg
22
+ from .device import DeviceEnv
23
+ from .ema import ModelEma
24
+
25
+ if TYPE_CHECKING:
26
+ from torch.distributed.device_mesh import DeviceMesh
27
+ from torch.distributed.fsdp import CPUOffloadPolicy, MixedPrecisionPolicy
28
+
29
+ _logger = logging.getLogger(__name__)
30
+
31
+ # model methods run outside forward() that must trigger FSDP all-gather / reshard like a forward
32
+ _FSDP_FORWARD_METHODS = ('encode_image', 'prefill', 'step')
33
+
34
+
35
+ def _amp_dtype(cfg: TaskCfg):
36
+ if not cfg.amp:
37
+ return None
38
+ if cfg.dtype in ('float16', 'fp16', 'half'):
39
+ return torch.float16
40
+ return torch.bfloat16
41
+
42
+
43
+ def move_to_device(x, device, non_blocking: bool = True):
44
+ if isinstance(x, torch.Tensor):
45
+ return x.to(device, non_blocking=non_blocking)
46
+ if isinstance(x, dict):
47
+ return {k: move_to_device(v, device, non_blocking) for k, v in x.items()}
48
+ if isinstance(x, (list, tuple)) and x and isinstance(x[0], torch.Tensor):
49
+ return type(x)(move_to_device(v, device, non_blocking) for v in x)
50
+ return x
51
+
52
+
53
+ def unwrap_model(model: nn.Module) -> nn.Module:
54
+ """ Strip DDP and torch.compile wrappers (FSDP2 shards in place, so needs no unwrapping). """
55
+ while True:
56
+ if isinstance(model, torch.nn.parallel.DistributedDataParallel):
57
+ model = model.module
58
+ elif isinstance(model, torch._dynamo.eval_frame.OptimizedModule):
59
+ model = model._orig_mod
60
+ else:
61
+ return model
62
+
63
+
64
+ def strip_compiled_keys(sd: Dict[str, Any]) -> Dict[str, Any]:
65
+ """ Remove the `_orig_mod.` segments torch.compile wrappers (model or per block) insert into state dict keys. """
66
+ return {k.replace('._orig_mod.', '.').removeprefix('_orig_mod.'): v for k, v in sd.items()}
67
+
68
+
69
+ class TrainingTask(nn.Module, abc.ABC):
70
+ """ Model + objective. Subclasses implement training_forward() and optionally evaluate() / train_metrics(). """
71
+ task_name: str = ''
72
+
73
+ def __init__(self, cfg: TaskCfg, model: nn.Module, device_env: DeviceEnv):
74
+ super().__init__()
75
+ self.cfg = cfg
76
+ self.trainable_module = model
77
+ self.trainable_module_ema: Optional[ModelEma] = None
78
+ self.device_env = device_env
79
+ self.amp_dtype = _amp_dtype(cfg)
80
+ self._fsdp_enabled = False
81
+ self._fsdp_reshard_after_forward = True
82
+ self._full_params_roots = [] # modules currently held unsharded by full_params()
83
+ self._compiled_training_forward = None
84
+
85
+ @property
86
+ def device(self) -> torch.device:
87
+ return self.device_env.device
88
+
89
+ @property
90
+ def fsdp_enabled(self) -> bool:
91
+ return self._fsdp_enabled
92
+
93
+ def autocast(self):
94
+ # w/ FSDP2 the MixedPrecisionPolicy casts params / inputs, autocast is suppressed (as per OpenCLIP)
95
+ if self.amp_dtype is None or self._fsdp_enabled:
96
+ return nullcontext()
97
+ return torch.autocast(device_type=self.device.type, dtype=self.amp_dtype)
98
+
99
+ # --- batch / forward contract ---
100
+ def prepare_batch(self, batch: Dict[str, Any]) -> Dict[str, Any]:
101
+ return {k: move_to_device(v, self.device) for k, v in batch.items()}
102
+
103
+ def batch_size(self, batch: Dict[str, Any]) -> int:
104
+ x = batch['image_input']
105
+ return (x['patches'] if isinstance(x, dict) else x).shape[0]
106
+
107
+ @abc.abstractmethod
108
+ def training_forward(self, batch: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, torch.Tensor]]:
109
+ """ Returns (outputs, losses), losses['loss'] is the combined training loss. """
110
+
111
+ def forward(self, batch: Dict[str, Any]):
112
+ forward_fn = self._compiled_training_forward or self.training_forward
113
+ return forward_fn(batch)
114
+
115
+ # --- optional hooks ---
116
+ def train_metrics(self, batch: Dict[str, Any], outputs: Dict[str, Any]) -> Optional[Dict[str, Any]]:
117
+ """ Expensive train metrics on the primary process every `metrics_frequency` steps. """
118
+ return None
119
+
120
+ def evaluate(self, loader, use_ema: bool = False) -> Dict[str, Any]:
121
+ raise NotImplementedError(f'Task {self.task_name} does not support evaluation.')
122
+
123
+ def train_data_cfg(self):
124
+ raise NotImplementedError
125
+
126
+ def eval_data_cfg(self):
127
+ raise NotImplementedError
128
+
129
+ def extra_state(self) -> Dict[str, Any]:
130
+ """ Extra (non-tensor) checkpoint state, e.g. model config, tokenizer tokens. """
131
+ return {}
132
+
133
+ # --- model access / EMA ---
134
+ def get_trainable_module(self, use_ema: bool = False) -> nn.Module:
135
+ """ The unwrapped model (or its EMA), use `trainable_module` directly for the wrapped training forward. """
136
+ if use_ema and self.trainable_module_ema is not None:
137
+ return self.trainable_module_ema.module
138
+ return unwrap_model(self.trainable_module)
139
+
140
+ def setup_ema(self, decay: float):
141
+ """ EMA copy of the model, kept out of the module tree (not wrapped by DDP / compile, not in parameters()).
142
+
143
+ Must be called before `prepare_fsdp()` (FSDP2 modules can't be deep-copied), which shards the copy too.
144
+ """
145
+ assert not self._fsdp_enabled, 'EMA must be set up before prepare_fsdp().'
146
+ import copy
147
+ ema = ModelEma(copy.deepcopy(unwrap_model(self.trainable_module)), decay=decay)
148
+ object.__setattr__(self, 'trainable_module_ema', ema)
149
+
150
+ @torch.no_grad()
151
+ def update_ema(self):
152
+ if self.trainable_module_ema is not None:
153
+ self.trainable_module_ema.update(self.get_trainable_module())
154
+
155
+ @property
156
+ def has_ema(self) -> bool:
157
+ return self.trainable_module_ema is not None
158
+
159
+ # --- compile / distributed ---
160
+ def compile(self, target: str = 'task', **compile_kwargs) -> 'TrainingTask':
161
+ """ torch.compile w/o replacing the task object.
162
+
163
+ Targets:
164
+ 'model': compile trainable_module (before DDP wrapping)
165
+ 'blocks': compile the model's blocks in place (before DDP wrapping), surrounding code (incl. inline grad
166
+ checkpointing) stays eager, so checkpoint recompute is sequenced block by block by eager autograd
167
+ instead of traced into the compiled backward (bounds compiled backward memory)
168
+ 'task': compile the task training forward (model + loss), after distributed wrapping
169
+ """
170
+ if target == 'model':
171
+ self.trainable_module = torch.compile(self.trainable_module, **compile_kwargs)
172
+ elif target == 'blocks':
173
+ blocks = self._get_fsdp_shard_modules()
174
+ if not blocks:
175
+ raise ValueError("compile target 'blocks' found no blocks (model needs fsdp_shard_modules())")
176
+ model = unwrap_model(self.trainable_module)
177
+ for name, mod in blocks:
178
+ parent_name, _, attr = name.rpartition('.')
179
+ parent = model.get_submodule(parent_name) if parent_name else model
180
+ setattr(parent, attr, torch.compile(mod, **compile_kwargs))
181
+ elif target == 'task':
182
+ self._compiled_training_forward = torch.compile(self.training_forward, **compile_kwargs)
183
+ else:
184
+ raise ValueError(f'Unsupported task compile target: {target}')
185
+ return self
186
+
187
+ def prepare_distributed(self, device_ids: Optional[List] = None, **ddp_kwargs) -> 'TrainingTask':
188
+ """ Wrap trainable_module w/ DistributedDataParallel. """
189
+ assert not self._fsdp_enabled, 'Cannot wrap an FSDP2 sharded module w/ DDP, use FSDP or DDP, not both.'
190
+ self.trainable_module = torch.nn.parallel.DistributedDataParallel(
191
+ self.trainable_module, device_ids=device_ids, **ddp_kwargs)
192
+ if self.trainable_module_ema is not None:
193
+ # DDP broadcasts rank 0's params / buffers on wrap, an EMA copied before then holds this rank's (possibly
194
+ # different, rank seeded) init, re-copy so the EMA starts from the synchronized model on every rank
195
+ self.trainable_module_ema.set(self.get_trainable_module())
196
+ return self
197
+
198
+ def _get_fsdp_shard_modules(self) -> List[Tuple[str, nn.Module]]:
199
+ """ (name, module) blocks to wrap individually for FSDP2, composable checkpointing and 'blocks' compile. """
200
+ model = unwrap_model(self.trainable_module)
201
+ if hasattr(model, 'fsdp_shard_modules'):
202
+ return model.fsdp_shard_modules()
203
+ return []
204
+
205
+ def prepare_fsdp(
206
+ self,
207
+ mesh: Optional['DeviceMesh'] = None,
208
+ reshard_after_forward: bool = True,
209
+ mp_policy: Optional['MixedPrecisionPolicy'] = None,
210
+ offload_policy: Optional['CPUOffloadPolicy'] = None,
211
+ compile_blocks: bool = False,
212
+ compile_kwargs: Optional[Dict[str, Any]] = None,
213
+ grad_checkpointing: bool = False,
214
+ ) -> 'TrainingTask':
215
+ """ Apply FSDP2 sharding to the trainable module: each shard block, then the root.
216
+
217
+ Activation checkpointing is applied here (composable, per shard block) instead of the models' inline
218
+ checkpointing. An EMA from setup_ema() is sharded the same way.
219
+ """
220
+ assert not isinstance(self.trainable_module, torch.nn.parallel.DistributedDataParallel), \
221
+ 'Cannot apply FSDP2 to a DDP wrapped module, use FSDP or DDP, not both.'
222
+ model = unwrap_model(self.trainable_module)
223
+ scalars = [n for n, p in model.named_parameters() if p.ndim == 0]
224
+ assert not scalars, f'FSDP2 needs params w/ a dim to shard, found 0-D params {scalars}.'
225
+
226
+ fsdp_kwargs = dict(mesh=mesh, reshard_after_forward=reshard_after_forward)
227
+ if mp_policy is not None:
228
+ fsdp_kwargs['mp_policy'] = mp_policy
229
+ if offload_policy is not None:
230
+ fsdp_kwargs['offload_policy'] = offload_policy
231
+
232
+ shard_modules = self._get_fsdp_shard_modules()
233
+ _logger.info(f'FSDP2: sharding {len(shard_modules)} blocks + root')
234
+ if not shard_modules:
235
+ _logger.warning('FSDP2: model defines no fsdp_shard_modules(), only the root will be sharded.')
236
+
237
+ if grad_checkpointing:
238
+ model.set_grad_checkpointing(False) # composable AC per block replaces the inline checkpointing
239
+ from torch.distributed._composable import checkpoint as composable_checkpoint
240
+
241
+ # Required ordering: compile -> composable AC -> FSDP.
242
+ #
243
+ # Composable AC hooks must be registered on the OptimizedModule wrapper, NOT on the inner block. If hooks are
244
+ # on the inner block, AC recomputation replays inside the compiled graph where FSDP DTensor metadata has
245
+ # changed, causing "Recomputed values have different metadata" errors. With hooks on the OptimizedModule, AC
246
+ # recomputation triggers at the module boundary (outside the compiled graph), which also triggers FSDP
247
+ # all-gather hooks, so parameters are correctly gathered during recompute.
248
+ wrapped_modules = []
249
+ for name, mod in shard_modules:
250
+ if compile_blocks:
251
+ mod = torch.compile(mod, **(compile_kwargs or {}))
252
+ parent_name, _, attr = name.rpartition('.')
253
+ parent = model.get_submodule(parent_name) if parent_name else model
254
+ parent.register_module(attr, mod) # re-register so FSDP sees the compiled wrapper
255
+ if grad_checkpointing:
256
+ composable_checkpoint(mod)
257
+ wrapped_modules.append((name, mod))
258
+ if compile_blocks:
259
+ _logger.info(f'FSDP2: compiled {len(wrapped_modules)} blocks w/ torch.compile')
260
+
261
+ self._fully_shard_model(model, wrapped_modules, fsdp_kwargs)
262
+
263
+ if self.trainable_module_ema is not None:
264
+ # The EMA only runs eval forwards, so it always reshards and isn't compiled or checkpointed.
265
+ ema_module = self.trainable_module_ema.module
266
+ self._fully_shard_model(
267
+ ema_module,
268
+ [(name, ema_module.get_submodule(name)) for name, _ in shard_modules],
269
+ {**fsdp_kwargs, 'reshard_after_forward': True},
270
+ )
271
+ self.trainable_module_ema.init_fsdp()
272
+
273
+ self._fsdp_enabled = True
274
+ self._fsdp_reshard_after_forward = reshard_after_forward
275
+ return self
276
+
277
+ @staticmethod
278
+ def _fully_shard_model(root: nn.Module, shard_modules: List[Tuple[str, nn.Module]], fsdp_kwargs: dict):
279
+ from torch.distributed.fsdp import fully_shard, register_fsdp_forward_method
280
+ for _, mod in shard_modules:
281
+ fully_shard(mod, **fsdp_kwargs)
282
+ # Root: remaining params (embeddings, norms, projections, heads). Left at FSDP's default (no reshard after
283
+ # forward for the root), the fused LM loss uses the root's (tied) output projection after the model forward.
284
+ root_kwargs = {k: v for k, v in fsdp_kwargs.items() if k != 'reshard_after_forward'}
285
+ fully_shard(root, **root_kwargs)
286
+ # generation entry points must all-gather / reshard like __call__
287
+ for method_name in _FSDP_FORWARD_METHODS:
288
+ if hasattr(root, method_name):
289
+ register_fsdp_forward_method(root, method_name)
290
+
291
+ def set_requires_gradient_sync(self, requires_sync: bool):
292
+ """ FSDP2 equivalent of DDP no_sync() for gradient accumulation. """
293
+ if self._fsdp_enabled:
294
+ self.get_trainable_module().set_requires_gradient_sync(requires_sync)
295
+
296
+ @contextmanager
297
+ def full_params(self, use_ema: bool = False):
298
+ """ Hold the full (unsharded) params of the model (or EMA) for the duration (FSDP2, collective on entry).
299
+
300
+ Within, forwards issue no collectives, so ranks can run eval / generation independently (different numbers
301
+ of batches or decode steps, or primary only). No-op w/o FSDP.
302
+ """
303
+ if not self._fsdp_enabled:
304
+ yield
305
+ return
306
+ from torch.distributed.fsdp import FSDPModule
307
+ root = self.get_trainable_module(use_ema)
308
+ # Scopes for the same root nest, only the outermost unshards / reshards. An inner exit resharding would
309
+ # leave the outer scope (and has_full_params) claiming full params over sharded ones.
310
+ outermost = not any(m is root for m in self._full_params_roots)
311
+ fsdp_modules = []
312
+ if outermost:
313
+ root._get_fsdp_state()._lazy_init()
314
+ fsdp_modules = [m for m in root.modules() if isinstance(m, FSDPModule)]
315
+ for m in fsdp_modules:
316
+ m.set_reshard_after_forward(False, recurse=False)
317
+ m.unshard()
318
+ self._full_params_roots.append(root)
319
+ try:
320
+ yield
321
+ finally:
322
+ self._full_params_roots.remove(root) # one entry per scope
323
+ restore = self._fsdp_reshard_after_forward if root is self.get_trainable_module() else True
324
+ for m in fsdp_modules:
325
+ m.reshard()
326
+ if m is not root: # the root keeps FSDP's default (no reshard after forward)
327
+ m.set_reshard_after_forward(restore, recurse=False)
328
+
329
+ def has_full_params(self, module: nn.Module) -> bool:
330
+ """ True if `module` (the model or its EMA) can run w/o collectives: no FSDP, or held by full_params(). """
331
+ return not self._fsdp_enabled or any(m is module for m in self._full_params_roots)
332
+
333
+ # --- checkpoint state ---
334
+ def _module_state_dict(self, module: nn.Module) -> Dict[str, Any]:
335
+ """ Full state dict for a module, gathering shards under FSDP (collective, populated on rank 0 only). """
336
+ if not self._fsdp_enabled:
337
+ return strip_compiled_keys(module.state_dict())
338
+ from torch.distributed.checkpoint.state_dict import get_model_state_dict, StateDictOptions
339
+ options = StateDictOptions(full_state_dict=True, cpu_offload=True)
340
+ return strip_compiled_keys(get_model_state_dict(module, options=options))
341
+
342
+ def _load_module_state_dict(self, module: nn.Module, state_dict: Dict[str, Any]):
343
+ sd = strip_compiled_keys(clean_state_dict(state_dict))
344
+ if self._fsdp_enabled:
345
+ # DCP state dict FQNs are already free of torch.compile `_orig_mod` segments, load the clean names
346
+ from torch.distributed.checkpoint.state_dict import set_model_state_dict, StateDictOptions
347
+ set_model_state_dict(module, sd, options=StateDictOptions(full_state_dict=True))
348
+ return
349
+ model_keys = list(module.state_dict().keys())
350
+ if any('_orig_mod' in k for k in model_keys):
351
+ # remap clean names onto the live model's keys when its blocks are compiled in place
352
+ clean_to_model = {k.replace('._orig_mod.', '.').removeprefix('_orig_mod.'): k for k in model_keys}
353
+ sd = {clean_to_model.get(k, k): v for k, v in sd.items()}
354
+ module.load_state_dict(sd)
355
+
356
+ def model_state_dict(self) -> Dict[str, Any]:
357
+ state = {'model': self._module_state_dict(self.get_trainable_module())}
358
+ if self.trainable_module_ema is not None:
359
+ state['model_ema'] = self._module_state_dict(self.trainable_module_ema.module)
360
+ return state
361
+
362
+ def load_model_state_dict(self, state: Dict[str, Any]):
363
+ self._load_module_state_dict(self.get_trainable_module(), state.get('model', state))
364
+ if self.trainable_module_ema is not None:
365
+ if 'model_ema' in state:
366
+ self._load_module_state_dict(self.trainable_module_ema.module, state['model_ema'])
367
+ else:
368
+ _logger.warning('Checkpoint has no EMA weights, initializing the EMA from the loaded model weights.')
369
+ self.trainable_module_ema.set(self.get_trainable_module())