genml-kit 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (423) hide show
  1. {genml_kit-0.2.0 → genml_kit-0.3.0}/PKG-INFO +1983 -68
  2. {genml_kit-0.2.0 → genml_kit-0.3.0}/README.md +1961 -66
  3. genml_kit-0.3.0/genml_kit/augmentations/multicrop.py +113 -0
  4. genml_kit-0.3.0/genml_kit/cli/__init__.py +12 -0
  5. {genml_kit-0.2.0/genml_kit/training → genml_kit-0.3.0/genml_kit/cli}/infer.py +43 -79
  6. genml_kit-0.3.0/genml_kit/cli/llm.py +53 -0
  7. genml_kit-0.3.0/genml_kit/cli/tokenizer.py +146 -0
  8. genml_kit-0.3.0/genml_kit/cli/train.py +526 -0
  9. genml_kit-0.3.0/genml_kit/cli/tune.py +303 -0
  10. genml_kit-0.3.0/genml_kit/datasets/__init__.py +49 -0
  11. genml_kit-0.3.0/genml_kit/datasets/base.py +156 -0
  12. genml_kit-0.3.0/genml_kit/datasets/batch_policy.py +139 -0
  13. genml_kit-0.3.0/genml_kit/datasets/capped.py +125 -0
  14. genml_kit-0.3.0/genml_kit/datasets/counting.py +784 -0
  15. genml_kit-0.3.0/genml_kit/datasets/det_folder.py +285 -0
  16. genml_kit-0.3.0/genml_kit/datasets/det_transforms.py +198 -0
  17. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/ensemble.py +216 -28
  18. genml_kit-0.3.0/genml_kit/datasets/factory.py +85 -0
  19. genml_kit-0.3.0/genml_kit/datasets/field_selector.py +124 -0
  20. genml_kit-0.3.0/genml_kit/datasets/hf_proxy.py +489 -0
  21. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/image_folder.py +3 -3
  22. genml_kit-0.3.0/genml_kit/datasets/iterable.py +282 -0
  23. genml_kit-0.3.0/genml_kit/datasets/length_bucket_sampler.py +92 -0
  24. genml_kit-0.3.0/genml_kit/datasets/packing_sampler.py +108 -0
  25. genml_kit-0.3.0/genml_kit/datasets/preference_proxy.py +138 -0
  26. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/replay_buffer.py +11 -0
  27. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/retry.py +10 -3
  28. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/rollout_buffer.py +109 -64
  29. genml_kit-0.3.0/genml_kit/datasets/seg_folder.py +206 -0
  30. genml_kit-0.3.0/genml_kit/datasets/split.py +213 -0
  31. genml_kit-0.3.0/genml_kit/datasets/stream_batching.py +212 -0
  32. genml_kit-0.3.0/genml_kit/datasets/text_proxy.py +267 -0
  33. genml_kit-0.3.0/genml_kit/datasets/transforms.py +223 -0
  34. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/vo_pairs.py +41 -10
  35. genml_kit-0.3.0/genml_kit/datasets/webdataset_source.py +180 -0
  36. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/geometry/similarity.py +5 -2
  37. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/io/checkpointing.py +141 -32
  38. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/io/storage_utils.py +33 -32
  39. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/__init__.py +28 -0
  40. genml_kit-0.3.0/genml_kit/losses/alignment.py +104 -0
  41. genml_kit-0.3.0/genml_kit/losses/builtin.py +208 -0
  42. genml_kit-0.3.0/genml_kit/losses/detection.py +342 -0
  43. genml_kit-0.3.0/genml_kit/losses/distillation.py +241 -0
  44. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/registry.py +2 -8
  45. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/rl.py +26 -19
  46. genml_kit-0.3.0/genml_kit/losses/segmentation.py +131 -0
  47. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/__init__.py +4 -5
  48. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/base.py +169 -49
  49. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/byol.py +29 -24
  50. genml_kit-0.3.0/genml_kit/methods/causal_lm.py +370 -0
  51. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/classification.py +141 -31
  52. genml_kit-0.3.0/genml_kit/methods/detection.py +326 -0
  53. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/dino.py +46 -27
  54. genml_kit-0.3.0/genml_kit/methods/distillation.py +577 -0
  55. genml_kit-0.3.0/genml_kit/methods/dpo.py +183 -0
  56. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/ijepa.py +138 -44
  57. genml_kit-0.3.0/genml_kit/methods/reward_model.py +187 -0
  58. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/rl_dqn.py +27 -25
  59. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/rl_ppo.py +31 -39
  60. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/rl_sac.py +83 -58
  61. genml_kit-0.3.0/genml_kit/methods/rl_utils.py +130 -0
  62. genml_kit-0.3.0/genml_kit/methods/segmentation.py +358 -0
  63. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/simmim.py +13 -7
  64. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/supcon.py +24 -8
  65. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/vo_pair.py +55 -35
  66. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/__init__.py +12 -5
  67. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/byol.py +4 -5
  68. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/loader.py +6 -4
  69. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/model.py +20 -1
  70. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/loader.py +25 -1
  71. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/masked_encoder.py +16 -1
  72. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/model.py +44 -7
  73. genml_kit-0.3.0/genml_kit/models/decoders/__init__.py +109 -0
  74. genml_kit-0.3.0/genml_kit/models/decoders/base.py +133 -0
  75. genml_kit-0.3.0/genml_kit/models/decoders/fcos_head.py +170 -0
  76. genml_kit-0.3.0/genml_kit/models/decoders/fpn.py +90 -0
  77. genml_kit-0.3.0/genml_kit/models/decoders/linear.py +60 -0
  78. genml_kit-0.3.0/genml_kit/models/decoders/unet.py +105 -0
  79. genml_kit-0.3.0/genml_kit/models/det_model_wrapper/__init__.py +7 -0
  80. genml_kit-0.3.0/genml_kit/models/det_model_wrapper/loader.py +147 -0
  81. genml_kit-0.3.0/genml_kit/models/det_model_wrapper/model.py +122 -0
  82. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/dino.py +3 -4
  83. genml_kit-0.3.0/genml_kit/models/ema.py +276 -0
  84. genml_kit-0.3.0/genml_kit/models/encoder_utils.py +157 -0
  85. genml_kit-0.3.0/genml_kit/models/generative.py +91 -0
  86. genml_kit-0.3.0/genml_kit/models/layer_match.py +203 -0
  87. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/registry.py +57 -22
  88. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/actor_critic.py +13 -2
  89. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/qnetwork.py +11 -3
  90. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/sac_critic.py +7 -7
  91. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/spaces.py +5 -2
  92. genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/__init__.py +7 -0
  93. genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/encoder.py +119 -0
  94. genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/loader.py +184 -0
  95. genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/model.py +118 -0
  96. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/timm/loader.py +3 -2
  97. genml_kit-0.3.0/genml_kit/models/tokenizer.py +291 -0
  98. genml_kit-0.3.0/genml_kit/models/tokenizer_train.py +346 -0
  99. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/uvito/loader.py +7 -1
  100. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/uvito/model.py +14 -10
  101. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/vo/__init__.py +2 -0
  102. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/vo/loader.py +16 -3
  103. genml_kit-0.3.0/genml_kit/models/vo/vo_similar.py +236 -0
  104. genml_kit-0.3.0/genml_kit/pipelines/__init__.py +34 -0
  105. genml_kit-0.3.0/genml_kit/pipelines/base.py +282 -0
  106. genml_kit-0.3.0/genml_kit/pipelines/contracts.py +91 -0
  107. genml_kit-0.3.0/genml_kit/pipelines/detection.py +225 -0
  108. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/images.py +187 -86
  109. genml_kit-0.3.0/genml_kit/pipelines/preference.py +197 -0
  110. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/rl.py +87 -137
  111. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/rl_normalize.py +3 -2
  112. genml_kit-0.3.0/genml_kit/pipelines/segmentation.py +421 -0
  113. genml_kit-0.3.0/genml_kit/pipelines/text.py +515 -0
  114. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/vo_pair.py +31 -16
  115. genml_kit-0.3.0/genml_kit/search/__init__.py +35 -0
  116. genml_kit-0.3.0/genml_kit/search/errors.py +17 -0
  117. genml_kit-0.3.0/genml_kit/search/pruners.py +242 -0
  118. genml_kit-0.3.0/genml_kit/search/runner.py +307 -0
  119. genml_kit-0.3.0/genml_kit/search/samplers.py +293 -0
  120. genml_kit-0.3.0/genml_kit/search/space.py +306 -0
  121. genml_kit-0.3.0/genml_kit/search/store.py +101 -0
  122. genml_kit-0.3.0/genml_kit/search/study.py +139 -0
  123. genml_kit-0.3.0/genml_kit/search/trial.py +120 -0
  124. genml_kit-0.3.0/genml_kit/training/budget.py +208 -0
  125. genml_kit-0.3.0/genml_kit/training/calibration.py +123 -0
  126. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/classifiers/__init__.py +24 -35
  127. genml_kit-0.3.0/genml_kit/training/det_metrics.py +224 -0
  128. genml_kit-0.3.0/genml_kit/training/generate.py +231 -0
  129. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/grad_monitor.py +9 -6
  130. genml_kit-0.3.0/genml_kit/training/infer.py +71 -0
  131. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/labels.py +11 -5
  132. genml_kit-0.3.0/genml_kit/training/llm_eval.py +366 -0
  133. genml_kit-0.3.0/genml_kit/training/llm_model.py +148 -0
  134. genml_kit-0.3.0/genml_kit/training/metrics_sinks.py +364 -0
  135. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/model_utils.py +158 -11
  136. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/optim_factory.py +141 -13
  137. genml_kit-0.3.0/genml_kit/training/packed_attention.py +199 -0
  138. genml_kit-0.3.0/genml_kit/training/preference_collator.py +112 -0
  139. genml_kit-0.3.0/genml_kit/training/rl_trainer.py +712 -0
  140. genml_kit-0.3.0/genml_kit/training/run_manifest.py +125 -0
  141. genml_kit-0.3.0/genml_kit/training/seg_metrics.py +126 -0
  142. genml_kit-0.3.0/genml_kit/training/sequence_mode.py +103 -0
  143. genml_kit-0.3.0/genml_kit/training/text_collator.py +390 -0
  144. genml_kit-0.3.0/genml_kit/training/train.py +292 -0
  145. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/train_compat.py +58 -13
  146. genml_kit-0.3.0/genml_kit/training/train_reporting.py +676 -0
  147. genml_kit-0.3.0/genml_kit/training/trainer.py +747 -0
  148. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/video_utils.py +1 -1
  149. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/vo/__init__.py +2 -0
  150. genml_kit-0.3.0/genml_kit/training/vo/eval_vo.py +132 -0
  151. genml_kit-0.3.0/genml_kit/training/vo/train_vo.py +241 -0
  152. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/xgb_pipeline.py +16 -12
  153. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/xgb_utils.py +7 -1
  154. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/args.py +71 -61
  155. genml_kit-0.3.0/genml_kit/utils/attr.py +113 -0
  156. genml_kit-0.3.0/genml_kit/utils/cli.py +237 -0
  157. genml_kit-0.3.0/genml_kit/utils/distributed.py +1056 -0
  158. genml_kit-0.3.0/genml_kit/utils/generation.py +449 -0
  159. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/gpu.py +18 -1
  160. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/logging.py +7 -2
  161. genml_kit-0.3.0/genml_kit/utils/script.py +213 -0
  162. genml_kit-0.3.0/genml_kit/utils/seg.py +236 -0
  163. genml_kit-0.3.0/genml_kit/utils/shared_args.py +537 -0
  164. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/signal.py +6 -4
  165. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/PKG-INFO +1983 -68
  166. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/SOURCES.txt +166 -1
  167. genml_kit-0.3.0/genml_kit.egg-info/entry_points.txt +6 -0
  168. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/requires.txt +24 -1
  169. genml_kit-0.3.0/pyproject.toml +97 -0
  170. genml_kit-0.3.0/tests/test_alignment_losses.py +101 -0
  171. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_args_frozen.py +1 -27
  172. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_attr_utils.py +26 -7
  173. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_balanced_sampler.py +4 -4
  174. genml_kit-0.3.0/tests/test_batch_policy.py +241 -0
  175. genml_kit-0.3.0/tests/test_budget.py +135 -0
  176. genml_kit-0.3.0/tests/test_budget_distributed.py +157 -0
  177. genml_kit-0.3.0/tests/test_cache_dir.py +92 -0
  178. genml_kit-0.3.0/tests/test_calibration.py +164 -0
  179. genml_kit-0.3.0/tests/test_capped_dataset.py +111 -0
  180. genml_kit-0.3.0/tests/test_causal_lm.py +280 -0
  181. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_checkpointing.py +150 -16
  182. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_classifiers.py +4 -3
  183. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_cli_help.py +1 -3
  184. genml_kit-0.3.0/tests/test_cli_utils.py +262 -0
  185. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_contrastive_models.py +22 -0
  186. genml_kit-0.3.0/tests/test_counting.py +543 -0
  187. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_custom_models.py +71 -9
  188. genml_kit-0.3.0/tests/test_dataset_arg.py +90 -0
  189. genml_kit-0.3.0/tests/test_det_e2e.py +213 -0
  190. genml_kit-0.3.0/tests/test_det_folder.py +176 -0
  191. genml_kit-0.3.0/tests/test_det_head.py +129 -0
  192. genml_kit-0.3.0/tests/test_det_loss.py +116 -0
  193. genml_kit-0.3.0/tests/test_det_metrics.py +114 -0
  194. genml_kit-0.3.0/tests/test_det_transforms.py +123 -0
  195. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_dino.py +2 -1
  196. genml_kit-0.3.0/tests/test_distillation_losses.py +154 -0
  197. genml_kit-0.3.0/tests/test_distillation_method.py +306 -0
  198. genml_kit-0.3.0/tests/test_distributed.py +832 -0
  199. genml_kit-0.3.0/tests/test_distributed_e2e.py +186 -0
  200. genml_kit-0.3.0/tests/test_dpo.py +186 -0
  201. genml_kit-0.3.0/tests/test_ema.py +147 -0
  202. genml_kit-0.3.0/tests/test_ema_weights.py +262 -0
  203. genml_kit-0.3.0/tests/test_ema_weights_trainer.py +154 -0
  204. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_encoder_utils.py +58 -1
  205. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_ensemble_collate.py +5 -5
  206. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_ensemble_labels.py +9 -8
  207. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_ensemble_mixed_labels.py +15 -4
  208. genml_kit-0.3.0/tests/test_generation.py +273 -0
  209. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_grad_monitor.py +97 -102
  210. genml_kit-0.3.0/tests/test_hyperband.py +97 -0
  211. genml_kit-0.3.0/tests/test_ijepa.py +145 -0
  212. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_init_orthogonal.py +12 -6
  213. genml_kit-0.3.0/tests/test_iterable_base.py +260 -0
  214. genml_kit-0.3.0/tests/test_iterable_ddp.py +343 -0
  215. genml_kit-0.3.0/tests/test_iterable_ensemble.py +195 -0
  216. genml_kit-0.3.0/tests/test_iterable_flush.py +129 -0
  217. genml_kit-0.3.0/tests/test_iterable_hf_proxy.py +123 -0
  218. genml_kit-0.3.0/tests/test_iterable_resume.py +175 -0
  219. genml_kit-0.3.0/tests/test_iterable_split.py +189 -0
  220. genml_kit-0.3.0/tests/test_iterable_webdataset.py +412 -0
  221. genml_kit-0.3.0/tests/test_iterable_workers.py +240 -0
  222. genml_kit-0.3.0/tests/test_layer_match.py +183 -0
  223. genml_kit-0.3.0/tests/test_length_bucket_sampler.py +49 -0
  224. genml_kit-0.3.0/tests/test_llm_cli.py +195 -0
  225. genml_kit-0.3.0/tests/test_llm_e2e.py +108 -0
  226. genml_kit-0.3.0/tests/test_llm_eval.py +119 -0
  227. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_log_format.py +35 -33
  228. genml_kit-0.3.0/tests/test_logging_helpers.py +49 -0
  229. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_lora.py +99 -1
  230. genml_kit-0.3.0/tests/test_lora_targets.py +171 -0
  231. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_loss_and_multipliers.py +1 -1
  232. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_loss_registry.py +15 -3
  233. genml_kit-0.3.0/tests/test_make_tiny_coco.py +83 -0
  234. genml_kit-0.3.0/tests/test_md_latex.py +322 -0
  235. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_method_lifecycle.py +14 -7
  236. genml_kit-0.3.0/tests/test_metrics_sinks.py +242 -0
  237. genml_kit-0.3.0/tests/test_midepoch_resume.py +275 -0
  238. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_optim_factory.py +11 -11
  239. genml_kit-0.3.0/tests/test_owner_arg_defaults.py +221 -0
  240. genml_kit-0.3.0/tests/test_packed_attention.py +130 -0
  241. genml_kit-0.3.0/tests/test_packing_sampler.py +108 -0
  242. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pipeline.py +82 -15
  243. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pipeline_method_registry.py +106 -41
  244. genml_kit-0.3.0/tests/test_preference_collator.py +109 -0
  245. genml_kit-0.3.0/tests/test_preference_e2e.py +90 -0
  246. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pretrain_methods.py +53 -0
  247. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pretrain_smoke.py +2 -2
  248. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pretrain_vis.py +14 -14
  249. genml_kit-0.3.0/tests/test_readme_cli.py +50 -0
  250. genml_kit-0.3.0/tests/test_readme_cli_coverage.py +163 -0
  251. genml_kit-0.3.0/tests/test_readme_cli_harness.py +394 -0
  252. genml_kit-0.3.0/tests/test_rl_dist_e2e.py +107 -0
  253. genml_kit-0.3.0/tests/test_rl_distributed.py +540 -0
  254. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_learning.py +18 -42
  255. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_losses_phase2.py +19 -23
  256. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_method.py +5 -22
  257. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_model_selector.py +7 -48
  258. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_normalize.py +10 -9
  259. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_pipeline.py +50 -10
  260. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_ppo.py +7 -24
  261. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_sac.py +5 -21
  262. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_trainer.py +270 -198
  263. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_video.py +67 -23
  264. genml_kit-0.3.0/tests/test_run_manifest.py +98 -0
  265. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_s3r2util.py +1 -1
  266. genml_kit-0.3.0/tests/test_safe_len.py +285 -0
  267. genml_kit-0.3.0/tests/test_script_utils.py +229 -0
  268. genml_kit-0.3.0/tests/test_search_runner.py +244 -0
  269. genml_kit-0.3.0/tests/test_search_samplers.py +113 -0
  270. genml_kit-0.3.0/tests/test_search_space.py +123 -0
  271. genml_kit-0.3.0/tests/test_search_store_trial.py +125 -0
  272. genml_kit-0.3.0/tests/test_search_study_pruners.py +119 -0
  273. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_seed_utils.py +9 -9
  274. genml_kit-0.3.0/tests/test_seg_class_map.py +254 -0
  275. genml_kit-0.3.0/tests/test_seg_decoders.py +137 -0
  276. genml_kit-0.3.0/tests/test_seg_e2e.py +188 -0
  277. genml_kit-0.3.0/tests/test_seg_folder.py +199 -0
  278. genml_kit-0.3.0/tests/test_seg_hf_proxy.py +69 -0
  279. genml_kit-0.3.0/tests/test_seg_transforms.py +239 -0
  280. genml_kit-0.3.0/tests/test_seg_vis.py +174 -0
  281. genml_kit-0.3.0/tests/test_sequence_mode.py +71 -0
  282. genml_kit-0.3.0/tests/test_shared_args.py +104 -0
  283. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_signal_utils.py +4 -4
  284. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_simmim.py +41 -0
  285. genml_kit-0.3.0/tests/test_ssl_ddp.py +203 -0
  286. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_storage_utils.py +4 -5
  287. genml_kit-0.3.0/tests/test_streaming_classification.py +238 -0
  288. genml_kit-0.3.0/tests/test_streaming_text_pipeline.py +206 -0
  289. genml_kit-0.3.0/tests/test_text_collator.py +222 -0
  290. genml_kit-0.3.0/tests/test_text_pipeline.py +243 -0
  291. genml_kit-0.3.0/tests/test_tokenizer.py +117 -0
  292. genml_kit-0.3.0/tests/test_tokenizer_arg.py +95 -0
  293. genml_kit-0.3.0/tests/test_tokenizer_train.py +259 -0
  294. genml_kit-0.3.0/tests/test_train_cli_integration.py +82 -0
  295. genml_kit-0.3.0/tests/test_train_reporting.py +640 -0
  296. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_train_smoke.py +2 -2
  297. genml_kit-0.3.0/tests/test_trainer.py +654 -0
  298. genml_kit-0.3.0/tests/test_trainer_ddp.py +224 -0
  299. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_transformer_encoder_init.py +0 -1
  300. genml_kit-0.3.0/tests/test_vo_e2e.py +163 -0
  301. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_vo_pairs.py +1 -1
  302. genml_kit-0.3.0/tests/test_vo_similar.py +222 -0
  303. genml_kit-0.3.0/tests/test_vo_training.py +247 -0
  304. genml_kit-0.3.0/tests/test_wandb_sink.py +160 -0
  305. genml_kit-0.3.0/tests/test_warmup_schedulers.py +216 -0
  306. genml_kit-0.3.0/tests/test_webdataset_pipeline.py +94 -0
  307. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_xgb_pipeline.py +4 -2
  308. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_xgb_utils.py +7 -6
  309. genml_kit-0.2.0/genml_kit/augmentations/multicrop.py +0 -123
  310. genml_kit-0.2.0/genml_kit/datasets/__init__.py +0 -23
  311. genml_kit-0.2.0/genml_kit/datasets/factory.py +0 -58
  312. genml_kit-0.2.0/genml_kit/datasets/field_dataset.py +0 -40
  313. genml_kit-0.2.0/genml_kit/datasets/hf_proxy.py +0 -239
  314. genml_kit-0.2.0/genml_kit/datasets/transforms.py +0 -60
  315. genml_kit-0.2.0/genml_kit/losses/builtin.py +0 -106
  316. genml_kit-0.2.0/genml_kit/methods/rl_utils.py +0 -69
  317. genml_kit-0.2.0/genml_kit/models/encoder_utils.py +0 -89
  318. genml_kit-0.2.0/genml_kit/models/vo/vo_similar.py +0 -165
  319. genml_kit-0.2.0/genml_kit/pipelines/__init__.py +0 -25
  320. genml_kit-0.2.0/genml_kit/pipelines/base.py +0 -62
  321. genml_kit-0.2.0/genml_kit/pipelines/contracts.py +0 -21
  322. genml_kit-0.2.0/genml_kit/training/rl_trainer.py +0 -495
  323. genml_kit-0.2.0/genml_kit/training/train.py +0 -493
  324. genml_kit-0.2.0/genml_kit/training/train_reporting.py +0 -447
  325. genml_kit-0.2.0/genml_kit/training/trainer.py +0 -318
  326. genml_kit-0.2.0/genml_kit/training/vo/eval_vo.py +0 -108
  327. genml_kit-0.2.0/genml_kit/training/vo/train_vo.py +0 -156
  328. genml_kit-0.2.0/genml_kit/utils/attr.py +0 -141
  329. genml_kit-0.2.0/genml_kit/utils/cli.py +0 -110
  330. genml_kit-0.2.0/genml_kit/utils/script.py +0 -82
  331. genml_kit-0.2.0/genml_kit.egg-info/entry_points.txt +0 -3
  332. genml_kit-0.2.0/pyproject.toml +0 -64
  333. genml_kit-0.2.0/tests/test_cli_utils.py +0 -146
  334. genml_kit-0.2.0/tests/test_script_utils.py +0 -67
  335. genml_kit-0.2.0/tests/test_train_reporting.py +0 -416
  336. genml_kit-0.2.0/tests/test_trainer.py +0 -278
  337. genml_kit-0.2.0/tests/test_vo_e2e.py +0 -73
  338. genml_kit-0.2.0/tests/test_vo_similar.py +0 -124
  339. genml_kit-0.2.0/tests/test_vo_training.py +0 -113
  340. {genml_kit-0.2.0 → genml_kit-0.3.0}/LICENSE +0 -0
  341. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/__init__.py +0 -0
  342. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/augmentations/__init__.py +0 -0
  343. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/augmentations/dual_view.py +0 -0
  344. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/balanced_sampler.py +0 -0
  345. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/weighted_sampler.py +0 -0
  346. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/geometry/__init__.py +0 -0
  347. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/io/__init__.py +0 -0
  348. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/byol.py +0 -0
  349. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/contrastive.py +0 -0
  350. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/dino.py +0 -0
  351. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/focal.py +0 -0
  352. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/registry.py +0 -0
  353. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/attention_pooling.py +0 -0
  354. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/__init__.py +0 -0
  355. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/contrastive.py +0 -0
  356. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/__init__.py +0 -0
  357. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/processor.py +0 -0
  358. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/init.py +0 -0
  359. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/masked_encoder.py +0 -0
  360. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/processors/__init__.py +0 -0
  361. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/processors/base.py +0 -0
  362. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/__init__.py +0 -0
  363. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/sac_model.py +0 -0
  364. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/simmim.py +0 -0
  365. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/timm/__init__.py +0 -0
  366. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/timm/model.py +0 -0
  367. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/timm/processor.py +0 -0
  368. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/transformer_utils.py +0 -0
  369. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/uvito/__init__.py +0 -0
  370. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/uvito/processor.py +0 -0
  371. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/vo/processor.py +0 -0
  372. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/registry.py +0 -0
  373. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/__init__.py +0 -0
  374. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/classifiers/base.py +0 -0
  375. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/classifiers/cls_attention.py +0 -0
  376. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/classifiers/mlp.py +0 -0
  377. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/eval.py +0 -0
  378. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/metrics.py +0 -0
  379. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/param_align.py +0 -0
  380. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/tta.py +0 -0
  381. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/__init__.py +0 -0
  382. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/env.py +0 -0
  383. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/image_dump.py +0 -0
  384. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/label.py +0 -0
  385. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/registry.py +0 -0
  386. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/seed.py +0 -0
  387. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/table.py +0 -0
  388. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/transformer.py +0 -0
  389. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/dependency_links.txt +0 -0
  390. {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/top_level.txt +0 -0
  391. {genml_kit-0.2.0 → genml_kit-0.3.0}/setup.cfg +0 -0
  392. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_actor_critic.py +0 -0
  393. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_attention_pooling.py +0 -0
  394. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_byol.py +0 -0
  395. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_datasets_ensemble.py +0 -0
  396. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_ensemble_ensure_label_space.py +0 -0
  397. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_env.py +0 -0
  398. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_evaluation_metrics.py +0 -0
  399. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_extract_features.py +0 -0
  400. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_geometry_similarity.py +0 -0
  401. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_headless_models.py +0 -0
  402. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_image_dump.py +0 -0
  403. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_infer_topk.py +0 -0
  404. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_logging_utils.py +0 -0
  405. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_metrics.py +0 -0
  406. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_model_script_loading.py +0 -0
  407. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_param_align.py +0 -0
  408. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_replay_buffer.py +0 -0
  409. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_retry.py +0 -0
  410. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_cli.py +0 -0
  411. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_losses.py +0 -0
  412. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_models.py +0 -0
  413. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_spaces.py +0 -0
  414. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rollout_buffer.py +0 -0
  415. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_set_train_mode.py +0 -0
  416. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_supcon_loss.py +0 -0
  417. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_supcon_method.py +0 -0
  418. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_table_utils.py +0 -0
  419. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_timm_models.py +0 -0
  420. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_tta.py +0 -0
  421. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_utils_registry.py +0 -0
  422. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_uvito.py +0 -0
  423. {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_weighted_sampler.py +0 -0
@@ -1,8 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: genml_kit
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: General-purpose ML toolkit: image-classification, visual-odometry and self-supervised pre-training through one unified harness.
5
5
  License: Apache-2.0
6
+ Project-URL: Homepage, https://github.com/davidel/genml_kit
7
+ Project-URL: Repository, https://github.com/davidel/genml_kit
8
+ Project-URL: Issues, https://github.com/davidel/genml_kit/issues
6
9
  Requires-Python: >=3.9
7
10
  Description-Content-Type: text/markdown
8
11
  License-File: LICENSE
@@ -11,6 +14,7 @@ Requires-Dist: torch
11
14
  Requires-Dist: torchvision
12
15
  Requires-Dist: transformers
13
16
  Requires-Dist: datasets
17
+ Requires-Dist: webdataset
14
18
  Requires-Dist: xgboost>=2.0
15
19
  Requires-Dist: scikit-learn
16
20
  Requires-Dist: pillow
@@ -21,6 +25,10 @@ Provides-Extra: s3
21
25
  Requires-Dist: boto3>=1.28; extra == "s3"
22
26
  Provides-Extra: lora
23
27
  Requires-Dist: peft>=0.7.0; extra == "lora"
28
+ Provides-Extra: llm
29
+ Requires-Dist: peft>=0.7.0; extra == "llm"
30
+ Provides-Extra: llm-spm
31
+ Requires-Dist: sentencepiece>=0.1.99; extra == "llm-spm"
24
32
  Provides-Extra: timm
25
33
  Requires-Dist: timm>=1.0; extra == "timm"
26
34
  Provides-Extra: uvito
@@ -29,21 +37,37 @@ Provides-Extra: rl
29
37
  Requires-Dist: gymnasium>=0.29; extra == "rl"
30
38
  Requires-Dist: pygame>=2.1; extra == "rl"
31
39
  Requires-Dist: imageio[ffmpeg]>=2.31; extra == "rl"
40
+ Requires-Dist: box2d>=2.3.10; extra == "rl"
41
+ Provides-Extra: tune
42
+ Requires-Dist: scipy; extra == "tune"
43
+ Provides-Extra: gpu
44
+ Requires-Dist: nvidia-ml-py; extra == "gpu"
45
+ Provides-Extra: flash
46
+ Requires-Dist: flash-attn; extra == "flash"
47
+ Provides-Extra: wandb
48
+ Requires-Dist: wandb>=0.16; extra == "wandb"
32
49
  Provides-Extra: all
33
- Requires-Dist: genml_kit[gcs,lora,rl,s3,timm,uvito]; extra == "all"
50
+ Requires-Dist: genml_kit[gcs,gpu,llm,llm-spm,lora,rl,s3,timm,tune,uvito,wandb]; extra == "all"
34
51
  Provides-Extra: dev
35
52
  Requires-Dist: pytest; extra == "dev"
53
+ Requires-Dist: pytest-timeout; extra == "dev"
54
+ Requires-Dist: pytest-split; extra == "dev"
36
55
  Requires-Dist: ruff; extra == "dev"
37
56
  Requires-Dist: yapf; extra == "dev"
57
+ Requires-Dist: pyyaml; extra == "dev"
38
58
  Dynamic: license-file
39
59
 
40
60
  # genml_kit
41
61
 
42
62
  [![CI](https://github.com/davidel/genml_kit/actions/workflows/ci.yml/badge.svg)](https://github.com/davidel/genml_kit/actions/workflows/ci.yml)
63
+ [![PyPI](https://img.shields.io/pypi/v/genml_kit.svg)](https://pypi.org/project/genml_kit/)
64
+ [![Python](https://img.shields.io/pypi/pyversions/genml_kit.svg)](https://pypi.org/project/genml_kit/)
65
+ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE)
43
66
 
44
- A general-purpose toolkit for image classification. Supports self-supervised
45
- pre-training, supervised fine-tuning, and XGBoost ensemble inference — all
46
- from the command line.
67
+ A general-purpose toolkit for image classification and decoder-only language
68
+ models. Supports self-supervised pre-training, supervised fine-tuning, LoRA,
69
+ causal-LM training / generation, preference alignment (DPO, reward models),
70
+ and XGBoost ensemble inference — all from the command line.
47
71
 
48
72
  `genml_kit` is domain-agnostic: point it at any HuggingFace dataset, local
49
73
  `ImageFolder` tree, or timm/HuggingFace backbone. It grew out of a
@@ -67,6 +91,14 @@ in [`vo/README.md`](vo/README.md).
67
91
  - [Gradient Monitor](#gradient-monitor)
68
92
  - [Custom Losses](#custom-losses)
69
93
  - [Custom Models](#custom-models)
94
+ - [Semantic Segmentation](#semantic-segmentation)
95
+ - [Object Detection](#object-detection)
96
+ - [Visual Odometry](#visual-odometry)
97
+ - [Knowledge Distillation](#knowledge-distillation)
98
+ - [Large Language Models (causal LM)](#large-language-models-causal-lm)
99
+ - [Reinforcement Learning](#reinforcement-learning)
100
+ - [Multi-GPU Training](#multi-gpu-training)
101
+ - [Hyperparameter Search](#hyperparameter-search)
70
102
  - [References And Further Reading](#references-and-further-reading)
71
103
  - [Development](#development)
72
104
  - [License](#license)
@@ -188,15 +220,17 @@ reuse the same hook.
188
220
 
189
221
  | Module | Contents |
190
222
  |---|---|
191
- | `genml_kit.training` | `train.py` and `infer.py` CLI harnesses, `optim_factory.py` (optimizers, LLRD, schedulers), `model_utils.py` (loading, freezing, feature extraction), `param_align.py`, `eval.py` + `metrics.py`, `grad_monitor.py`, `train_reporting.py`, `tta.py`, `xgb_utils.py` + `xgb_pipeline.py`, `classifiers/` (pluggable heads) |
192
- | `genml_kit.methods` | objective-side registry (classification, vo_pair, SimMIM, I-JEPA, DINO, BYOL, SupCon) |
193
- | `genml_kit.losses` | shared loss library (focal, NT-Xent contrastive, DINO self-distillation, BYOL) |
223
+ | `genml_kit.training` | `train.py` (dataset loading: `load_and_split_dataset`, plus legacy compat re-exports) and `infer.py` (inference helpers), `optim_factory.py` (optimizers, LLRD, schedulers), `model_utils.py` (loading, freezing, feature extraction), `param_align.py`, `eval.py` + `metrics.py`, `grad_monitor.py`, `train_reporting.py`, `tta.py`, `xgb_utils.py` + `xgb_pipeline.py`, `text_collator.py` + `preference_collator.py` (text / preference batches), `llm_model.py` + `generate.py` + `llm_eval.py` (LLM inference, generation, eval), `classifiers/` (pluggable heads) |
224
+ | `genml_kit.methods` | objective-side registry (classification, segmentation, detection, vo_pair, SimMIM, I-JEPA, DINO, BYOL, SupCon, distillation, `causal_lm`, `dpo`, `reward_model`) |
225
+ | `genml_kit.losses` | shared loss library (focal, Dice+CE segmentation, NT-Xent contrastive, DINO self-distillation, BYOL, KD distillation, alignment: `sequence_logps` / `dpo_loss` / `bradley_terry_loss`) |
194
226
  | `genml_kit.augmentations` | objective-level transforms (dual-view, multi-crop) |
195
- | `genml_kit.models` | model/processor registry; `timm/`, `convvit/`, `uvito/`, `cls_model_wrapper/` backends; `processors/base.py` |
196
- | `genml_kit.datasets` | `hf_proxy.py` (HuggingFace → PyTorch bridge), `image_folder.py`, `ensemble.py`, `field_dataset.py`, `balanced_sampler.py`, `weighted_sampler.py`, `retry.py`, `vo_pairs.py` (synthetic VO pairs) |
227
+ | `genml_kit.models` | model/processor/tokenizer registry; `timm/`, `convvit/`, `uvito/`, `cls_model_wrapper/`, `seg_model_wrapper/` backends; `decoders/` (pluggable segmentation decoders); `generative.py` (`GenerativeLM` contract), `tokenizer.py` (`TOKENIZERS`), `processors/base.py` |
228
+ | `genml_kit.datasets` | `hf_proxy.py` (HuggingFace → PyTorch bridge), `image_folder.py`, `seg_folder.py` (paired image/mask), `det_folder.py` (COCO detection), `text_proxy.py` (HF text → items), `preference_proxy.py` (chosen/rejected pairs), `ensemble.py`, `field_dataset.py`, `balanced_sampler.py`, `weighted_sampler.py`, `retry.py`, `vo_pairs.py` (synthetic VO pairs) |
197
229
  | `genml_kit.geometry` | `similarity.py` (2-D similarity algebra, batched differentiable Umeyama, backward-map warp) — the math behind the visual-odometry front-end lives in [`vo/README.md`](vo/README.md) |
198
230
  | `genml_kit.io` | `checkpointing.py` (atomic saves, LoRA state, remote fetch), `storage_utils.py` (S3 / GCS / R2) |
199
- | `genml_kit.utils` | glog-style logging, CLI arg groups, seeding, signal handling, GPU info, tables, external `.py` script loading, `image_dump`, transformer init helpers |
231
+ | `genml_kit.utils` | glog-style logging, CLI arg groups + `shared_args.py` (globally shared `ARG_*` CLI flags), seeding, signal handling, GPU info, tables, external `.py` script loading, `image_dump`, transformer init helpers, `generation.py` (samplers + batched `generate`) |
232
+ | `genml_kit.cli` | console-script entry points: `train.py` (`genml-kit-train`), `infer.py` (`genml-kit-infer`), `llm.py` (`genml-kit-llm`, `generate` / `eval` subcommands), `tune.py` (`genml-kit-tune`, `run` / `report` subcommands) |
233
+ | `genml_kit.search` | hyperparameter search: `space.py` (typed dimensions + argv encoding), `study.py` (declarative study spec), `samplers.py` (`SEARCHERS`: grid/random/tpe/gpei), `pruners.py` (`PRUNERS`: none/median/asha/hyperband), `runner.py` (subprocess-isolated trials), `store.py` (resumable ledger), `trial.py` |
200
234
 
201
235
  ## Installation
202
236
 
@@ -215,10 +249,30 @@ pip install "genml_kit[s3]"
215
249
  # With LoRA fine-tuning:
216
250
  pip install "genml_kit[lora]"
217
251
 
252
+ # With causal-LM fine-tuning (LoRA + transformers):
253
+ pip install "genml_kit[llm]"
254
+
255
+ # With a bare SentencePiece tokenizer.model (the 'spm' tokenizer entry):
256
+ pip install "genml_kit[llm-spm]"
257
+
218
258
  # With UVito model support:
219
259
  pip install "genml_kit[uvito]"
220
260
 
221
- # Everything above in one shot (gcs, s3, lora, timm, uvito):
261
+ # With the scipy-backed hyperparameter sampler (genml-kit-tune gpei):
262
+ pip install "genml_kit[tune]"
263
+
264
+ # With NVML GPU utilization in the training logs (the 'util=...%' field):
265
+ pip install "genml_kit[gpu]"
266
+
267
+ # With Weights & Biases metric streaming (the 'wandb' metrics sink):
268
+ pip install "genml_kit[wandb]"
269
+
270
+ # With flash-attn packing support (requires the CUDA toolkit with CUDA_HOME
271
+ # set to an nvcc, plus torch already installed):
272
+ pip install "genml_kit[flash]"
273
+
274
+ # Everything above in one shot (gcs, s3, lora, llm, llm-spm, timm, uvito, rl, tune, gpu, wandb):
275
+ # (flash is intentionally excluded: it needs a CUDA toolkit to build)
222
276
  pip install "genml_kit[all]"
223
277
  ```
224
278
 
@@ -229,10 +283,19 @@ NumPy, scikit-learn ≥ 1.3, XGBoost ≥ 2.0, Pillow, tensorboard.
229
283
 
230
284
  The fastest way to get started:
231
285
 
286
+ <!-- genml-cli-test
287
+ name: quick-start-cifar10
288
+ network: true
289
+ override:
290
+ - --dataset {tmp}/img
291
+ - -.--image_column
292
+ extra: --epochs 1 --max_items 16
293
+ -->
232
294
  ```bash
233
295
  # Fine-tune a ViT on an image-classification dataset (5 epochs, ~2 minutes on GPU)
234
296
  genml-kit-train --model google/vit-base-patch16-224 \
235
- --dataset cifar10 \
297
+ --dataset uoft-cs/cifar10 \
298
+ --image_column img \
236
299
  --label_column label \
237
300
  --epochs 5 \
238
301
  --batch_size 32 \
@@ -293,12 +356,13 @@ SimMIM minimizes the mean squared reconstruction error over masked patches:
293
356
 
294
357
  $$
295
358
  \large
296
- \mathcal{L}_{\text{MIM}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\| \hat{x}_i - x_i \right\|_2^2
359
+ \mathcal{L}_{\text{MIM}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\\| \hat{x}_i - x_i \right\\|_2^2
297
360
  $$
298
361
 
299
362
  where:
300
363
 
301
- - $`x_i`$ — the pixels of the $`i`$-th original patch;
364
+ - $`x_i`$ — the pixels of the $`i`$-th original patch ($`i = 1, \dots, N`$;
365
+ $`N`$ = total number of patches);
302
366
  - $`\hat{x}_i`$ — the decoder's reconstruction of that patch;
303
367
  - $`\mathcal{M}`$ — the set of masked patch indices, and $`|\mathcal{M}|`$ its
304
368
  size (how many patches are masked);
@@ -325,7 +389,7 @@ masked region. The objective is regression in representation space:
325
389
 
326
390
  $$
327
391
  \large
328
- \mathcal{L}_{\text{IJEPA}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\| q_\theta(f_\theta(\text{context}))_i - \mathop{\mathrm{stopgrad}}\left(f_\xi(x_i)\right) \right\|_2^2
392
+ \mathcal{L}_{\text{IJEPA}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\\| q_\theta(f_\theta(\text{context}))_i - \mathop{\mathrm{stopgrad}}\left(f_\xi(x_i)\right) \right\\|_2^2
329
393
  $$
330
394
 
331
395
  where:
@@ -344,7 +408,7 @@ an exponential moving average:
344
408
 
345
409
  $$
346
410
  \large
347
- \xi \leftarrow m \, \xi + (1 - m) \, \theta
411
+ \xi \leftarrow m \\, \xi + (1 - m) \\, \theta
348
412
  $$
349
413
 
350
414
  where $`m`$ is the momentum (`--teacher_momentum`) and the arrow denotes
@@ -371,6 +435,28 @@ Three mechanism details complete the picture:
371
435
  matures, the teacher freezes progressively, stabilizing the targets late in
372
436
  training.
373
437
 
438
+ Run it like any other image pre-training configuration — here on a small
439
+ unlabeled collection:
440
+
441
+ <!-- genml-cli-test
442
+ name: pretrain-ijepa
443
+ network: true
444
+ gated: true
445
+ override:
446
+ - --datasets {tmp}/img
447
+ extra: --epochs 1 --batch_size 2 --max_items 16 --predictor_heads 8 --predictor_depth 2 --ijepa_weight 0.5
448
+ -->
449
+ ```bash
450
+ genml-kit-train --pipeline images --method ijepa \
451
+ --model convvit \
452
+ --datasets uoft-cs/cifar10 \
453
+ --image_size 32 \
454
+ --batch_size 32 \
455
+ --epochs 100 \
456
+ --predictor_heads 8 \
457
+ --lr 1e-4
458
+ ```
459
+
374
460
  **SupCon** (Supervised Contrastive Learning): Uses labels to define "positive"
375
461
  pairs (same class) and "negative" pairs (different classes). The loss pulls
376
462
  features of same-class images together and pushes different-class features
@@ -406,7 +492,7 @@ Fix one example $`i`$, called the *anchor*. Its *positive set* is
406
492
 
407
493
  $$
408
494
  \large
409
- \mathcal{P}(i) = \{\, j : j \neq i \text{ and } y_j = y_i \,\}
495
+ \mathcal{P}(i) = \\{\\, j : j \neq i \text{ and } y_j = y_i \\,\\}
410
496
  $$
411
497
 
412
498
  i.e. all other examples in the batch that share the anchor's class label
@@ -469,25 +555,58 @@ into 2 **global** crops plus `--dino_local_num` (8) small **local** crops:
469
555
  the teacher sees only global views, the student sees all of them, so the
470
556
  student must produce globally consistent features from partial views.
471
557
 
472
- Both towers end in an MLP projection head whose output is turned into a
473
- probability vector over the projection dimension. The teacher's distribution
474
- is *sharpened* by its small temperature and *centered* by a running mean
475
- $`c \in \mathbb{R}^{D}`$ updated once per step,
558
+ Both towers end in an MLP projection head that maps each backbone feature to a
559
+ $`D`$-dimensional vector (`--dino_proj_dim`, 256); these vectors are turned
560
+ into probability distributions over the $`D`$ projection dimensions. The
561
+ teacher's logits are *centered* (a running-mean vector $`c`$ is subtracted) and
562
+ *sharpened* (divided by a small temperature $`\tau_\xi`$) before the softmax,
563
+ while the student's logits are divided only by its larger temperature
564
+ $`\tau_\theta`$. Writing $`f_\theta(x)`$ and $`f_\xi(x)`$ for the student and
565
+ teacher projections of an image $`x`$, the center is updated once per step,
476
566
 
477
567
  $$
478
568
  \large
479
- c \leftarrow m_c \, c + (1 - m_c) \cdot \frac{1}{B}\sum_{i=1}^{B} f_\xi(x_i)
569
+ c \leftarrow m_c \\, c + (1 - m_c) \cdot \frac{1}{B_g}\sum_{i=1}^{B_g} f_\xi(x_i)
480
570
  $$
481
571
 
482
- (`--dino_center_momentum` 0.9), and the loss is the cross-entropy between the
483
- teacher's processed distribution $`p_\xi`$ and the student's $`p_\theta`$,
484
- averaged over all (teacher global, student crop) pairings:
572
+ and the teacher target and student prediction are
573
+
574
+ $$
575
+ \large
576
+ p_\xi(x) = \mathrm{softmax}\left(\frac{f_\xi(x) - c}{\tau_\xi}\right),
577
+ \qquad
578
+ p_\theta(x) = \mathrm{softmax}\left(\frac{f_\theta(x)}{\tau_\theta}\right)
579
+ $$
485
580
 
486
581
  $$
487
582
  \large
488
- \mathcal{L} = - \sum_{j} p_{\xi,j} \, \log p_{\theta,j}
583
+ \mathcal{L} = - \frac{1}{N}\sum_{n=1}^{N} \sum_{d=1}^{D} p_\xi(x^{(n)})_d \\, \log p_\theta(x^{(n)})_d
489
584
  $$
490
585
 
586
+ where:
587
+
588
+ - $`f_\theta(x), f_\xi(x) \in \mathbb{R}^{D}`$ — the student and teacher
589
+ projection-head outputs for image $`x`$ ($`D`$ = projection dimension,
590
+ `--dino_proj_dim`);
591
+ - $`c \in \mathbb{R}^{D}`$ — the running center: a per-dimension mean of the
592
+ teacher's outputs, updated once per step with momentum $`m_c`$
593
+ (`--dino_center_momentum`, 0.9) over the $`B_g`$ teacher *global-crop*
594
+ outputs in the batch (two global views per image, so $`B_g = 2B`$). It is
595
+ subtracted from the teacher logits so that no single dimension can dominate
596
+ the target;
597
+ - $`\tau_\xi`$ — the teacher temperature (`--dino_teacher_temp`, 0.04); small,
598
+ so the teacher distribution is sharp (confident);
599
+ - $`\tau_\theta`$ — the student temperature (`--dino_student_temp`, 0.1);
600
+ larger, so the student distribution is softer and its gradients stay alive;
601
+ - $`p_\xi(x)`$ — the teacher's target distribution over the $`D`$ dimensions
602
+ (centered, sharpened, softmaxed, and detached from the gradient graph);
603
+ - $`p_\theta(x)`$ — the student's predicted distribution over the same
604
+ $`D`$ dimensions;
605
+ - $`x^{(n)}`$ — the $`n`$-th (student crop, teacher view) pairing: each global
606
+ view is paired with the *other* global view of the same image, and each local
607
+ crop with the teacher's second global view. The loss averages the per-pairing
608
+ cross-entropy over all $`N`$ pairings.
609
+
491
610
  Centering and sharpening are the two anti-collapse mechanisms: centering
492
611
  cancels any component the teacher produces for *every* input (a constant
493
612
  output carries no information and would otherwise still lower the loss), and
@@ -496,27 +615,72 @@ its argmax rather than drifting toward a flat, uninformative distribution.
496
615
  Because the targets come from the teacher and the teacher follows the student,
497
616
  this is self-distillation — there are no external labels anywhere.
498
617
 
618
+ A DINO run adds the multi-crop knobs on top of the standard configuration:
619
+
620
+ <!-- genml-cli-test
621
+ name: pretrain-dino
622
+ network: true
623
+ gated: true
624
+ override:
625
+ - --datasets {tmp}/img
626
+ extra: --epochs 1 --batch_size 2 --max_items 16 --dino_local_num 2 --dino_local_size 16 --dino_global_size 32
627
+ -->
628
+ ```bash
629
+ genml-kit-train --pipeline images --method dino \
630
+ --model convvit \
631
+ --datasets uoft-cs/cifar10 \
632
+ --image_size 32 \
633
+ --batch_size 32 \
634
+ --epochs 100 \
635
+ --lr 1e-4
636
+ ```
637
+
499
638
  **BYOL** (**B**ootstrap **Y**our **O**wn **L**atent): Grill et al.,
500
639
  [Bootstrap Your Own Latent](https://arxiv.org/abs/2006.07733), NeurIPS 2020.
501
640
  The same two-tower idea as DINO but with two augmented views and no
502
641
  multi-crop. The **online** network (encoder + projector + predictor MLP) and
503
642
  the **target** network (encoder + projector, EMA copy) process the two views;
504
- only the online tower receives gradients. With $`z_1, z_2`$ the target
505
- projections of the two views and $`p_1, p_2`$ the online *predictions* of
506
- those projections, the loss is the symmetric cosine regression
643
+ only the online tower receives gradients. The loss is the symmetric cosine
644
+ regression
507
645
 
508
646
  $$
509
647
  \large
510
648
  \mathcal{L} = 2 - \cos(p_1, z_2) - \cos(p_2, z_1)
511
649
  $$
512
650
 
513
- which is 0 when predictions perfectly match target projections and reaches 4
514
- when they point in opposite directions; $`\cos(u,v) = u^\top v / (\lVert u
515
- \rVert_2 \lVert v \rVert_2)`$ is the cosine of the angle between the two
516
- vectors. Two asymmetries keep this from collapsing to a constant: the
517
- predictor exists only on the online side (predicting a target is easier than
518
- being one), and the targets move slowly under the EMA — exactly the
519
- teacher-follows-student trick noted under I-JEPA.
651
+ where:
652
+
653
+ - $`z_1, z_2`$ — the target network's projections of the two views
654
+ (stop-gradient, so no gradients flow into the target tower);
655
+ - $`p_1, p_2`$ — the online network's *predictions* of those projections;
656
+ - $`\cos(u, v) = u^\top v / (\lVert u \rVert_2 \lVert v \rVert_2)`$ — the
657
+ cosine of the angle between the two vectors.
658
+
659
+ The loss is 0 when predictions perfectly match the target projections and
660
+ reaches 4 when they point in opposite directions. Two asymmetries keep this
661
+ from collapsing to a constant: the predictor exists only on the online side
662
+ (predicting a target is easier than being one), and the targets move slowly
663
+ under the EMA — exactly the teacher-follows-student trick noted under I-JEPA.
664
+
665
+ The BYOL configuration differs only in the method name:
666
+
667
+ <!-- genml-cli-test
668
+ name: pretrain-byol
669
+ network: true
670
+ gated: true
671
+ override:
672
+ - --datasets {tmp}/img
673
+ extra: --epochs 1 --batch_size 2 --max_items 16 --byol_proj_hidden 256 --byol_predictor_hidden 256 --byol_final_momentum 1.0
674
+ -->
675
+ ```bash
676
+ genml-kit-train --pipeline images --method byol \
677
+ --model convvit \
678
+ --datasets uoft-cs/cifar10 \
679
+ --image_size 32 \
680
+ --batch_size 32 \
681
+ --epochs 100 \
682
+ --lr 1e-4
683
+ ```
520
684
 
521
685
  ### Typical Hyperparameters
522
686
 
@@ -546,6 +710,9 @@ size and GPU memory:
546
710
  - DINO and BYOL train on smaller crops (224) with a larger LR because their
547
711
  loss acts on a small projection space rather than every pixel; the defaults
548
712
  above match the method-specific `--dino_*` / `--byol_*` flags.
713
+ - The DINO/BYOL loss lives in that projection space (`--dino_proj_dim` /
714
+ `--byol_proj_dim`, both 256), not in the backbone feature space: the
715
+ objective compares 256-dimensional distributions or vectors, not pixels.
549
716
  - `--samples_per_class 16` with `--batch_size 64` gives 4 classes per batch
550
717
  on a 7-class dataset. batch_size need not be divisible by
551
718
  samples_per_class; the remainder is spread evenly across the per-class
@@ -555,6 +722,16 @@ size and GPU memory:
555
722
 
556
723
  ### Example: Full Pre-Training Pipeline
557
724
 
725
+ <!-- genml-cli-test
726
+ name: pretrain-full-pipeline
727
+ network: true
728
+ override:
729
+ - --datasets {tmp}/img
730
+ - --dataset {tmp}/img
731
+ - --label_column label
732
+ - -.--source_checkpoint
733
+ extra: --epochs 1
734
+ -->
558
735
  ```bash
559
736
  # Step 1: Pre-train with SimMIM on two large image collections.
560
737
  # v4.2: pre-training is a *configuration* of the unified genml-kit-train
@@ -584,6 +761,14 @@ genml-kit-train --model convvit \
584
761
 
585
762
  ### Example: Supervised Contrastive Pre-Training
586
763
 
764
+ <!-- genml-cli-test
765
+ name: pretrain-supcon
766
+ network: true
767
+ override:
768
+ - --datasets {tmp}/img
769
+ - --label_column label
770
+ extra: --epochs 1
771
+ -->
587
772
  ```bash
588
773
  genml-kit-train --pipeline images --method supcon \
589
774
  --model convvit \
@@ -611,7 +796,6 @@ genml-kit-train --pipeline images --method supcon \
611
796
  | `--datasets` | (required) | Space-separated dataset names or local paths. |
612
797
  | `--cache_dir` | `None` | HuggingFace cache directory for downloads. |
613
798
  | `--remote_checkpoint` | `None` | Remote URI for checkpoint sync (`gs://BUCKET/PREFIX`, `r2://BUCKET/PREFIX`, or `s3://BUCKET/PREFIX`). |
614
- | `--hf_token` | `None` | HuggingFace token for gated datasets (or set `HF_TOKEN` env var). |
615
799
  | `--image_column` | auto-detected | Explicit HF image column name. |
616
800
  | `--label_column` | `None` | Explicit HF label column name. Required by `--method supcon` if non-standard. |
617
801
  | `STRICT_DATASETS` (env) | `False` | Abort on first dataset-loading failure instead of skipping. |
@@ -619,6 +803,8 @@ genml-kit-train --pipeline images --method supcon \
619
803
  | `--batch_size` | `32` | Per-GPU batch size. |
620
804
  | `--seed` | `None` | Explicit RNG seed: pins the shuffling and enables full determinism (cuDNN deterministic kernels, benchmark off). Omit (default) to keep runs fast — RNG streams are still seeded from `$GENML_KIT_SEED` (default 42). See [Reproducibility](#reproducibility). |
621
805
  | `--epochs` | `5` | Number of epochs. The recipes below override this for pre-training. |
806
+ | `--max_batches` | unset | Cap the micro-batches **per epoch**; the next epoch still runs (up to `--epochs`). Unset drains the loader each epoch. |
807
+ | `--max_train_time` | unset | Wall-clock budget **in seconds for the whole run**. Stops mid-epoch once exceeded (DDP-synchronized); unset means no time limit. |
622
808
  | `--lr` | `3e-5` | Peak learning rate (base). The recipes below override this. |
623
809
  | `--amp_dtype` | `None` | Mixed precision: `float16` or `bfloat16`. Omit to disable. |
624
810
  | `--num_workers` | `4` | DataLoader worker processes. |
@@ -633,6 +819,10 @@ genml-kit-train --pipeline images --method supcon \
633
819
  | `--norm_history` | `0` | Keep last N norm snapshots per parameter for trend analysis. |
634
820
  | `--trend_top_n` | `10` | Show top N params in trend table by abs change %. `0` = show all. |
635
821
  | `--grad_clip` | `1.0` | Maximum gradient norm for clipping. `0` disables. |
822
+ | `--ema_decay` | `0.0` | Weights-EMA decay for validation and the best checkpoint. `0` disables the shadow (supported methods only -- see [Weights EMA](#weights-ema)). |
823
+ | `--ema_final_decay` | `None` | Optional ramp target: the effective decay moves linearly to this value over the run. |
824
+ | `--ema_warmup_steps` | `0` | Linear warmup of the effective decay from `0` up to `--ema_decay` over this many optimizer steps. Ignored when `--ema_final_decay` is set. |
825
+ | `--ema_eval` | `False` | Evaluate and best-save the EMA weights instead of the raw trajectory (`best.pt` carries EMA weights; `latest.pt` always carries the raw trajectory plus the shadow). |
636
826
  | `--lr_group` | `None` | Per-parameter-group learning rates (repeatable). Format: `"REGEX=LR"`. |
637
827
  | `--llrd_decay` | `None` | Layer-wise learning rate decay factor. |
638
828
  | `--vis_every` | `0` | Log reconstruction visualisation every N steps (SimMIM only). |
@@ -641,8 +831,9 @@ genml-kit-train --pipeline images --method supcon \
641
831
  | `--proc_arg` | `{}` | Override processor configuration (repeatable). |
642
832
  | `--optimizer` | `AdamW` | `torch.optim` optimizer class name or `.py` script path. |
643
833
  | `--opt_arg` | `{}` | Extra optimizer kwargs (repeatable). |
644
- | `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path. |
834
+ | `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path, or a built-in warmup shape: `warmup_cosine`, `warmup_linear`, `warmup_rsqrt`. |
645
835
  | `--sched_arg` | `{}` | Extra scheduler kwargs (repeatable). |
836
+ | `--warmup_epochs` | `0` | Linear LR ramp length in epochs (integer or fraction) for the `warmup_*` schedulers; 0 disables the ramp. See [Warmup](#warmup). |
646
837
  | `--source_checkpoint` | `None` | Path to source checkpoint to absorb parameters from. |
647
838
  | `--param_rename` | `None` | Regex-based key rename patterns (`SEARCH;REPLACE`). |
648
839
  | `--grad_checkpoint` | `False` | Enable gradient checkpointing. Reduces activation memory by ~40-50% at the cost of ~25-35% more compute per step. Enables larger batch sizes. See [Gradient Checkpointing](#gradient-checkpointing). |
@@ -669,8 +860,8 @@ single dataset is large enough for effective pre-training on its own.
669
860
 
670
861
  Supported dataset types:
671
862
  - **HuggingFace datasets** — any HF dataset ID that returns decoded image
672
- data (e.g. `cifar10`, `food101`). Gated datasets require `--hf_token` or
673
- `HF_TOKEN`.
863
+ data (e.g. `cifar10`, `food101`). Gated datasets require the `HF_TOKEN`
864
+ environment variable.
674
865
  - **Local image directories** — pass a path to a folder of images
675
866
  (ImageFolder format).
676
867
 
@@ -692,6 +883,167 @@ Labels are automatically remapped to a shared global label space across all
692
883
  datasets, so mixing datasets with overlapping but differently-named classes
693
884
  works transparently.
694
885
 
886
+ #### Streaming (Iterable) Datasets
887
+
888
+ A dataset does not have to fit on disk or in RAM. When a source can only be
889
+ *iterated* (no index, no length), the pipeline streams it: items are read once,
890
+ converted on the fly, and never materialised into an index space.
891
+
892
+ Streaming sources can be used in two places:
893
+
894
+ - **`--dataset` with `--pipeline text`** — pass `streaming=True` through
895
+ `--dataset_arg` to have the HuggingFace loader return a streaming source
896
+ instead of a materialised split:
897
+
898
+ <!-- genml-cli-test-exclude: illustrative streaming run against a user corpus; covered by tests/test_streaming_text_pipeline.py -->
899
+ ```bash
900
+ genml-kit-train --pipeline text --model gpt2 \
901
+ --dataset my-org/huge-corpus \
902
+ --text_column text \
903
+ --dataset_arg streaming=True \
904
+ --epochs 1
905
+ ```
906
+
907
+ - **`--datasets` (image pre-training ensemble)** — use a `webdataset:` spec
908
+ (see below). If every dataset in an ensemble streams, the ensemble streams
909
+ too, in declaration order.
910
+
911
+ - **`--dataset` with `--pipeline images` (classification)** — pass
912
+ `streaming=True` through `--dataset_arg` to stream an image *classification*
913
+ source instead of materialising it. The train/val split becomes a positional
914
+ one (`IterableDatasetFilter`), and the label column must be a typed
915
+ `ClassLabel` so the class vocabulary is read from the dataset's `features`
916
+ without scanning the corpus. Class weighting (`--sampler weighted` /
917
+ `balanced`, inverse-frequency weights) needs the whole label column, so it is
918
+ refused for streams with a clear error.
919
+
920
+ **What works, and what does not.** A stream has no index space, so anything
921
+ that needs one is refused with a clear error rather than silently misbehaving:
922
+
923
+ | Feature | Random-access | Streaming |
924
+ | --- | --- | --- |
925
+ | Plain batching | yes | yes |
926
+ | `--length_bucket` | yes | yes (over a bounded buffer) |
927
+ | `--packing wrapped` / `bfd` | yes | yes (over a bounded buffer) |
928
+ | `--val_split` | index slice | positional split (`IterableDatasetFilter`) |
929
+ | `--sampler weighted` / `balanced` | yes | not available (needs labels across the whole corpus) |
930
+ | `--max_items` | yes | yes (caps the stream) |
931
+ | Classification (`--dataset`) | yes | yes, with a typed `ClassLabel` label column |
932
+
933
+ Because a stream cannot be shuffled by index, `shuffle` is disabled for it and
934
+ its order is the source order. Under DDP, `--shuffle_buffer_k` supplies
935
+ batch-level shuffling instead — see below.
936
+
937
+ ### `--shuffle_buffer_k` (Streaming Shuffle)
938
+
939
+ A streaming source cannot be shuffled by sampler: there is no index space to
940
+ permute. Instead the *batch* stream is shuffled, with a bounded round-robin
941
+ buffer.
942
+
943
+ `--shuffle_buffer_k K` sets the size of that buffer in **batches per rank**, so
944
+ the buffer holds `world_size * K` batches. It only has an effect on a streaming
945
+ loader; random-access datasets shuffle by sampler and ignore it.
946
+
947
+ - **`K = 10`** (default) — a reasonable trade-off between shuffle quality and
948
+ the number of batches buffered in memory.
949
+ - **`K = 1`** — the minimal round-robin: each rank takes one batch per round,
950
+ with no permuting.
951
+ - **`K = 0`** — round-robin without permuting.
952
+
953
+ Memory cost scales with `K`: each buffered batch is a full batch of decoded
954
+ samples, so raise `K` only as far as your RAM allows.
955
+
956
+ <!-- genml-cli-test-exclude: illustrative DDP invocation over local shards; covered by tests/test_iterable_ddp.py -->
957
+ ```bash
958
+ torchrun --nproc_per_node=4 -m genml_kit.cli.train --model convvit \
959
+ --datasets "webdataset:/data/train-{0000..0999}.tar" \
960
+ --shuffle_buffer_k 20 \
961
+ --batch_size 64
962
+ ```
963
+
964
+ ### WebDataset Tar Shards (`webdataset:`)
965
+
966
+ For corpora already stored as webdataset tar shards, pass a spec of the form
967
+ `webdataset:<shard-list>` inside `--datasets`. The shard list is passed to
968
+ webdataset verbatim, so its brace expansion applies:
969
+
970
+ <!-- genml-cli-test-exclude: illustrative webdataset pre-training run; covered by tests/test_webdataset_pipeline.py -->
971
+ ```bash
972
+ genml-kit-train --pipeline images --model convvit --method simmim \
973
+ --datasets "webdataset:/data/train-{0000..0999}.tar" \
974
+ --batch_size 64 \
975
+ --epochs 100
976
+ ```
977
+
978
+ A webdataset "sample" is a group of members named `<key>.<extension>` inside the
979
+ tar (`s0001.png`, `s0001.cls`, `s0001.txt`). Members are decoded by extension:
980
+ `png`/`jpg` become images, `txt` a string, `cls` an integer, `json` a dict, `npy`
981
+ an ndarray, `pth` a tensor; anything else stays raw bytes.
982
+
983
+ The image member is detected from the first sample (its name is the extension,
984
+ e.g. `png`). If a shard carries more than one image member, name it explicitly:
985
+
986
+ ```bash
987
+ --datasets "webdataset:/data/train-{0000..0999}.tar" --image_column png
988
+ ```
989
+
990
+ The `webdataset` package ships with the core install (it is a hard dependency,
991
+ like `datasets` and `transformers`), so no extra is needed.
992
+
993
+ A webdataset member is keyed by extension, so these sources are unlabeled by
994
+ default (there is no dataset-level class vocabulary). Use a method that does not
995
+ need labels (`simmim`, `dino`, `byol`, `ijepa`), or select the label member
996
+ explicitly in code with `IterableFieldSelectorDataset`.
997
+
998
+ ### HuggingFace Datasets
999
+
1000
+ Any HF dataset ID whose rows carry decoded image data works directly as
1001
+ `--dataset` (classification) or inside `--datasets` (ensemble). Public
1002
+ datasets need no credentials:
1003
+
1004
+ <!-- genml-cli-test
1005
+ name: hf-dataset-public
1006
+ network: true
1007
+ service: hf
1008
+ extra: --epochs 1 --max_items 64 --batch_size 8
1009
+ -->
1010
+ ```bash
1011
+ genml-kit-train --model convvit \
1012
+ --dataset uoft-cs/cifar10 \
1013
+ --image_column img \
1014
+ --label_column label \
1015
+ --image_size 224 \
1016
+ --epochs 5
1017
+ ```
1018
+
1019
+ `--max_items` caps each split (applied *after* the train/val split), which
1020
+ keeps these examples fast (the full `cifar10` train split is 50k images).
1021
+
1022
+ ### Gated / Private Datasets
1023
+
1024
+ Gated datasets require a HuggingFace token whose owner has been granted access.
1025
+ Set the `HF_TOKEN` environment variable:
1026
+
1027
+ <!-- genml-cli-test
1028
+ name: hf-dataset-gated
1029
+ network: true
1030
+ service: hf
1031
+ gated: true
1032
+ extra: --epochs 1 --batch_size 2
1033
+ -->
1034
+ ```bash
1035
+ genml-kit-train --model convvit \
1036
+ --dataset sirtolkien/autotrain-data-test \
1037
+ --image_column image \
1038
+ --label_column target \
1039
+ --image_size 224 \
1040
+ --epochs 5
1041
+ ```
1042
+
1043
+ Because the gated dataset cannot be fetched without a grant, the README test
1044
+ harness remaps `--dataset` to a synthesized local fixture and runs the
1045
+ example offline.
1046
+
695
1047
  ### Preparing Datasets
696
1048
 
697
1049
  Some datasets store images inside zip archives or need custom preprocessing
@@ -704,10 +1056,18 @@ python my_prepare_script.py --output_dir ./prepared_images
704
1056
 
705
1057
  Then use the extracted directory as a local dataset:
706
1058
 
1059
+ <!-- genml-cli-test
1060
+ name: pretrain-prepared-images
1061
+ network: true
1062
+ override:
1063
+ - --datasets {tmp}/img
1064
+ extra: --epochs 1
1065
+ -->
707
1066
  ```bash
708
1067
  genml-kit-train --pipeline images --method simmim \
1068
+ --model convvit \
709
1069
  --datasets ./prepared_images ./other-images \
710
- --image_size 448 --batch_size 32 ...
1070
+ --image_size 448 --batch_size 32 --epochs 20
711
1071
  ```
712
1072
 
713
1073
  For a concrete worked example of such a preparation script, see
@@ -823,6 +1183,37 @@ The confusion matrix uses rows for true classes and columns for predicted
823
1183
  classes. Diagonal entries are correct predictions. The compact confusion
824
1184
  summary reports each class's recall and its largest confusion destinations.
825
1185
 
1186
+ Two ranking/calibration metrics are also reported when the validation
1187
+ predictions carry logits (the default for model-based methods):
1188
+
1189
+ - **AUROC (macro, one-vs-rest):** how well the model's scores rank a random
1190
+ positive of a class above a random negative, averaged over classes. It is
1191
+ threshold-free and therefore the right summary for imbalanced clinical
1192
+ workloads where the operating point is chosen later.
1193
+ - **ECE (expected calibration error):** the average absolute gap between the
1194
+ model's confidence and its observed accuracy over 15 confidence bins. A
1195
+ model can be accurate and still overconfident; ECE quantifies that gap.
1196
+
1197
+ Overconfidence is fixed post hoc with `--fit_temperature`: after training, a
1198
+ single scalar $`T`$ is fitted on the validation set (minimising negative
1199
+ log-likelihood of `logits / T`), written to `temperature.json` next to the
1200
+ checkpoint, and applied at inference via `genml-kit-infer --temperature T`:
1201
+
1202
+ <!-- genml-cli-test
1203
+ name: fit-temperature
1204
+ override:
1205
+ - --dataset {tmp}/img
1206
+ - --label_column label
1207
+ extra: --epochs 1 --max_items 16 --image_size 32
1208
+ -->
1209
+ ```bash
1210
+ genml-kit-train --model convvit --method classification --fit_temperature \
1211
+ --dataset my-org/my-labeled-images --label_column category \
1212
+ --epochs 5 --batch_size 32
1213
+ genml-kit-infer --model convvit --checkpoint genml_kit_best.pt \
1214
+ --temperature 1.31 --top_k 3 path/to/lesion.jpg
1215
+ ```
1216
+
826
1217
 
827
1218
  When TTA is enabled, validation loss is computed from the original image only,
828
1219
  while Top-1 accuracy, balanced accuracy, macro F1, weighted F1, and per-class
@@ -836,6 +1227,14 @@ the same sampling and preprocessing scheme.
836
1227
 
837
1228
  ### Basic Fine-Tuning
838
1229
 
1230
+ <!-- genml-cli-test
1231
+ name: fine-tune-basic
1232
+ network: true
1233
+ override:
1234
+ - --dataset {tmp}/img
1235
+ - --label_column label
1236
+ extra: --epochs 1
1237
+ -->
839
1238
  ```bash
840
1239
  genml-kit-train --model google/vit-base-patch16-224 \
841
1240
  --dataset my-org/my-labeled-images \
@@ -843,7 +1242,7 @@ genml-kit-train --model google/vit-base-patch16-224 \
843
1242
  --epochs 5 \
844
1243
  --batch_size 32 \
845
1244
  --lr 3e-5 \
846
- --image_size 448
1245
+ --image_size 224
847
1246
  ```
848
1247
 
849
1248
  ### With A Custom Classifier Head
@@ -851,11 +1250,20 @@ genml-kit-train --model google/vit-base-patch16-224 \
851
1250
  Replace the default linear head with a custom MLP or attention-based
852
1251
  classifier:
853
1252
 
1253
+ <!-- genml-cli-test
1254
+ name: fine-tune-custom-head
1255
+ network: true
1256
+ override:
1257
+ - --dataset {tmp}/img
1258
+ - --label_column label
1259
+ extra: --epochs 1
1260
+ -->
854
1261
  ```bash
855
1262
  # Freeze backbone, train only the custom head
856
1263
  genml-kit-train --model cls_model_wrapper:google/vit-base-patch16-224 \
857
1264
  --dataset my-org/my-labeled-images \
858
1265
  --label_column category \
1266
+ --image_size 224 \
859
1267
  --model_arg classifier=mlp:hidden=512,dropout=0.3 \
860
1268
  --freeze ".*\.(head|pool)"
861
1269
  ```
@@ -865,11 +1273,20 @@ genml-kit-train --model cls_model_wrapper:google/vit-base-patch16-224 \
865
1273
  Freeze the entire backbone and train only small low-rank adapter matrices.
866
1274
  Reduces trainable parameters by ~97% while often matching full fine-tuning:
867
1275
 
1276
+ <!-- genml-cli-test
1277
+ name: fine-tune-lora
1278
+ network: true
1279
+ extras: lora
1280
+ override:
1281
+ - --dataset {tmp}/img
1282
+ - --label_column label
1283
+ extra: --epochs 1
1284
+ -->
868
1285
  ```bash
869
1286
  genml-kit-train \
870
1287
  --model cls_model_wrapper:facebook/dinov2-with-registers-large \
1288
+ --model_arg classifier=mlp:hidden=512 \
871
1289
  --lora --lora_r 16 --lora_alpha 32 \
872
- --lora_target_modules "query,key,value" \
873
1290
  --freeze "classifier\.(head|pool|encoder)" \
874
1291
  --lr 3e-5 \
875
1292
  --dataset my-org/my-labeled-images \
@@ -881,6 +1298,14 @@ genml-kit-train \
881
1298
 
882
1299
  Load encoder weights from a pre-training run (SimMIM, I-JEPA, or SupCon):
883
1300
 
1301
+ <!-- genml-cli-test
1302
+ name: fine-tune-from-checkpoint
1303
+ override:
1304
+ - --dataset {tmp}/img
1305
+ - --label_column label
1306
+ - -.--source_checkpoint
1307
+ extra: --epochs 1
1308
+ -->
884
1309
  ```bash
885
1310
  genml-kit-train --model convvit \
886
1311
  --dataset my-org/my-labeled-images \
@@ -901,7 +1326,7 @@ schedule is (introduced by Howard & Ruder, [Universal Language Model Fine-tuning
901
1326
 
902
1327
  $$
903
1328
  \large
904
- \text{lr}(\text{layer}) = \text{lr}_{\text{base}} \cdot d^{\,L - \text{layer}}
1329
+ \text{lr}(\text{layer}) = \text{lr}_{\text{base}} \cdot d^{\\,L - \text{layer}}
905
1330
  $$
906
1331
 
907
1332
  where:
@@ -936,6 +1361,88 @@ the model settle into a minimum instead of bouncing around it. Late-training
936
1361
  low learning rates are also what make the EMA teacher of DINO/BYOL
937
1362
  effective: the teacher tracks a student that changes gently.
938
1363
 
1364
+ ### Warmup
1365
+
1366
+ Transformer fine-tuning is unstable at the step-off: the randomly initialised
1367
+ head meets a pre-trained backbone with full force. The three warmup
1368
+ schedulers ramp the learning rate up linearly from zero over `--warmup_epochs`
1369
+ epochs (an integer or fraction) before handing over to the chosen decay
1370
+ shape:
1371
+
1372
+ <!-- genml-cli-test
1373
+ name: warmup-scheduler
1374
+ override:
1375
+ - --dataset {tmp}/img
1376
+ - --label_column label
1377
+ extra: --epochs 2 --max_items 16
1378
+ -->
1379
+ ```bash
1380
+ genml-kit-train --model convvit --method classification \
1381
+ --dataset my-org/my-labeled-images --label_column category \
1382
+ --scheduler warmup_cosine --warmup_epochs 2.5 \
1383
+ --epochs 20 --batch_size 32
1384
+ ```
1385
+
1386
+ `--warmup_epochs 0` (the default) disables the ramp and every `warmup_*`
1387
+ name then behaves exactly like its plain shape. Available shapes:
1388
+
1389
+ | Name | Shape after warmup |
1390
+ |---|---|
1391
+ | `warmup_cosine` | half-cosine decay to `eta_min` (0), as above |
1392
+ | `warmup_linear` | linear decay to 0 over `--epochs` |
1393
+ | `warmup_rsqrt` | inverse square-root decay |
1394
+
1395
+ A `--sched_arg warmup_epochs=...` override wins over the generic flag, and
1396
+ `--sched_arg T_max=...` overrides the cosine total (the epoch of the first
1397
+ minimum). Plain torch names (`CosineAnnealingLR`, ...) ignore the warmup
1398
+ flag entirely, so existing invocations are unchanged.
1399
+
1400
+ ### Weights EMA
1401
+
1402
+ Alongside the *loss-coupled* EMA teachers (DINO/BYOL/I-JEPA, and
1403
+ `--teacher_kind ema` in distillation), the trainer can maintain an
1404
+ *eval-time* shadow of the trained weights: after every optimizer step the
1405
+ shadow blends toward the live weights (`shadow <- decay * shadow +
1406
+ (1 - decay) * model`), damping the step-to-step noise of the optimisation
1407
+ trajectory. Validation and the best checkpoint then run on the shadow
1408
+ instead of the raw weights -- the classic timm / MMDetection weights-EMA,
1409
+ which typically buys a fraction of a point on noisy runs.
1410
+
1411
+ The feature is **opt-in per method**: only the method knows whether a
1412
+ shadow of "the model" is meaningful for its topology (DPO owns a frozen
1413
+ reference tower, distillation owns teacher towers -- neither is EMA
1414
+ material). Methods opt in by inheriting `WeightsEMAMixin`
1415
+ (`ClassificationMethod` does); methods without the mixin never see the
1416
+ feature, and setting `--ema_decay` on one logs a warning instead of
1417
+ silently doing nothing. The RL trainer shares the loop and simply never
1418
+ wires a shadow. The `--ema_*` flags are declared by the mixin's
1419
+ `add_args`, which adopting methods chain into their own `--help` group,
1420
+ so the flags render inside "classification method".
1421
+
1422
+ <!-- genml-cli-test
1423
+ name: weights-ema
1424
+ override:
1425
+ - --model convvit
1426
+ - --dataset {tmp}/img
1427
+ - --label_column label
1428
+ extra: --epochs 1 --max_items 16
1429
+ -->
1430
+ ```bash
1431
+ genml-kit-train --model google/vit-base-patch16-224 --method classification \
1432
+ --dataset my-org/my-labeled-images --label_column category \
1433
+ --ema_decay 0.9995 --ema_eval \
1434
+ --epochs 5 --batch_size 32
1435
+ ```
1436
+
1437
+ `--ema_decay 0` (the default) keeps the run exactly as before. With
1438
+ `--ema_eval`, validation and `best.pt` carry the EMA weights while
1439
+ `latest.pt` keeps the raw trajectory plus the shadow in `ema_state`, so an
1440
+ interrupted run resumes exactly. `--ema_final_decay` ramps the effective
1441
+ decay linearly over the run (the DINO-style momentum schedule) and
1442
+ `--ema_warmup_steps` ramps it up from zero; when both are absent the decay
1443
+ is constant. `run.json` records the decay, so provenance of "which weights
1444
+ the best checkpoint holds" is unambiguous.
1445
+
939
1446
  ### Mixup, Label Smoothing, And Imbalance
940
1447
 
941
1448
  Mixup forms a virtual example from two training examples.
@@ -989,13 +1496,13 @@ The loss for one example is
989
1496
 
990
1497
  $$
991
1498
  \large
992
- \mathcal{L} = \left(1 - p_t\right)^{\gamma} \sum_{c=1}^{C} w_c\, q_c
993
- \left(-\log p_c\right), \qquad p_t = \sum_{c=1}^{C} q_c\, p_c
1499
+ \mathcal{L} = \left(1 - p_t\right)^{\gamma} \sum_{c=1}^{C} w_c\\, q_c
1500
+ \left(-\log p_c\right), \qquad p_t = \sum_{c=1}^{C} q_c\\, p_c
994
1501
  $$
995
1502
 
996
1503
  Two details matter for the reader combining these flags:
997
1504
 
998
- - **Unified $`p_t`.** The modulating factor uses the *expected* probability
1505
+ - **Unified $`p_t`$.** The modulating factor uses the *expected* probability
999
1506
  under the target distribution, $`p_t = \sum_c q_c p_c`$, rather than the
1000
1507
  probability of a single class. For a one-hot target $`q`$ this reduces
1001
1508
  exactly to the basic form above ($`p_t`$ becomes the correct-class
@@ -1011,6 +1518,91 @@ Two details matter for the reader combining these flags:
1011
1518
  Use these tools deliberately; combining every regularizer is not
1012
1519
  automatically better.
1013
1520
 
1521
+ ### Data Augmentation
1522
+
1523
+ By default the `classification` method augments the training set with the
1524
+ chain built by `build_transforms` in
1525
+ `genml_kit/training/train_compat.py`:
1526
+
1527
+ ```python
1528
+ import torch
1529
+ from torchvision.transforms import v2
1530
+
1531
+ train_augmentations = v2.Compose([
1532
+ # Randomly crop 20-100% of the image, then resize to image_size.
1533
+ v2.RandomResizedCrop(size=(image_size, image_size),
1534
+ scale=(0.2, 1.0),
1535
+ antialias=True),
1536
+ # Mirror left/right, applied half the time.
1537
+ v2.RandomHorizontalFlip(p=0.5),
1538
+ # Mirror top/bottom, applied half the time.
1539
+ v2.RandomVerticalFlip(p=0.5),
1540
+ # Rotate by an arbitrary angle.
1541
+ v2.RandomRotation(degrees=360),
1542
+ # Photometric jitter on brightness, contrast, saturation and hue.
1543
+ v2.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.15, hue=0.05),
1544
+ # Local elastic warping.
1545
+ v2.ElasticTransform(alpha=50.0, sigma=5.0),
1546
+ # Fixed tail, always appended automatically:
1547
+ # PIL image -> tensor.
1548
+ v2.ToImage(),
1549
+ # uint8 -> float in [0, 1].
1550
+ v2.ToDtype(torch.float32, scale=True),
1551
+ # Model-specific normalization.
1552
+ v2.Normalize(mean=processor.image_mean, std=processor.image_std),
1553
+ ])
1554
+ ```
1555
+
1556
+ The validation chain is never augmented: it is always `Resize` →
1557
+ `CenterCrop` → `Normalize`.
1558
+
1559
+ #### Custom Training Augmentation
1560
+
1561
+ To override the training chain, pass `--train_augmentation_script <path>.py`
1562
+ (a local path or an HTTP/HTTPS URL). The script is imported lazily, so a
1563
+ plain run never loads it. It must define
1564
+
1565
+ ```python
1566
+ def create_train_transform(image_size: int):
1567
+ ...
1568
+ ```
1569
+
1570
+ for example:
1571
+
1572
+ ```python
1573
+ # my_aug.py
1574
+ from torchvision.transforms import v2
1575
+
1576
+ def create_train_transform(image_size):
1577
+ return [
1578
+ v2.RandomResizedCrop(image_size, scale=(0.4, 1.0), antialias=True),
1579
+ v2.RandomHorizontalFlip(p=0.5),
1580
+ v2.ColorJitter(brightness=0.3, contrast=0.3),
1581
+ ]
1582
+ ```
1583
+
1584
+ The contract is:
1585
+
1586
+ - The function is called once as `create_train_transform(image_size)` and
1587
+ **must return a `list`** of transforms. Any other return type raises a
1588
+ `TypeError` (`create_train_transform() must return a list of transforms,
1589
+ got <type>`).
1590
+ - The returned list is composed as `v2.Compose(user_transforms + tail)`,
1591
+ where the fixed tail above (`ToImage` → `ToDtype` → `Normalize`) is
1592
+ **always appended automatically**. Your transforms therefore run on the
1593
+ **raw PIL image**, and you must **not** add `ToImage`, `ToDtype`, or
1594
+ `Normalize` yourself.
1595
+ - This differs from the segmentation method's `create_seg_transform`, which
1596
+ owns the whole image + mask chain *including* normalization. For
1597
+ classification, normalization stays under the processor's control.
1598
+ - Only the **train** chain is replaced; validation is unaffected. Use
1599
+ `--tta` for test-time augmentation (see
1600
+ [XGBoost And Test-Time Augmentation](#xgboost-and-test-time-augmentation)).
1601
+
1602
+ Modules placed next to the script (or importable via `import` from the same
1603
+ directory) resolve by plain name, so a custom transform can live in a
1604
+ sibling file.
1605
+
1014
1606
  ### Hyperparameter Guidance
1015
1607
 
1016
1608
  | Scenario | `--lr` | `--epochs` | `--batch_size` | Notes |
@@ -1047,8 +1639,17 @@ Two levels of determinism:
1047
1639
  Passing an explicit `--seed` therefore buys full determinism at a
1048
1640
  throughput cost (typically 10–20%); omit it for maximum speed:
1049
1641
 
1642
+ <!-- genml-cli-test
1643
+ name: reproducibility-seed
1644
+ override:
1645
+ - --dataset {tmp}/img
1646
+ - --label_column label
1647
+ extra: --epochs 1 --batch_size 4 --max_items 16
1648
+ -->
1050
1649
  ```bash
1051
- genml-kit-train --model convvit --seed 42 ...
1650
+ genml-kit-train --model convvit --seed 42 \
1651
+ --dataset my-org/my-labeled-images --label_column category \
1652
+ --epochs 5 --batch_size 32
1052
1653
  ```
1053
1654
 
1054
1655
  The train/val split and the XGBoost stage follow the user's seed: with
@@ -1068,6 +1669,67 @@ Checkpointing is atomic: `_latest.pt` is written to a temporary file
1068
1669
  and renamed into place, so an interrupted run never leaves a truncated
1069
1670
  resume point.
1070
1671
 
1672
+ ### Run Manifest
1673
+
1674
+ Every training run writes `<log_dir>/run.json` — a reproducibility
1675
+ snapshot of *how* the run was launched: the fully resolved argparse
1676
+ namespace (defaults filled in), the git SHA of the checkout, the
1677
+ `torch` / `transformers` versions, hostname, device, resolved seeds, and
1678
+ timestamp. Together with `metrics.jsonl` this makes any result traceable
1679
+ back to the exact code and invocation that produced it:
1680
+
1681
+ <!-- genml-cli-test-exclude: illustrative ellipsis, no dataset; not a runnable command -->
1682
+ ```bash
1683
+ genml-kit-train --model convvit --method classification ...
1684
+ cat <checkpoint_dir>/logs/run.json
1685
+ ```
1686
+
1687
+ The same facts are emitted to the metric sinks as `manifest/*` and
1688
+ `args/<name>` run-facts, so they land in TensorBoard and any configured
1689
+ sink alongside the metrics. `--dry_run` logs the manifest as JSON and
1690
+ exits before the dataset is touched — a fast way to inspect what an
1691
+ invocation would actually do:
1692
+
1693
+ <!-- genml-cli-test
1694
+ name: run-manifest-dry-run
1695
+ -->
1696
+ ```bash
1697
+ genml-kit-train --model convvit --method classification --dry_run
1698
+ ```
1699
+
1700
+ ### Metric Sinks
1701
+
1702
+ `--metrics_sinks` selects where training metrics go, as a comma-separated
1703
+ fan-out: `tb` (TensorBoard event files), `jsonl` (one JSON record per line in
1704
+ `<log_dir>/metrics.jsonl` — the file the search subsystem reads), and `wandb`
1705
+ (a live Weights & Biases run). The default `tb,jsonl` is unchanged; sinks
1706
+ are independent, so any combination works:
1707
+
1708
+ <!-- genml-cli-test
1709
+ name: metric-sinks
1710
+ extras: wandb
1711
+ override:
1712
+ - --dataset {tmp}/img
1713
+ - --label_column label
1714
+ extra: --epochs 1 --max_items 16 --wandb_mode disabled
1715
+ -->
1716
+ ```bash
1717
+ genml-kit-train --model convvit --method classification \
1718
+ --dataset my-org/my-labeled-images --label_column category \
1719
+ --metrics_sinks tb,jsonl,wandb --wandb_project my-team \
1720
+ --epochs 5 --batch_size 32
1721
+ ```
1722
+
1723
+ The `wandb` sink requires the optional extra (`pip install
1724
+ "genml_kit[wandb]"`); the run name defaults to `--run_name` when given, the
1725
+ run finishes on exit, and the fully resolved args are uploaded as the run
1726
+ config. `--wandb_mode` selects how the run is recorded: `online` (default,
1727
+ syncs to the W&B service), `offline` (a local `.wandb` transaction log,
1728
+ synced later with `wandb sync` — no network or credentials needed), or
1729
+ `disabled` (wandb's no-op stub). New sinks implement the same tiny
1730
+ duck-typed protocol (`scalar` / `image` / `video` / `run_fact` / `close`) in
1731
+ `genml_kit/training/metrics_sinks.py`.
1732
+
1071
1733
  ### Fine-Tuning CLI Reference
1072
1734
 
1073
1735
  | Argument | Default | Description |
@@ -1103,7 +1765,12 @@ resume point.
1103
1765
  | `--num_workers` | `2` | DataLoader worker processes. |
1104
1766
  | `--log_level` | `INFO` | Logging level. |
1105
1767
  | `--log_targets` | `STDERR` | Comma-separated log destinations. `STDERR` logs to standard error; any other entry is a log file path (appended). Example: `STDERR,/tmp/train.log` logs to both. |
1106
- | `--log_dir` | `None` | TensorBoard log directory (default: `<checkpoint_dir>/logs`). Requires the `tensorboard` package (installed by the `dev` extra). |
1768
+ | `--log_dir` | `None` | TensorBoard log directory (default: `<checkpoint_dir>/logs`). Requires the `tensorboard` package. |
1769
+ | `--metrics_sinks` | `tb,jsonl` | Comma-separated metric sinks: `tb` (TensorBoard), `jsonl` (`<log_dir>/metrics.jsonl`), `wandb` (Weights & Biases, requires `pip install "genml_kit[wandb]"`), `none`. |
1770
+ | `--wandb_project` | `None` | W&B project name when `wandb` is in `--metrics_sinks` (default: `genml-kit`). |
1771
+ | `--wandb_mode` | `online` | W&B run mode: `online` (sync to the service), `offline` (local `.wandb` log, sync later with `wandb sync`), `disabled` (stub, no network/files). |
1772
+ | `--run_name` | `None` | W&B run name when `wandb` is in `--metrics_sinks`. |
1773
+ | `--fit_temperature` | `False` | After training, fit a temperature-scaling T on the validation set of the best checkpoint and write `temperature.json` next to it. Classification methods only. |
1107
1774
  | `--cache_dir` | `None` | HuggingFace cache directory. |
1108
1775
  | `--remote_checkpoint` | `None` | Remote URI for checkpoint sync (`gs://BUCKET/PREFIX`, `r2://BUCKET/PREFIX`, or `s3://BUCKET/PREFIX`). |
1109
1776
  | `--source_checkpoint` | `None` | Path to source checkpoint to absorb parameters from. |
@@ -1113,11 +1780,12 @@ resume point.
1113
1780
  | `--lora_r` | `8` | LoRA rank. |
1114
1781
  | `--lora_alpha` | `16` | LoRA alpha (scaling = `alpha / r`). |
1115
1782
  | `--lora_dropout` | `0.0` | Dropout on LoRA layers. |
1116
- | `--lora_target_modules` | `None` | Comma-separated module names for LoRA (e.g. `"query,key,value"`). |
1783
+ | `--lora_target_modules` | `None` | Comma-separated module names for LoRA (e.g. `"q_proj,k_proj,v_proj"`). When omitted, targets are resolved from the model (see LoRA section). |
1117
1784
  | `--optimizer` | `AdamW` | `torch.optim` optimizer class name or `.py` script path. |
1118
1785
  | `--opt_arg` | `{}` | Extra optimizer kwargs (repeatable). |
1119
- | `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path. |
1786
+ | `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path, or a built-in warmup shape: `warmup_cosine`, `warmup_linear`, `warmup_rsqrt`. |
1120
1787
  | `--sched_arg` | `{}` | Extra scheduler kwargs (repeatable). |
1788
+ | `--warmup_epochs` | `0` | Linear LR ramp length in epochs (integer or fraction) for the `warmup_*` schedulers; 0 disables the ramp. See [Warmup](#warmup). |
1121
1789
  | `--state_save` | `opt,sched,amp` | States to save: `opt`, `sched`, `amp`, `none`. |
1122
1790
  | `--state_load` | `opt,sched,amp` | States to restore on resume. |
1123
1791
  | `--xgboost_model` | `None` | Output path for XGBoost model (trains after PyTorch). |
@@ -1131,6 +1799,15 @@ resume point.
1131
1799
  Training automatically resumes from an existing `_latest.pt` or `_best.pt`
1132
1800
  checkpoint if one exists at the `--checkpoint` path.
1133
1801
 
1802
+ Alongside the epoch-end and exit saves, `--save_every` writes `_latest.pt`
1803
+ every N optimizer steps (grad-accum flush boundary) so a hard kill between
1804
+ epochs does not lose the whole epoch. When such a checkpoint carries the epoch
1805
+ in progress, resume **continues** it rather than replaying it: supervised runs
1806
+ fast-forward to the recorded step, and PPO skips re-collecting the rollout and
1807
+ re-enters the optimize sweep (a resume taken mid-collection re-runs the epoch,
1808
+ since the environment is not persisted). Off-policy RL (DQN/SAC) restarts the
1809
+ epoch: the replay buffer is not checkpointed.
1810
+
1134
1811
  ### Remote Checkpoint Sync (GCS / R2 / S3)
1135
1812
 
1136
1813
  `--remote_checkpoint` uploads each saved checkpoint to cloud storage.
@@ -1206,6 +1883,15 @@ Only $`A`$, $`B`$ (and the classifier head) receive gradients.
1206
1883
 
1207
1884
  LoRA can be combined with a custom classifier:
1208
1885
 
1886
+ <!-- genml-cli-test
1887
+ name: lora-custom-classifier
1888
+ network: true
1889
+ extras: lora
1890
+ override:
1891
+ - --dataset {tmp}/img
1892
+ - --label_column label
1893
+ extra: --epochs 1
1894
+ -->
1209
1895
  ```bash
1210
1896
  genml-kit-train \
1211
1897
  --model cls_model_wrapper:facebook/dinov2-with-registers-large \
@@ -1213,13 +1899,23 @@ genml-kit-train \
1213
1899
  --lora --lora_r 16 --lora_alpha 32 \
1214
1900
  --freeze 'classifier\.(head|pool|encoder)' \
1215
1901
  --lr_group 'backbone.*=1e-5' 'classifier.*=3e-4' \
1216
- ...
1902
+ --dataset my-org/my-labeled-images \
1903
+ --label_column category \
1904
+ --epochs 20
1217
1905
  ```
1218
1906
 
1219
1907
  ### Cross-Dataset Resume
1220
1908
 
1221
1909
  Switch from one dataset to another while keeping backbone weights:
1222
1910
 
1911
+ <!-- genml-cli-test
1912
+ name: cross-dataset-resume
1913
+ network: true
1914
+ override:
1915
+ - --dataset {tmp}/img
1916
+ - --label_column label
1917
+ extra: --epochs 1
1918
+ -->
1223
1919
  ```bash
1224
1920
  genml-kit-train --model facebook/convnextv2-base-22k-224 \
1225
1921
  --dataset my-org/other-labeled-images \
@@ -1245,14 +1941,24 @@ limits the maximum batch size even on a 24 GB GPU. Gradient checkpointing
1245
1941
  trades compute for memory by discarding intermediate activations during the
1246
1942
  forward pass and recomputing them during the backward pass.
1247
1943
 
1944
+ <!-- genml-cli-test
1945
+ name: gradient-checkpointing
1946
+ network: true
1947
+ override:
1948
+ - --dataset {tmp}/img
1949
+ - --label_column label
1950
+ extra: --epochs 1
1951
+ -->
1248
1952
  ```bash
1249
1953
  genml-kit-train \
1250
1954
  --model timm:eva02_base_patch14_448.mim_in22k_ft_in22k_in1k \
1955
+ --dataset my-org/my-labeled-images \
1956
+ --label_column category \
1251
1957
  --batch_size 32 \
1252
1958
  --grad_accum_steps 2 \
1253
1959
  --grad_checkpoint \
1254
1960
  --amp_dtype bfloat16 \
1255
- ...
1961
+ --epochs 20
1256
1962
  ```
1257
1963
 
1258
1964
  **What it does:** Each transformer block stores only its input and output.
@@ -1279,10 +1985,27 @@ HuggingFace, ConvViT, and UVito — via native APIs or per-block
1279
1985
 
1280
1986
  Run inference on individual images:
1281
1987
 
1988
+ <!-- genml-cli-test
1989
+ name: inference-guide
1990
+ network: true
1991
+ override:
1992
+ - --checkpoint {tmp}/chk_infer/model_best.pt
1993
+ - --model convvit --image_size 32
1994
+ extra: --top_k 3 --image_size 32
1995
+ -->
1282
1996
  ```bash
1283
1997
  genml-kit-infer --model facebook/convnextv2-base-22k-224 \
1284
1998
  --checkpoint genml_kit_best.pt \
1285
1999
  path/to/image.jpg path/to/other_image.png
2000
+
2001
+ # Structured output and an XGBoost head on the backbone features (the .json
2002
+ # model is the one trained by `train_xgboost_on_backbone`, see
2003
+ # tests/test_xgb_pipeline.py).
2004
+ genml-kit-infer --model facebook/convnextv2-base-22k-224 \
2005
+ --checkpoint genml_kit_best.pt \
2006
+ --xgboost_model path/to/xgboost_model.json \
2007
+ --output results.json \
2008
+ path/to/image.jpg
1286
2009
  ```
1287
2010
 
1288
2011
  Output is JSON with per-class probabilities:
@@ -1303,7 +2026,7 @@ The neural classifier makes decisions through its head. The XGBoost option
1303
2026
  takes the encoder representation $`h = f_\theta(x)`$ instead and fits an
1304
2027
  ensemble of decision trees to those vectors.
1305
2028
  Reference: Chen & Guestrin, [XGBoost: A Scalable Tree Boosting System](https://arxiv.org/abs/1603.02754), KDD 2016.
1306
- A tree partitions feature space with rules such as $`h_{37} < t`$ (comparing
2029
+ A tree partitions feature space with rules such as $`h_{37} \lt t`$ (comparing
1307
2030
  the 37th feature of $`h`$ against a learned threshold $`t`$); boosting adds
1308
2031
  trees sequentially so each new tree focuses on errors left by previous trees.
1309
2032
  This can work well when the dataset is small
@@ -1334,6 +2057,7 @@ does not erase the cue that distinguishes the classes.
1334
2057
  | `--model` | (required) | HuggingFace model name or custom model. |
1335
2058
  | `--checkpoint` | (required) | Path to state dict or wrapped checkpoint. |
1336
2059
  | `--top_k` | `None` | Show top-K predictions; omit for all classes. |
2060
+ | `--temperature` | `1.0` | Divide logits by this before the softmax (1.0 = unchanged). Use the value written to `temperature.json` by `--fit_temperature`. |
1337
2061
  | `--output` | `None` | Write JSON results to file. |
1338
2062
  | `--device` | `None` | Force PyTorch device (`cuda`, `cpu`). Auto-detected if omitted. |
1339
2063
  | `--cache_dir` | `None` | HuggingFace cache directory. |
@@ -1369,12 +2093,26 @@ training run. `--source_checkpoint` imports weights from another run before
1369
2093
  the new optimizer is created. When moving a SupCon encoder into classification,
1370
2094
  use:
1371
2095
 
2096
+ <!-- genml-cli-test
2097
+ name: param-rename
2098
+ network: true
2099
+ override:
2100
+ - --dataset {tmp}/img
2101
+ - --label_column label
2102
+ - --checkpoint {tmp}/ckpt_param_rename
2103
+ - -.--source_checkpoint
2104
+ extra: --epochs 1
2105
+ -->
1372
2106
  ```bash
1373
2107
  genml-kit-train \
2108
+ --model google/vit-base-patch16-224 \
2109
+ --image_size 224 \
1374
2110
  --checkpoint /content/eva02_finetune \
1375
2111
  --source_checkpoint /content/eva02_supcon_latest.pt \
1376
2112
  --param_rename 'encoder\\.model\\.(.*);model.$1' \
1377
- ...
2113
+ --dataset my-org/my-labeled-images \
2114
+ --label_column category \
2115
+ --epochs 20
1378
2116
  ```
1379
2117
 
1380
2118
  The rename is needed because the pre-training wrapper stores backbone keys
@@ -1414,7 +2152,10 @@ The periodic `GPU:` line reports memory in MiB and SM utilization as a
1414
2152
  percentage sampled at log time (`mem=allocated/totalMiB (pct) res=…MiB
1415
2153
  util=…%`). `mem` counts live tensors, `res` is the caching allocator's
1416
2154
  pool, and `util` is one instant — not an average — so an isolated 0 is
1417
- normal. A *persistent* pattern of `util=0` at every log line, with
2155
+ normal. The `util=` field needs the optional NVML bindings
2156
+ (`pip install "genml_kit[gpu]"`); when they are absent, `util=` is simply
2157
+ omitted from the `GPU:` line — the run is never affected.
2158
+ A *persistent* pattern of `util=0` at every log line, with
1418
2159
  `mem`/`res` byte-identical between samples, means the GPU finished the
1419
2160
  last batch and is idle waiting for the next one: the run is **input
1420
2161
  bound**, not compute bound. The throughput line (`img/s=`) is the ground
@@ -1533,9 +2274,20 @@ def build_loss(alpha=1.0, **kwargs):
1533
2274
  return MyLoss(alpha=alpha)
1534
2275
  ```
1535
2276
 
2277
+ <!-- genml-cli-test
2278
+ name: custom-loss
2279
+ network: true
2280
+ override:
2281
+ - --dataset {tmp}/img
2282
+ - --label_column label
2283
+ extra: --epochs 1
2284
+ -->
1536
2285
  ```bash
1537
2286
  genml-kit-train --model google/vit-base-patch16-224 \
1538
- --loss ./my_loss.py --loss_args '{"alpha": 2.0}' ...
2287
+ --dataset my-org/my-labeled-images --label_column category \
2288
+ --image_size 224 \
2289
+ --loss ./my_loss.py --loss_args '{"alpha": 2.0}' \
2290
+ --epochs 20
1539
2291
  ```
1540
2292
 
1541
2293
  **RL losses** work the same way but return a *bundle* (dict of
@@ -1549,8 +2301,14 @@ def build_loss(**kwargs):
1549
2301
  return {"entropy": lambda ent: ent.mean() * 2.0} # override only entropy
1550
2302
  ```
1551
2303
 
2304
+ <!-- genml-cli-test
2305
+ name: rl-custom-loss
2306
+ extras: rl
2307
+ extra: --epochs 1
2308
+ -->
1552
2309
  ```bash
1553
- genml-kit-train --method ppo --loss ./my_ppo_entropy.py ...
2310
+ genml-kit-train --pipeline rl --method ppo --env_id CartPole-v1 \
2311
+ --loss ./my_ppo_entropy.py
1554
2312
  ```
1555
2313
 
1556
2314
  The built-in bundles are merged over (`ppo`: `policy`/`value`/`entropy`;
@@ -1607,8 +2365,18 @@ def create_model(num_labels, image_size=224, id2label=None, label2id=None,
1607
2365
  return Tiny(num_labels=num_labels, hidden=kwargs.get("hidden", 128))
1608
2366
  ```
1609
2367
 
2368
+ <!-- genml-cli-test
2369
+ name: custom-model
2370
+ network: true
2371
+ override:
2372
+ - --dataset {tmp}/img
2373
+ - --label_column label
2374
+ extra: --epochs 1
2375
+ -->
1610
2376
  ```bash
1611
- genml-kit-train --model ./my_model.py --loss focal ...
2377
+ genml-kit-train --model ./my_model.py --loss focal \
2378
+ --dataset my-org/my-labeled-images --label_column category \
2379
+ --epochs 20
1612
2380
  ```
1613
2381
 
1614
2382
  The script is **fully responsible for weight loading**: when
@@ -1632,11 +2400,13 @@ class Classifier(nn.Module):
1632
2400
  super().__init__()
1633
2401
  self.head = nn.Linear(hidden_size, num_labels)
1634
2402
 
1635
- def forward(self, hidden_states): # (B, N, D) tensor
2403
+ def forward(self, hidden_states):
2404
+ # (B, N, D) tensor.
1636
2405
  features = hidden_states[:, 0] # CLS token
1637
2406
  return self.head(features)
1638
2407
 
1639
- def extract_features(self, hidden_states): # (B, N, D) → (B, D)
2408
+ def extract_features(self, hidden_states):
2409
+ # (B, N, D) → (B, D).
1640
2410
  return hidden_states[:, 0]
1641
2411
  ```
1642
2412
 
@@ -1650,8 +2420,8 @@ The head is selected through `--model_arg`, not a dedicated flag:
1650
2420
  ```
1651
2421
 
1652
2422
  The `classifier` value is an inline spec: the head name (a registered
1653
- built-in — `mlp`, `cls_attention` — or a path to a `.py` file defining a
1654
- `Classifier` class), optionally followed by `:key=value,...` kwargs. Values
2423
+ built-in \u2014 `mlp`, `cls_attention` \u2014 or a path to a `.py` file defining a
2424
+ `create_classifier(num_labels, hidden_size, **kwargs)` function), optionally followed by `:key=value,...` kwargs. Values
1655
2425
  are parsed as int/float/bool, and bracketed lists such as
1656
2426
  `cls_slice=(0, 1)` work. A head is required: `cls_model_wrapper` fails fast
1657
2427
  with a clear message when `classifier` is missing. Models that embed their
@@ -1662,6 +2432,855 @@ head before loading the checkpoint weights.
1662
2432
 
1663
2433
  ---
1664
2434
 
2435
+ ## Semantic Segmentation
2436
+
2437
+ Segmentation predicts a class for **every pixel**: an image `(B, 3, H, W)`
2438
+ goes in and per-pixel logits `(B, num_classes, H, W)` come out. `genml_kit`
2439
+ models this as `encoder + decoder`: the encoder is any hierarchical timm
2440
+ backbone (ResNet / ConvNeXt / Swin / MiT) emitting a feature pyramid, and
2441
+ the decoder fuses that pyramid back up to full resolution (with skip
2442
+ connections for `unet`/`fpn`).
2443
+
2444
+ ### Quick start
2445
+
2446
+ <!-- genml-cli-test
2447
+ name: segmentation-quick-start
2448
+ network: true
2449
+ override:
2450
+ - --dataset segfolder/{tmp}/seg
2451
+ extra: --epochs 1
2452
+ -->
2453
+ ```bash
2454
+ genml-kit-train \
2455
+ --pipeline segmentation --method segmentation \
2456
+ --dataset segfolder//data/voc2012 \
2457
+ --model seg_wrapper:timm/resnet34 \
2458
+ --model_arg decoder=unet:depth=4 \
2459
+ --loss dice_ce \
2460
+ --image_size 512 --batch_size 8 --epochs 40
2461
+ ```
2462
+
2463
+ The layout expected by `segfolder/` is a paired tree:
2464
+
2465
+ ```
2466
+ /data/voc2012/images/2007_000032.jpg
2467
+ /data/voc2012/masks/2007_000032.png
2468
+ ```
2469
+
2470
+ or, for a corpus with named splits, `train/`, `val/` and `test/`
2471
+ sub-directories each containing `images/` + `masks/`. Image and mask are
2472
+ matched on the file stem (case-insensitive) and a mismatch fails fast.
2473
+
2474
+ HuggingFace datasets that carry a mask feature are used with `--mask_column`:
2475
+
2476
+ <!-- genml-cli-test
2477
+ name: segmentation-mask-column
2478
+ network: true
2479
+ override:
2480
+ - --dataset {tmp}/hf_seg
2481
+ extra: --epochs 1
2482
+ -->
2483
+ ```bash
2484
+ genml-kit-train --pipeline segmentation --method segmentation \
2485
+ --dataset scene_parse_150 --mask_column annotation \
2486
+ --model seg_wrapper:timm/convnext_tiny --model_arg decoder=fpn
2487
+ ```
2488
+
2489
+ ### Class-id compaction
2490
+
2491
+ Masks are expected to carry integer class ids, but the ids do **not** have
2492
+ to be dense. The pipeline scans every mask (train **and** val, so a class
2493
+ that only appears in validation is still mapped) and compacts whatever ids
2494
+ it finds into a dense `[0, C)` range:
2495
+
2496
+ | Source id | Compact id |
2497
+ |---|---|
2498
+ | 12 | 0 |
2499
+ | 17 | 1 |
2500
+ | 255 (= ignore) | dropped |
2501
+
2502
+ - **segfolder path:** the mapping is rebuilt on every run (a full scan)
2503
+ and remapped through `apply_mask_map` on every access.
2504
+ - **HuggingFace path:** the `{raw → compact}` map is cached to disk under
2505
+ `<cache_dir>/seg_class_maps/segclassmap__<hash>.json`, or
2506
+ `~/.cache/genml_kit/seg_class_maps/` when no `--cache_dir` is given
2507
+ (falling back to a temp dir if that is unwritable). The cache is keyed
2508
+ on the dataset spec **and** a content hash of the masks themselves, so
2509
+ editing a mask invalidates the entry even if the spec string is
2510
+ unchanged.
2511
+
2512
+ The class count reported by training (and the default for
2513
+ `--seg_num_classes`) is the **compact** count, i.e. the number of
2514
+ distinct ids found after dropping `--seg_ignore_index` (`len(mask_map)`).
2515
+ (The count falls back to `max_label + 1` over a sample of loader masks
2516
+ only when the pipeline did not build a mask map, e.g. a method wired
2517
+ without the segmentation pipeline.)
2518
+
2519
+ ### Data augmentation
2520
+
2521
+ The default transforms for the `segmentation` method (in
2522
+ `genml_kit/datasets/transforms.py`) are split into a geometric stage and
2523
+ a photometric stage:
2524
+
2525
+ - **Geometric stage** — applied to the image **and** the mask together,
2526
+ so they always stay pixel-aligned:
2527
+ - `v2.RandomResizedCrop(image_size, scale=(0.5, 2.0),
2528
+ ratio=(0.75, 4/3))`
2529
+ - `v2.RandomHorizontalFlip(p=0.5)`
2530
+ - **Photometric stage** — applied to the **image only** (masks contain
2531
+ class ids, never color):
2532
+ - `v2.RandomApply([v2.ColorJitter(brightness=0.4, contrast=0.6,
2533
+ saturation=0.8, hue=0.1)], p=0.8)`
2534
+
2535
+ Validation is deterministic: `v2.Resize(image_size, BICUBIC)` then
2536
+ `CenterCrop(image_size)`.
2537
+
2538
+ The geometric/photometric split is a correctness contract: geometric ops
2539
+ go through `tv_tensors.Mask`, so torchvision issues one random draw per
2540
+ sample and the mask gets the same crop/translation as the image; color
2541
+ ops never see the mask.
2542
+
2543
+ To take full control, pass `--seg_augmentation_script <path>.py` pointing
2544
+ at a module that defines
2545
+
2546
+ ```python
2547
+ def create_seg_transform(image_size: int, *, is_train: bool):
2548
+ ...
2549
+ ```
2550
+
2551
+ When set, the script **owns the whole paired transform** (image + mask),
2552
+ including normalization (the model processor's `image_mean`/`image_std`
2553
+ are *not* applied automatically in this mode), and is loaded lazily so a
2554
+ plain run never imports it.
2555
+
2556
+ ### Encoder + decoder
2557
+
2558
+ `--model seg_wrapper:<backbone>` selects the encoder; the decoder is chosen
2559
+ with `--model_arg decoder=<spec>`. Built-in decoders:
2560
+
2561
+ | Decoder | Skip connections | Notes |
2562
+ |---|---|---|
2563
+ | `unet` | yes (all levels) | Classic U-Net top-down fusion; `depth=N` selects the deepest N encoder levels |
2564
+ | `fpn` | yes (additive) | Feature-pyramid fusion into one high-resolution map |
2565
+ | `linear` | no | 1×1 conv + bilinear upsample; good baseline, ViT-friendly |
2566
+
2567
+ The decoder spec accepts `:key=value` kwargs (`decoder=unet:depth=4,decoder_channels=256`)
2568
+ and, like classifier heads, a path to a `.py` file defining
2569
+ `create_decoder(in_channels, num_classes, **kwargs)` where `in_channels` is
2570
+ the probed per-level encoder channel list.
2571
+
2572
+ ### Training from scratch vs. fine-tuning
2573
+
2574
+ The decoder is always randomly initialized and trained from scratch; the
2575
+ encoder is what you initialise. Options, in increasing order of GPU cost:
2576
+
2577
+ - **Pretrained encoder (recommended):** `seg_wrapper` loads timm ImageNet
2578
+ weights by default (`pretrained=False` disables this).
2579
+ - **Reuse a self-supervised checkpoint:** pre-train with SimMIM / I-JEPA /
2580
+ DINO, then load the encoder with `--source_checkpoint <ckpt>` (see
2581
+ [From A Pre-Trained Checkpoint](#from-a-pre-trained-checkpoint)).
2582
+ - **CNN encoder with a frozen backbone:** `--freeze 'decoder.*'` keeps the
2583
+ decoder trainable and freezes everything else.
2584
+ - **Transformer encoder with LoRA:** `--lora` (PEFT discovers the
2585
+ `q_proj`/`k_proj`/`v_proj` projections automatically). CNN encoders are
2586
+ almost all `Conv2d`, so LoRA does not apply to them — use `--freeze`.
2587
+
2588
+ ### Loss and metric
2589
+
2590
+ The default criterion is `dice_ce` (Dice + cross-entropy), selectable via
2591
+ `--loss dice_ce`; the validation metric is **mean IoU** (`miou`, maximize,
2592
+ also reporting `mdice` and `pixel_acc`). The class count is inferred from
2593
+ the training masks (ignoring `--seg_ignore_index`, default 255) unless
2594
+ `--seg_num_classes` overrides it. Pixels equal to `--seg_ignore_index` are
2595
+ excluded from both the loss and the metric.
2596
+
2597
+ ### Validation overlays
2598
+
2599
+ Set `--vis_every N` to have the trainer call the method's
2600
+ `log_validation` every N epochs, which logs a `seg/overlay` TensorBoard
2601
+ grid of **image / ground-truth mask / predicted mask** rows (8 samples by
2602
+ default) into the run's log dir (default `<checkpoint_dir>/logs`, see
2603
+ `--log_dir`). The masks are rendered with a fixed 20-hue palette; the
2604
+ ignore class (`--seg_ignore_index`, or 255) renders black. Overlays use
2605
+ the *current* model state, so they double as a quick training-progress
2606
+ check (e.g. whether the decoder is learning the class layout before the
2607
+ mIoU catches up).
2608
+
2609
+ Note that some backbones (e.g. plain ViT, Swin) require their native input
2610
+ resolution (224) and reject other sizes.
2611
+
2612
+ ### Inference
2613
+
2614
+ `genml-kit-infer` is classification-only today: it predicts a top-K label
2615
+ for each image and does **not** emit per-pixel masks. For segmentation,
2616
+ use the validation loop (mIoU + overlays above) or the checkpoint's
2617
+ weights directly.
2618
+
2619
+ ---
2620
+
2621
+ ## Object Detection
2622
+
2623
+ Detection predicts a set of **bounding boxes**, each with a class label, for
2624
+ an image. `genml_kit` ships an anchor-free FCOS head over the same encoder
2625
+ feature pyramid that segmentation uses, wired as the `detection` pipeline and
2626
+ the `detection` method.
2627
+
2628
+ ### How It Works
2629
+
2630
+ The model is `det_wrapper:<backbone>` = **encoder + FCOS head**:
2631
+
2632
+ * The encoder is any timm backbone loaded with `features_only=True`, exactly
2633
+ as for segmentation, so it yields a feature pyramid (finest level first).
2634
+ * The head runs a small convolutional tower on **every** pyramid level and
2635
+ predicts, per spatial location, a `num_classes`-way class score, four
2636
+ positive box distances `(l, t, r, b)` in input pixels, and a centerness
2637
+ score. The distances are `exp()`-activated and scaled by the level's
2638
+ stride, so a single location grid spans all scales.
2639
+
2640
+ Training uses the FCOS objective (`--loss fcos` by default), a sum of three
2641
+ terms:
2642
+
2643
+ * **Focal loss** over all locations (the class imbalance between the few
2644
+ positives and the many background locations is large).
2645
+ * **GIoU loss** over the positive locations only.
2646
+ * **Centerness BCE**, which teaches each positive location how central it is
2647
+ inside its target box, so NMS can rank central boxes higher.
2648
+
2649
+ The **assignment** is purely geometric and needs no Hungarian matcher: a
2650
+ location is a *candidate* for a box when it lies inside the box, and a
2651
+ *positive* when it additionally lies inside the box's centre region
2652
+ (`--loss_args center_sampling_radius=1.5`). When several boxes cover a
2653
+ location, the **smallest-area** box wins, so small objects are not starved of
2654
+ positives by large ones.
2655
+
2656
+ At validation time the dense predictions are decoded to `xyxy` boxes,
2657
+ per-class NMS is applied (`--det_nms_thresh`), and the result feeds a
2658
+ COCO-style **mAP** meter (`--det_max_detections`, default 100). The reported
2659
+ metric is `map` (COCO `mAP@[.5:.95]`, maximize), so the best checkpoint is
2660
+ stored under `best_map`; `map_50` and `map_75` are logged alongside it.
2661
+
2662
+ ### Dataset Format
2663
+
2664
+ The `detection` pipeline reads **COCO-JSON** corpora via the `detfolder/`
2665
+ spec:
2666
+
2667
+ ```
2668
+ <root>/images/<file_name>
2669
+ <root>/annotations.json
2670
+ ```
2671
+
2672
+ or, with named splits:
2673
+
2674
+ ```
2675
+ <root>/train/images/<file_name>
2676
+ <root>/train/annotations.json
2677
+ <root>/val/images/<file_name>
2678
+ <root>/val/annotations.json
2679
+ ```
2680
+
2681
+ `annotations.json` is the standard COCO object-detection schema (`images`,
2682
+ `annotations` with `[x, y, width, height]` `bbox`, and `categories`). The
2683
+ loader converts `bbox` to `xyxy` and **compacts** the (1-based, sparse) COCO
2684
+ category ids into a dense `[0, C)` label space, so the head width equals the
2685
+ number of categories actually present. Images with no annotations are legal
2686
+ (pure-background samples) and still contribute negatives to the loss. The
2687
+ annotation file name and image sub-directory are configurable via
2688
+ `--det_annotation_file` and `--det_image_subdir`.
2689
+
2690
+ To generate a throwaway corpus for a CPU smoke run without downloading COCO,
2691
+ use the bundled generator (`scripts/make_tiny_coco.py`), then point the
2692
+ pipeline at it:
2693
+
2694
+ <!-- genml-cli-test
2695
+ name: detection-dataset-format
2696
+ network: true
2697
+ override:
2698
+ - --dataset detfolder/{tmp}/det
2699
+ extra: --epochs 1
2700
+ -->
2701
+ ```bash
2702
+ genml-kit-train --pipeline detection --method detection \
2703
+ --dataset detfolder/tiny_coco --model det_wrapper:timm/resnet18 \
2704
+ --image_size 640 --batch_size 4 --epochs 20
2705
+ ```
2706
+
2707
+ ### Example
2708
+
2709
+ <!-- genml-cli-test
2710
+ name: detection-example
2711
+ network: true
2712
+ override:
2713
+ - --dataset detfolder/{tmp}/det
2714
+ extra: --epochs 1
2715
+ -->
2716
+ ```bash
2717
+ genml-kit-train \
2718
+ --pipeline detection --method detection \
2719
+ --dataset detfolder//data/coco_tiny \
2720
+ --model det_wrapper:timm/resnet50 \
2721
+ --model_arg head=fcos_head:num_convs=2 \
2722
+ --image_size 640 --batch_size 4 \
2723
+ --epochs 20 --lr 1e-4
2724
+ ```
2725
+
2726
+ Override the inferred class count with `--det_num_classes`; tune the decode
2727
+ with `--det_score_thresh` and `--det_nms_thresh`; override the criterion with
2728
+ `--loss` / `--loss_args` (e.g. `--loss_args center_sampling_radius=1.0`).
2729
+
2730
+ ### Transforms
2731
+
2732
+ Detection does **not** reuse the segmentation transform stack: `torchvision`
2733
+ remaps masks automatically through geometric ops but leaves boxes untouched,
2734
+ so every resize / flip must move the coordinates itself. The default
2735
+ pipeline is therefore a hand-written **letterbox** (aspect-preserving resize
2736
+ then pad, which keeps boxes rectangular) followed by a box-aware horizontal
2737
+ flip (train only) and a `(C, H, W)` float conversion. Boxes stay in absolute
2738
+ `xyxy` pixels throughout.
2739
+
2740
+ ---
2741
+
2742
+ ## Visual Odometry
2743
+
2744
+ Visual odometry (VO) estimates the 2-D **similarity transform** — rotation,
2745
+ uniform scale, and translation — that maps one image onto another taken a
2746
+ fraction of a second later. `genml_kit` ships a learned VO front-end plus a
2747
+ synthetic pair generator, wired as the `vo_pair` pipeline and the `vo_pair`
2748
+ method.
2749
+
2750
+ The full mathematics (coordinate frames, the rotation matrix, Umeyama's
2751
+ closed-form solution, the photometric objective, the `mce` metric, and a
2752
+ failure-mode appendix) lives in [`vo/README.md`](vo/README.md) — a
2753
+ from-the-ground-up tutorial. This section is the practical *usage* guide.
2754
+
2755
+ ### Quick start
2756
+
2757
+ <!-- genml-cli-test
2758
+ name: vo-quick-start
2759
+ extra: --epochs 1 --vo_length 16 --vo_val_length 8
2760
+ -->
2761
+ ```bash
2762
+ # Supervised VO on synthetic pairs (default npu-small profile)
2763
+ genml-kit-train \
2764
+ --pipeline vo_pair --method vo_pair \
2765
+ --model vo/npu-small \
2766
+ --image_size 64 --batch_size 32 --epochs 10
2767
+
2768
+ # Photometric self-supervised stage (adds the photometric residual term)
2769
+ genml-kit-train \
2770
+ --pipeline vo_pair --method vo_pair \
2771
+ --vo_stage photometric --epochs 10
2772
+ ```
2773
+
2774
+ ### Model profiles
2775
+
2776
+ The VO net is selected through the model registry as `vo/<profile>`:
2777
+
2778
+ | `--model` | Profile | Encoder widths |
2779
+ |---|---|---|
2780
+ | `vo/npu-small` | default | `(16, 32, 64, 128)` |
2781
+ | `vo/edge-mid` | mid-size | see `genml_kit/models/vo/vo_similar.py` |
2782
+ | `vo/station` | large | see `genml_kit/models/vo/vo_similar.py` |
2783
+
2784
+ ### Pipeline flags (`--pipeline vo_pair`)
2785
+
2786
+ | Flag | Default | Meaning |
2787
+ |---|---|---|
2788
+ | `--vo_length` | `20000` | number of synthetic training pairs |
2789
+ | `--vo_val_length` | `2000` | number of synthetic validation pairs |
2790
+ | `--pitch_deg` | `45.0` | camera pitch for the ground-plane render |
2791
+ | `--terrain` | `field` | terrain profile name |
2792
+ | `--seeded_pairs` | `0` | seed the pair generator for reproducible pairs |
2793
+
2794
+ ### Method flags (`--method vo_pair`)
2795
+
2796
+ | Flag | Default | Meaning |
2797
+ |---|---|---|
2798
+ | `--vo_stage` | `supervised` | `supervised` (mce + similarity deltas) or `photometric` (adds the photometric residual) |
2799
+ | `--vo_profile` | `npu-small` | VO model profile (also selectable via `--model vo/<profile>`) |
2800
+ | `--vo_loss_cfg` | — | comma-separated loss weights `w_log_s,w_theta,w_conf,w_photo` |
2801
+
2802
+ The reported metric is `mce` (mean corner error, in pixels), minimized — the
2803
+ best checkpoint is stored under `best_mce`. Set `--image_size 64` for the
2804
+ synthetic pairs, whose default resolution is 64×64.
2805
+
2806
+ ---
2807
+
2808
+ ## Knowledge Distillation
2809
+
2810
+ Knowledge distillation trains a small **student** to mimic a large **teacher**.
2811
+ `genml_kit` implements the three families of distillation (response / logit
2812
+ matching, feature / FitNets, attention, and relation / RKD) and both teacher
2813
+ sources: a separately-loaded frozen model (`--teacher_kind external`) and an
2814
+ exponential-moving-average copy of the student itself
2815
+ (`--teacher_kind ema`, i.e. self-distillation). It is wired as the
2816
+ `distillation` method over the standard `images` pipeline.
2817
+
2818
+ The full mathematics (temperature and the soft-target insight, the KL↔CE
2819
+ collapse and the `T²` scaling, attention-transfer normalisation, RKD, and the
2820
+ EMA-teacher variance argument) lives in
2821
+ [`knowledge_distillation/README.md`](knowledge_distillation/README.md) — a
2822
+ from-the-ground-up tutorial. This section is the practical *usage* guide.
2823
+
2824
+ ### Quick start
2825
+
2826
+ <!-- genml-cli-test
2827
+ name: distillation-quick-start
2828
+ network: true
2829
+ override:
2830
+ - --dataset {tmp}/img
2831
+ - -.--image_column
2832
+ - --teacher_model google/vit-base-patch16-224
2833
+ - --teacher_checkpoint {tmp}/teacher/model_best.pt
2834
+ extra: --epochs 1
2835
+ -->
2836
+ ```bash
2837
+ # Self-distillation: teacher is an EMA of the student (no external model)
2838
+ genml-kit-train \
2839
+ --pipeline images --method distillation \
2840
+ --model google/vit-base-patch16-224 --image_size 224 \
2841
+ --dataset uoft-cs/cifar10 --image_column img --label_column label \
2842
+ --teacher_kind ema --kd_temperature 4.0 --kd_alpha 1.0 \
2843
+ --epochs 10
2844
+
2845
+ # External teacher: frozen checkpoint hands over its soft targets
2846
+ genml-kit-train \
2847
+ --pipeline images --method distillation \
2848
+ --model google/vit-base-patch16-224 --image_size 224 \
2849
+ --dataset uoft-cs/cifar10 --image_column img --label_column label \
2850
+ --teacher_kind external \
2851
+ --teacher_model google/vit-large-patch16-224 \
2852
+ --teacher_checkpoint /path/to/teacher.pt \
2853
+ --kd_temperature 4.0 --kd_alpha 1.0 --epochs 10
2854
+ ```
2855
+
2856
+ ### Method flags (`--method distillation`)
2857
+
2858
+ | Flag | Default | Meaning |
2859
+ |---|---|---|
2860
+ | `--teacher_kind` | `external` | `external` loads a frozen teacher; `ema` self-distils from an EMA of the student |
2861
+ | `--teacher_model` | — | teacher model spec (required for `--teacher_kind external`) |
2862
+ | `--teacher_checkpoint` | — | checkpoint holding the external teacher's weights |
2863
+ | `--kd_temperature` | `4.0` | distillation temperature $`T`$ |
2864
+ | `--kd_alpha` | `1.0` | weight of the response (logit) KD term (`0` disables it) |
2865
+ | `--kd_feature_alpha` | `0.0` | weight of the FitNets feature KD term (`0` disables it) |
2866
+ | `--kd_attention` / `--no-kd_attention` | off | use attention transfer on the matched layers instead of raw feature MSE |
2867
+ | `--kd_relation_alpha` | `0.0` | weight of the relational (RKD) KD term (`0` disables it) |
2868
+ | `--kd_layer_strategy` | `uniform` | how to pair teacher/student layers: `by_index`, `uniform`, `by_name` |
2869
+ | `--kd_layer_pattern` | — | regex selecting the feature layers to pair (feature KD) |
2870
+ | `--kd_proj_hidden` | `512` | hidden width of the FitNets projection |
2871
+ | `--kd_ema_momentum` | `0.996` | initial EMA momentum for `--teacher_kind ema` |
2872
+ | `--kd_ema_final_momentum` | `1.0` | final EMA momentum for `--teacher_kind ema` |
2873
+
2874
+ Enable one family at a time and watch the validation metric after each addition
2875
+ (see Appendix B of the distillation tutorial for the pitfalls). A labeled
2876
+ `--dataset` is required: the ground-truth cross-entropy branch anchors the
2877
+ student on examples where the teacher is confidently wrong.
2878
+
2879
+ ---
2880
+
2881
+ ## Large Language Models (causal LM)
2882
+
2883
+ genml_kit can fine-tune decoder-only language models through the same
2884
+ branchless driver: a new `text` pipeline (token collation) pairs with a new
2885
+ `causal_lm` method (shifted next-token cross-entropy, perplexity metric).
2886
+
2887
+ ### Quick start
2888
+
2889
+ <!-- genml-cli-test
2890
+ name: causal-lm-quick-start
2891
+ network: true
2892
+ extras: lora
2893
+ override:
2894
+ - --dataset {tmp}/text
2895
+ extra: --epochs 1
2896
+ -->
2897
+ ```bash
2898
+ # Fine-tune a tiny GPT-2 on a local text corpus
2899
+ genml-kit-train --pipeline text \
2900
+ --method causal_lm \
2901
+ --model sshleifer/tiny-gpt2 \
2902
+ --dataset ./my_text_corpus \
2903
+ --text_column text \
2904
+ --max_seq_len 256 \
2905
+ --lora
2906
+ ```
2907
+
2908
+ The `--dataset` value may be a HuggingFace dataset name, or a directory saved
2909
+ with `datasets.Dataset.save_to_disk` (containing `train` / `validation`
2910
+ splits). With `--val_split` a held-out fraction of the training split is used
2911
+ for validation instead.
2912
+
2913
+ ### Pipeline flags (`--pipeline text`)
2914
+
2915
+ | Flag | Default | Meaning |
2916
+ |---|---|---|
2917
+ | `--text_column` | — | column holding a raw LM sequence (plain-LM mode) |
2918
+ | `--instruction_column` | — | prompt column (instruction mode, with `--response_column`) |
2919
+ | `--response_column` | — | response column (instruction mode) |
2920
+ | `--max_seq_len` | `512` | token sequence length (truncation / packing window) |
2921
+ | `--sequence_mode` | `document` | sequence recipe: `document` (no packing, document-exact boundary loss, prompt framed as a document prefix) or `open` (packed concatenation, boundary loss kept, prompt framed as a whole document) |
2922
+ | `--mask_prompt` | off | mask prompt tokens out of the loss (instruction tuning) |
2923
+ | `--packing [wrapped\|bfd]` | off | pack sequences into fixed `max_seq_len` windows (`wrapped` = concatenate-and-slice, `bfd` = best-fit-decreasing) |
2924
+ | `--length_bucket` | off | batch items of similar length so padded batches waste little compute (mutually exclusive with `--packing`) |
2925
+ | `--pad_to_multiple_of` | — | round padded length up to a multiple (e.g. `8`) |
2926
+ | `--max_items` | — | cap the number of loaded items (smoke tests) |
2927
+
2928
+ #### Packing and document boundaries
2929
+
2930
+ Padding every item up to the batch's longest wastes compute on short documents;
2931
+ packing removes that waste but, naively, lets tokens of one document attend to a
2932
+ *different* document sharing the same window. Three modes trade padding for
2933
+ compute, and packing stays document-aware without binding the data side to any
2934
+ model:
2935
+
2936
+ | Mode | How it batches | Document isolation |
2937
+ |---|---|---|
2938
+ | *(default)* | pad every row to the batch's longest item | plain, document-exact 2-D mask |
2939
+ | `--packing wrapped` | concatenate the tokenized stream (EOS-separated) and slice into `max_seq_len` windows | via `segment_ids` (model-dependent, below) |
2940
+ | `--packing bfd` | best-fit-decreasing: longest document first into the open bin with the least remaining space | via `segment_ids` (model-dependent, below) |
2941
+ | `--length_bucket` | no packing: sort by a cheap character-length proxy, group into buckets, shuffle within and across them, emit similar-length batches | plain, document-exact 2-D mask |
2942
+
2943
+ `wrapped` is the historical behaviour and may split a document across two
2944
+ windows; `bfd` yields fewer, fuller windows but reorders documents, so batch
2945
+ order is **not** reproducible with respect to `--seed` (it warns once).
2946
+ `--length_bucket` is mutually exclusive with `--packing`.
2947
+
2948
+ **Batching and memory.** On every path except `wrapped`, `--batch_size` counts
2949
+ *documents* (each becomes one model row, or one packed row for `bfd`). Under
2950
+ `wrapped` the collator concatenates the batch and slices it into `max_seq_len`
2951
+ windows, so the row count is `ceil(total_tokens / max_seq_len)` -- which the
2952
+ document count does **not** bound. A batch of long documents can therefore fan
2953
+ out into far more rows than `--batch_size` and exhaust VRAM (one long document
2954
+ alone becomes `len(doc) / max_seq_len` rows), and lowering `--batch_size` or
2955
+ `--grad_accum_steps` barely helps because the cost tracks *tokens*, not
2956
+ documents.
2957
+
2958
+ To make `wrapped` safe, `--batch_size` is honored as a **token budget**:
2959
+ `PackingBatchSampler` groups documents until a batch fills `--batch_size *
2960
+ max_seq_len` tokens (the constant-length packing recipe Megatron-LM and
2961
+ HuggingFace packed datasets use), and the collator hard-caps every emitted batch
2962
+ at `--batch_size` windows. A document longer than one window is still streamed
2963
+ in -- its tail lands in a later window of the same batch rather than being
2964
+ truncated away -- and only the overflow past the whole batch budget (a single
2965
+ document longer than `--batch_size` windows) is dropped, with a one-time
2966
+ warning. If you see that warning, raise `--batch_size` or lower `--max_items`.
2967
+
2968
+ `--sequence_mode` compiles the windowing flags and the two coupled policies
2969
+ below into one validated object (`genml_kit/training/sequence_mode.py`), so an
2970
+ incoherent pairing cannot be expressed. The two recipes are:
2971
+
2972
+ - `document` (default) -- unpacked, **boundary loss masked**, prompts framed as
2973
+ a document *prefix* (`[BOS] ...`, no trailing `EOS`). The validation
2974
+ perplexity is document-exact and the first sampled token is a trained
2975
+ "continue this document" state.
2976
+ - `open` -- packed (`wrapped`), **boundary loss kept**, prompts framed as a whole
2977
+ *document* (`[BOS] ... [EOS]`). This is the GPT-2/GPT-3 pre-training recipe:
2978
+ the token following a document's `EOS` keeps a trained next-token
2979
+ distribution, so "what follows a document" is a learned state rather than an
2980
+ untrained one. `open` requires packing, and `--length_bucket` is rejected
2981
+ with it.
2982
+
2983
+ Explicit `--packing` / `--length_bucket` override the recipe's default
2984
+ windowing; the boundary-loss and framing policies still come from the recipe.
2985
+ A kept boundary loss is meaningful only under packing: unpacked, the tokens
2986
+ after a document's `EOS` are padding, so keeping the boundary label would train
2987
+ `EOS -> PAD` -- which is why `open` (and the collator) reject it.
2988
+
2989
+ Document isolation (independent of the boundary loss) works as follows:
2990
+
2991
+ - Under `--packing`, the collator emits two extra tensors per batch, requiring
2992
+ no model knowledge: `segment_ids` (the contiguous run -- i.e. source document
2993
+ -- of each token, `-1` for padding) and `position_ids` (restarts at `0` at
2994
+ every run start, so a window is self-describing).
2995
+ - The `causal_lm` method, the only place that holds both the batch and the
2996
+ built model, translates those tensors to whatever the model declares via its
2997
+ `PACKED_ATTENTION` class attribute:
2998
+
2999
+ | `PACKED_ATTENTION` | What the model receives | Document isolation |
3000
+ |---|---|---|
3001
+ | `"none"` (default) | nothing extra | `--packing` is refused with a clear error |
3002
+ | `"block_diagonal"` | a 4-D `[B, 1, L, L]` additive mask | exact (no flash kernel) |
3003
+ | `"varlen"` | `FlashAttentionKwargs` (`cu_seq_lens_q/k`, `max_length_q/k`) on a flattened `[1, total]` batch | exact (flash when `flash-attn` is installed) |
3004
+ | `"segment_ids"` | the raw `[B, L]` `segment_ids` runs plus the 2-D padding mask and `position_ids` | model-defined (reset state per run) |
3005
+ | `"causal"` | only the 2-D padding mask and `position_ids` (no boundary tensors) | none -- documents are concatenated and may attend across boundaries (the standard LLM pre-training recipe) |
3006
+
3007
+ HuggingFace decoders are inferred automatically: `--attn_implementation
3008
+ flash_attention_2` implies `varlen`, otherwise `block_diagonal`. Custom
3009
+ models (subclasses of `GenerativeLM`) opt in explicitly. The `segment_ids`
3010
+ mode is for models that compute attention *without* materializing the `L x L`
3011
+ score matrix (linear / kernelized, streaming or recurrent attention): a dense
3012
+ block-diagonal mask would defeat the cost such a mechanism is built to avoid,
3013
+ so the model instead receives the raw per-token document runs and resets its
3014
+ state at each boundary. The `causal` mode opts *out* of document isolation
3015
+ altogether: the model receives only the ordinary 2-D key-padding mask and
3016
+ `position_ids`, and a plain causal mask attends across document boundaries.
3017
+ This is the recipe used by GPT-3, PaLM, LLaMA, Falcon and Mistral
3018
+ (concatenate with an EOD marker, no cross-document blocking), so it is the
3019
+ right choice when reproducing that setup. The collator still emits
3020
+ `segment_ids`; a `causal` model simply ignores them. Use `causal` for models
3021
+ that compute attention without an `L x L` score matrix *and* want to match
3022
+ mainstream pre-training; keep `segment_ids` when document isolation is itself
3023
+ the property under test.
3024
+
3025
+ Because document boundaries are derived from `segment_ids` (not the raw
3026
+ `attention_mask`), unless the model declares `causal`, packing a model that only
3027
+ understands a 2-D mask is a hard error rather than a silent cross-document leak.
3028
+ The collator also masks the label opening each document, so the causal shift
3029
+ never predicts across a boundary -- packed training stays equivalent to the
3030
+ per-document padded case and `--mask_prompt` is honoured. `flash-attn` is an
3031
+ optional extra (`pip install genml_kit[flash]`); without it everything runs on
3032
+ the block-diagonal (pure-torch) path, so CPU and CI never require a GPU.
3033
+
3034
+ `--dataset`, `--split`, `--val_split`, `--cache_dir`, `--image_column`,
3035
+ `--max_seq_len`, `--sequence_mode`, `--mask_prompt`, `--pad_to_multiple_of` and
3036
+ `--max_items` are
3037
+ **globally shared** CLI arguments: they are declared once by the harness
3038
+ (`genml_kit/utils/shared_args.py`), so every pipeline and method can read
3039
+ `args.<flag>` directly instead of guarding with `getattr(...)`.
3040
+
3041
+ ### Method flags (`--method causal_lm`)
3042
+
3043
+ | Flag | Default | Meaning |
3044
+ |---|---|---|
3045
+ | `--tokenizer` | `--model` | tokenizer name/path (defaults to the model) |
3046
+ | `--trust_remote_code` | off | allow custom code from the model hub |
3047
+ | `--attn_implementation` | — | attention backend (`sdpa`, `eager`, `flash_attention_2`) |
3048
+
3049
+ The metric is `ppl` (perplexity, lower is better), so the best checkpoint is
3050
+ stored under `best_ppl`. `--lora`, `--grad_checkpoint`, `--freeze`,
3051
+ `--source_checkpoint` and `--resume` all work for causal LMs through the same
3052
+ shared model-extras path every other objective uses.
3053
+
3054
+ ### Efficient fine-tuning (LoRA, gradient checkpointing, bf16)
3055
+
3056
+ LoRA is the recommended way to fine-tune a causal LM. When
3057
+ `--lora_target_modules` is omitted, genml_kit **resolves the target modules
3058
+ from the model's own structure** rather than from a hardcoded name list, so it
3059
+ works across architectures without configuration:
3060
+
3061
+ | Model family | Auto-resolved targets |
3062
+ |---|---|
3063
+ | GPT-2 (HF `Conv1D`) | `c_attn`, `attn.c_proj` |
3064
+ | Llama / Mistral / Qwen2 / HF ViT | `q_proj`, `k_proj`, `v_proj`, `o_proj` |
3065
+ | BERT | `query`, `key`, `value`, `attention.output.dense` |
3066
+ | timm ViT | `qkv`, `attn.proj` |
3067
+ | No attention (plain MLP backbone) | every `nn.Linear` (with a warning) |
3068
+
3069
+ The resolver detects attention blocks by class/structure, keeps only their
3070
+ projection layers, and emits the shortest dotted suffix that does **not** also
3071
+ match a non-attention layer (so GPT-2's `attn.c_proj` is not confused with
3072
+ `mlp.c_proj`). Pass `--lora_target_modules` explicitly to override.
3073
+
3074
+ <!-- genml-cli-test
3075
+ name: causal-lm-lora
3076
+ network: true
3077
+ extras: lora
3078
+ override:
3079
+ - --dataset {tmp}/text
3080
+ - --model sshleifer/tiny-gpt2
3081
+ -->
3082
+ ```bash
3083
+ # LoRA + gradient checkpointing + bf16 on a small LM
3084
+ genml-kit-train --pipeline text \
3085
+ --method causal_lm \
3086
+ --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
3087
+ --dataset ./my_text_corpus \
3088
+ --text_column text \
3089
+ --max_seq_len 1024 \
3090
+ --lora --lora_r 16 --lora_alpha 32 \
3091
+ --grad_checkpoint \
3092
+ --amp_dtype bfloat16
3093
+ ```
3094
+
3095
+ `--lora`, `--lora_r`, `--lora_alpha`, `--lora_dropout` and
3096
+ `--lora_target_modules` are the shared LoRA flags; `--grad_checkpoint` enables
3097
+ per-block activation checkpointing (HF `gradient_checkpointing_enable` for
3098
+ transformers models, native APIs for timm/custom backbones); `--amp_dtype
3099
+ bfloat16` selects the autocast dtype.
3100
+
3101
+ When the model is a PEFT `PeftModel`, checkpoints also embed a **portable
3102
+ adapter blob** (`lora_state_blob`: a tar of `adapter_model.safetensors` +
3103
+ `adapter_config.json`), so the adapters can be loaded with stock PEFT or
3104
+ published to the Hub, while `--resume` continues to restore the raw tensor
3105
+ state exactly.
3106
+
3107
+ ### Tokenizers
3108
+
3109
+ `load_tokenizer(name)` mirrors the model/processor split. Normal model IDs
3110
+ resolve through `transformers.AutoTokenizer`, which already handles both
3111
+ SentencePiece and tiktoken Hub vocabularies. Two registry entries cover the
3112
+ gaps: `byte` (a dependency-free tokenizer for offline tests) and
3113
+ `spm` (a bare SentencePiece `tokenizer.model`, behind the optional
3114
+ `sentencepiece` dependency).
3115
+
3116
+ A byte-level BPE is built separately, from a corpus subset, by
3117
+ `genml-kit-tokenizer`. Building is a one-shot offline job rather than a side
3118
+ effect of a training run, so the text behind the vocabulary is a reproducible
3119
+ artifact: pass the resulting directory straight to `--tokenizer`. The
3120
+ `tokenizer_meta.json` sidecar records the corpus, split, fraction, seed and
3121
+ row counts it was trained on.
3122
+
3123
+ <!-- genml-cli-test
3124
+ name: tokenizer-build
3125
+ override:
3126
+ - --dataset {tmp}/text
3127
+ - --out {tmp}/tok
3128
+ extra: --vocab_size 128 --fraction 0.5 --seed 0
3129
+ -->
3130
+ ```bash
3131
+ genml-kit-tokenizer --dataset roneneldan/TinyStories --text_column text \
3132
+ --fraction 0.1 --seed 42 --vocab_size 4096 \
3133
+ --cache_dir hf --out hf/tokenizers/ts_bpe_4096_s0.1
3134
+ ```
3135
+
3136
+ `--fraction (0.0, 1.0]` trains on that share of the rows: an indexable dataset
3137
+ yields exactly `round(n*fraction)` rows, drawn without replacement and
3138
+ seed-stable, while `--streaming` corpora get a seeded per-row sample (use
3139
+ `--max_rows` to bound how much of a stream is read).
3140
+
3141
+ ### Generation
3142
+
3143
+ `genml-kit-llm generate` turns prompts into completions. Generation is a
3144
+ **model contract**, not a HuggingFace-only path: a model opts in by
3145
+ subclassing `GenerativeLM` (`genml_kit/models/generative.py`), whose default
3146
+ `generate` runs a portable token loop over the architecture-agnostic
3147
+ `forward(...) -> .logits` contract that `causal_lm` already trains against.
3148
+ HuggingFace models are *not* wrapped (their `state_dict` keys must stay intact
3149
+ for `--resume`); the dispatcher calls their own `generate` directly. So custom
3150
+ models registered in `MODELS` generate exactly like HF ones.
3151
+
3152
+ <!-- genml-cli-test
3153
+ name: llm-generate
3154
+ network: true
3155
+ override:
3156
+ - --prompts {tmp}/prompts/prompts.txt
3157
+ - --dataset {tmp}/prompt_ds --split test
3158
+ - --model sshleifer/tiny-gpt2
3159
+ - --checkpoint {tmp}/chk_infer/model_best.pt
3160
+ - -.--lora_adapter
3161
+ -->
3162
+ ```bash
3163
+ # Prompts from a file (one per line, or JSONL with --prompt_column)
3164
+ genml-kit-llm generate --model sshleifer/tiny-gpt2 \
3165
+ --prompts prompts.txt \
3166
+ --max_new_tokens 64 \
3167
+ --do_sample --temperature 0.8 --top_p 0.95 \
3168
+ --output completions.jsonl
3169
+
3170
+ # Prompts from a HuggingFace dataset column, with a fine-tuned checkpoint
3171
+ genml-kit-llm generate --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
3172
+ --dataset my-org/prompts --split test \
3173
+ --prompt_column question \
3174
+ --checkpoint ./checkpoints/llm_best.pt \
3175
+ --lora_adapter ./adapters/run1
3176
+ ```
3177
+
3178
+ | Argument | Default | Description |
3179
+ |---|---|---|
3180
+ | `--model` | (required) | HuggingFace id or registered custom model. |
3181
+ | `--tokenizer` | `--model` | Tokenizer name/path override. |
3182
+ | `--checkpoint` | `None` | State-dict `.pt`, or a `<root>` resolving to `<root>_best.pt` / `<root>_latest.pt`. |
3183
+ | `--lora_adapter` | `None` | PEFT adapter directory (`save_pretrained` layout). |
3184
+ | `--prompts` / `--dataset` | — | Prompt source (file/JSONL, or HF dataset + `--prompt_column`). |
3185
+ | `--max_new_tokens` | `64` | Tokens to generate. |
3186
+ | `--do_sample` | off | Sample instead of greedy decoding. |
3187
+ | `--temperature` / `--top_k` / `--top_p` | `1.0` / `0` / `1.0` | Sampling knobs. |
3188
+ | `--repetition_penalty` | `1.0` | Down-weight already-seen tokens (`1.0` disables). |
3189
+ | `--num_beams` | `1` | Beam count (portable backend; `1` = greedy). |
3190
+ | `--num_return_sequences` | `1` | Completions per prompt. |
3191
+ | `--no_chat_template` | off | Skip the tokenizer's chat template. |
3192
+
3193
+ When the tokenizer exposes a chat template (most instruct models do),
3194
+ prompts are automatically wrapped as a single user turn; `--no_chat_template`
3195
+ keeps the raw prompt for base-model completion.
3196
+
3197
+ ### Evaluation
3198
+
3199
+ `genml-kit-llm eval` reports quality metrics for a labeled dataset:
3200
+
3201
+ - **perplexity** — teacher-forced (`exp` of the mean shifted cross-entropy),
3202
+ over the `--text_column`; no generation involved.
3203
+ - **exact-match** — fraction of generations matching the reference exactly.
3204
+ - **token-F1** — SQuAD-style token-overlap F1 against the reference.
3205
+ - **pass@k** — optional and pluggable: `--pass_at_k K --scorer score.py`
3206
+ invokes `score(prompt, completion, reference) -> float` `K` times per item
3207
+ and reports the pass rate. No code execution is built in.
3208
+
3209
+ <!-- genml-cli-test
3210
+ name: llm-eval
3211
+ network: true
3212
+ override:
3213
+ - --dataset {tmp}/eval
3214
+ - --model sshleifer/tiny-gpt2
3215
+ -->
3216
+ ```bash
3217
+ genml-kit-llm eval --model sshleifer/tiny-gpt2 \
3218
+ --dataset ./my_eval_corpus --split test \
3219
+ --text_column text --prompt_column prompt \
3220
+ --reference_column reference \
3221
+ --metrics ppl exact_match token_f1 \
3222
+ --output report.json
3223
+ ```
3224
+
3225
+ Both subcommands load their model through the same registry the trainer uses,
3226
+ so a registered custom model generates and evaluates with no code changes.
3227
+
3228
+ ### Alignment (DPO and reward models)
3229
+
3230
+ Preference-based post-training runs through the same branchless driver: a new
3231
+ `preference` pipeline (chosen/rejected pairs) pairs with the `dpo` and
3232
+ `reward_model` methods. The typical recipe is SFT → DPO → (optional) reward
3233
+ model:
3234
+
3235
+ <!-- genml-cli-test
3236
+ name: preference-chain
3237
+ network: true
3238
+ extras: lora
3239
+ extra: --epochs 1
3240
+ -->
3241
+ ```bash
3242
+ # 1. Supervised fine-tune (masking the prompt out of the loss)
3243
+ genml-kit-train --pipeline text --method causal_lm \
3244
+ --model sshleifer/tiny-gpt2 --dataset ./sft_corpus \
3245
+ --instruction_column prompt --response_column response \
3246
+ --mask_prompt --checkpoint ./sft/model
3247
+
3248
+ # 2. DPO against a frozen reference (defaults to --model)
3249
+ genml-kit-train --pipeline preference --method dpo \
3250
+ --model sshleifer/tiny-gpt2 \
3251
+ --reference_checkpoint ./sft/model_best.pt \
3252
+ --dataset ./preferences --chosen_column chosen \
3253
+ --rejected_column rejected --prompt_column prompt \
3254
+ --mask_prompt --beta 0.1 --lora
3255
+
3256
+ # 3. Train a reward model on the same pairs
3257
+ genml-kit-train --pipeline preference --method reward_model \
3258
+ --model sshleifer/tiny-gpt2 --dataset ./preferences \
3259
+ --chosen_column chosen --rejected_column rejected
3260
+ ```
3261
+
3262
+ **DPO** compares the policy against a **frozen reference** and pushes the
3263
+ chosen response's implicit reward above the rejected one. The reference is
3264
+ loaded explicitly (`--reference_model`, defaulting to `--model`, plus
3265
+ `--reference_checkpoint`) and held as a plain attribute — it is never written
3266
+ to `state_dict` and is rebuilt from args on resume. The sequence score is the
3267
+ mean per-token log-probability of the response tokens, and `--beta` (default
3268
+ `0.1`) sets the KL strength. Validation reports pairwise preference accuracy.
3269
+
3270
+ **Reward model** attaches a scalar `nn.Linear` head directly to the backbone
3271
+ as `model.score` — no wrapper class, so the backbone's `state_dict` keys stay
3272
+ unprefixed and `--source_checkpoint` matches them exactly (only `score.*` is
3273
+ new). It scores the last non-pad hidden state and trains with the
3274
+ Bradley–Terry ranking loss; validation reports pairwise ranking accuracy. The
3275
+ hidden size is read from the config (`hidden_size` / `n_embd` / `d_model`),
3276
+ falling back to a one-token probe.
3277
+
3278
+ The backbone must expose last-layer hidden states (accept
3279
+ `output_hidden_states=True`); HuggingFace models do, custom models opt in —
3280
+ the same contract used by `GenerativeLM`.
3281
+
3282
+ ---
3283
+
1665
3284
  ## Reinforcement Learning
1666
3285
 
1667
3286
  genml_kit ships a self-contained reinforcement-learning (RL) toolkit built on
@@ -1688,7 +3307,12 @@ pip install 'genml_kit[rl]'
1688
3307
 
1689
3308
  ### Quick start
1690
3309
 
1691
- ```
3310
+ <!-- genml-cli-test
3311
+ name: rl-quick-start
3312
+ extras: rl
3313
+ extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
3314
+ -->
3315
+ ```bash
1692
3316
  # DQN on CartPole (discrete)
1693
3317
  genml-kit-train --pipeline rl --method dqn --env_id CartPole-v1 \
1694
3318
  --epochs 20 --batch_size 64
@@ -1713,7 +3337,12 @@ Every RL method resolves its network through the same model registry as the
1713
3337
  rest of the framework: `--model` selects a registered factory and
1714
3338
  `--model_arg key=value` forwards constructor kwargs.
1715
3339
 
1716
- ```
3340
+ <!-- genml-cli-test
3341
+ name: rl-custom-models
3342
+ extras: rl
3343
+ extra: --epochs 1
3344
+ -->
3345
+ ```bash
1717
3346
  # DQN: dueling Q-head (also the default for --dqn_dueling)
1718
3347
  genml-kit-train --pipeline rl --method dqn --model rl/qnet_dueling
1719
3348
 
@@ -1907,12 +3536,34 @@ a small positive default as light regularization.
1907
3536
 
1908
3537
  A typical "serious" continuous-PPO setup (Pendulum-style) looks like:
1909
3538
 
1910
- ```
3539
+ <!-- genml-cli-test
3540
+ name: rl-ppo-serious
3541
+ extras: rl
3542
+ extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
3543
+ -->
3544
+ ```bash
1911
3545
  genml-kit-train --pipeline rl --method ppo --env_id Pendulum-v1 \
1912
3546
  --ppo_continuous --obs_normalize --reward_normalize --param_init ortho \
1913
3547
  --ppo_target_kl 0.03 --epochs 500
1914
3548
  ```
1915
3549
 
3550
+ SAC's temperature auto-tuning and critic/policy learning rates follow the
3551
+ same pattern (the critic head is `rl/sac_critic`, the policy is
3552
+ `rl/actor_critic`):
3553
+
3554
+ <!-- genml-cli-test
3555
+ name: rl-sac-tuning
3556
+ extras: rl
3557
+ extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
3558
+ -->
3559
+ ```bash
3560
+ genml-kit-train --pipeline rl --method sac --env_id Pendulum-v1 \
3561
+ --sac_auto_alpha --sac_alpha_lr 3e-4 --sac_target_entropy -1.0 \
3562
+ --sac_critic_lr 3e-4 --sac_actor_lr 3e-4 --sac_tau 0.005 \
3563
+ --replay_capacity 5000 --batch_size 64 \
3564
+ --epochs 200
3565
+ ```
3566
+
1916
3567
  ### Custom environments
1917
3568
 
1918
3569
  Pass `--env_script /path/to/env.py` to load a custom environment. The
@@ -1928,6 +3579,31 @@ Relevant flags:
1928
3579
  - `--env_id`, `--obs_dim`, `--env_script` -- environment selection.
1929
3580
  - `--replay_capacity`, `--batch_size`, `--n_step`, `--prioritized`,
1930
3581
  `--per_alpha`, `--per_beta_start`, `--per_beta_frames` -- replay tuning.
3582
+
3583
+ A DQN run with the full replay/epsilon machinery (dueling head, double-Q,
3584
+ n-step returns, prioritized replay):
3585
+
3586
+ <!-- genml-cli-test
3587
+ name: rl-dqn-replay
3588
+ extras: rl
3589
+ extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
3590
+ -->
3591
+ ```bash
3592
+ genml-kit-train --pipeline rl --method dqn --env_id CartPole-v1 \
3593
+ --dqn_dueling --dqn_ddqn --n_step 3 --prioritized \
3594
+ --replay_capacity 5000 --batch_size 32 \
3595
+ --dqn_gamma 0.99 --dqn_tau 0.005 --dqn_target_update_freq 100 \
3596
+ --dqn_epsilon_start 1.0 --dqn_epsilon_end 0.05 \
3597
+ --dqn_epsilon_decay_steps 500 \
3598
+ --epochs 50
3599
+ ```
3600
+ - **Discount factor** -- there is intentionally **no** global `--gamma`: each
3601
+ method owns its own because RL objectives are not active simultaneously and
3602
+ tune it differently. DQN uses `--dqn_gamma`, PPO `--ppo_gamma`, SAC
3603
+ `--sac_gamma` (all default `0.99`). The n-step returns computed by the
3604
+ shared replay buffer use the discount of the **selected method**; the
3605
+ method installs it into the buffer during `wire_data`, so a single run never
3606
+ mixes discounts.
1931
3607
  - `--obs_normalize`, `--obs_norm_clip` -- running observation
1932
3608
  normalization (Welford). Both train and eval observations are
1933
3609
  normalized with the same RMS statistics.
@@ -1960,6 +3636,209 @@ bootstraps `γ V(s′)`, while a terminated one does not. Environments using
1960
3636
  the legacy 4-tuple gym API (no `terminated`/`truncated` split) fall back to
1961
3637
  treating `done` as `terminated`.
1962
3638
 
3639
+ ## Multi-GPU Training
3640
+
3641
+ The harness runs on a single device by default and needs no launcher for that.
3642
+ To use more than one GPU, launch the same `genml-kit-train` command under
3643
+ `torchrun` (which ships with PyTorch -- it is the `torch.distributed.run`
3644
+ module, so there is nothing extra to install):
3645
+
3646
+ <!-- genml-cli-test-exclude: illustrative GPU/nccl launch; the runnable distributed example follows -->
3647
+ ```bash
3648
+ # 4 GPUs, one process per GPU
3649
+ torchrun --standalone --nproc_per_node=4 -m genml_kit.cli.train \
3650
+ --model convvit \
3651
+ --dataset /data/imagenet \
3652
+ --epochs 90 \
3653
+ --batch_size 64 \
3654
+ --dist_arg backend=nccl
3655
+ ```
3656
+
3657
+ The same command runs on CPU with the `gloo` backend, which is what the
3658
+ runnable example below does (two ranks, a tiny synthetic dataset):
3659
+
3660
+ <!-- genml-cli-test
3661
+ name: multi-gpu-torchrun
3662
+ launcher: torchrun
3663
+ nproc: 2
3664
+ extra: --device cpu --batch_size 4 --image_size 32 --num_workers 0
3665
+ override:
3666
+ - --dataset {tmp}/img_ddp
3667
+ -->
3668
+ ```bash
3669
+ # 2 processes on CPU (gloo); swap in 4 GPUs + `backend=nccl` for a real run
3670
+ torchrun --standalone --nproc_per_node=2 -m genml_kit.cli.train \
3671
+ --model convvit \
3672
+ --dataset /data/imagenet \
3673
+ --epochs 1 \
3674
+ --dist_arg backend=gloo
3675
+ ```
3676
+
3677
+ Every rank reads `RANK` / `WORLD_SIZE` / `LOCAL_RANK` / `MASTER_ADDR` /
3678
+ `MASTER_PORT` from the environment; when they are absent (a plain
3679
+ `genml-kit-train ...` invocation) the distributed layer is completely inert --
3680
+ no process group is created, the model is not wrapped, and the data loader is
3681
+ untouched. Single-GPU behaviour is therefore unchanged, and `torchrun` is
3682
+ never required.
3683
+
3684
+ What the launcher turns on automatically:
3685
+
3686
+ | Behaviour | Effect |
3687
+ |---|---|
3688
+ | Process group | `nccl` on CUDA, `gloo` otherwise; override with `--dist_arg backend=` |
3689
+ | Device | each rank pins to `cuda:$LOCAL_RANK` |
3690
+ | Seeding | `base_seed + rank`, so ranks shuffle differently but reproducibly |
3691
+ | Data sharding | each rank gets a disjoint slice of the training set (see below) |
3692
+ | Checkpoints | written by rank 0 only; keys carry no `module.` prefix, so a multi-GPU checkpoint loads back into a single-GPU run unchanged |
3693
+ | Metrics / log files | TensorBoard, JSONL and log files are written by rank 0 only |
3694
+ | Validation | sharded across ranks like training; each rank evaluates its disjoint slice, the per-rank results are merged (`Method.merge_validation`), and rank 0 reports the whole-set metric |
3695
+
3696
+ Distributed knobs travel through a single repeatable `--dist_arg KEY=VALUE`
3697
+ flag, matching `--model_arg` / `--opt_arg`. A bare key means `true`:
3698
+
3699
+ | Key | Default | Description |
3700
+ |---|---|---|
3701
+ | `backend` | auto (`nccl` on CUDA, else `gloo`) | Process-group backend. |
3702
+ | `shard` | `true` when more than one rank | Shard the training set across ranks. Set `shard=false` to run every rank over the full dataset (rarely what you want: it multiplies the compute with no error). |
3703
+ | `find_unused_parameters` | `false` (declared `true` by BYOL / DINO / SupCon) | Pass through to `DistributedDataParallel`. Lets a forward that skips some trainable parameters survive DDP's unused-parameter check. The SSL methods declare it defensively; a stock run leaves nothing unused and would pass without it. |
3704
+ | `static_graph` | `false` | Pass through to `DistributedDataParallel`. Only valid when the autograd graph is identical every step. |
3705
+ | `rollout_shard` | `true` when more than one rank | RL pipelines only. Shard rollout collection (and the minibatch it yields) across ranks. Set `rollout_shard=false` to collect one full rollout per rank (rarely what you want: it multiplies the environment compute with no error). |
3706
+
3707
+ Defaults are chosen so that a multi-GPU run only has to be correct, not
3708
+ tuned: sharding is on whenever it can matter. The validation set is sharded the
3709
+ same way as training, each rank evaluates its disjoint slice, and the per-rank
3710
+ results are merged (`Method.merge_validation`) so the reported metric always
3711
+ describes the whole set; only rank 0 logs it.
3712
+
3713
+ These defaults are *method-aware*: the selected pipeline or method may declare
3714
+ its own default for any flag via `arg_defaults()`, and `--dist_arg` merges on
3715
+ top of it key by key. An explicit command-line flag always wins, so
3716
+ `--dist_arg static_graph` still runs a method's declared
3717
+ `find_unused_parameters` alongside it.
3718
+
3719
+ Three caveats worth knowing before reaching for more GPUs:
3720
+
3721
+ - **Effective batch size scales with `--nproc_per_node`.** Four ranks at
3722
+ `--batch_size 64` see 256 samples per optimizer step; scale `--lr`
3723
+ accordingly.
3724
+ - **`shard` assumes a trainable `DataLoader`.** Pipelines that bypass it are
3725
+ not covered by *that* path: the RL pipelines drive environments and replay
3726
+ buffers directly, so they use their own `rollout_shard` knob (see the table
3727
+ above) instead of `shard`.
3728
+ - **A checkpoint from a multi-GPU run is portable**, but resuming with a
3729
+ *different* `--nproc_per_node` changes the shard boundaries, so the epoch
3730
+ position is approximate if the dataset size is not divisible by the rank
3731
+ count.
3732
+
3733
+ ---
3734
+
3735
+ ## Hyperparameter Search
3736
+
3737
+ `genml-kit-tune` runs a hyperparameter *study* on top of the unified trainer.
3738
+ It is deliberately decoupled from the training code: **every trial is a fresh
3739
+ `genml-kit-train` subprocess**, and the only contracts between the tuner and a
3740
+ trial are the command line it is handed and the `<log_dir>/metrics.jsonl` file
3741
+ the JSONL sink already writes. No training internals are imported, so the
3742
+ isolation boundary *is* the process boundary.
3743
+
3744
+ A study is data, not code — describe it in YAML (or all on the command line):
3745
+
3746
+ ```yaml
3747
+ # study.yaml
3748
+ objective: val/macro_f1 # metric tag to optimise
3749
+ direction: max # max | min
3750
+ sampler: tpe # grid | random | tpe | gpei
3751
+ pruner: asha # none | median | asha
3752
+ max_trials: 50
3753
+ max_epochs_per_trial: 20 # budget caps each trial
3754
+ space:
3755
+ opt.lr: log(1e-4:1e-1) # log-uniform float range
3756
+ batch_size: [32, 64, 128] # categorical choice
3757
+ dropout: 0.0:0.5 # uniform float range
3758
+ model.embed_dim: 64:1024 # integer range
3759
+ base_args: ["--pipeline", "images", "--method", "classification"]
3760
+ ```
3761
+
3762
+ ### Running a study
3763
+
3764
+ Points after a literal `--` are forwarded verbatim to every trial:
3765
+
3766
+ <!-- genml-cli-test
3767
+ name: tune-run-study
3768
+ override:
3769
+ - --max_trials 2
3770
+ - --workers 1
3771
+ - -.--model
3772
+ - -.--dataset
3773
+ - -- --model convvit --dataset {tmp}/img --image_size 32 --max_items 16
3774
+ -->
3775
+ ```bash
3776
+ genml-kit-tune run \
3777
+ --space opt.lr=log(1e-4:1e-1) --space batch_size=32,64 \
3778
+ --objective val/macro_f1 --direction max --sampler tpe --pruner asha \
3779
+ --max_trials 8 --workers 2 \
3780
+ --epochs 1 \
3781
+ -- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
3782
+ ```
3783
+
3784
+ The equivalent all-YAML form is `genml-kit-tune run --study study.yaml`; command
3785
+ line flags override the file. Trials land under
3786
+ `runs/study-<timestamp>/trials/<id>/` (each with its own `metrics.jsonl` and
3787
+ checkpoints) and the study ledger is `runs/study-<timestamp>/study.jsonl`.
3788
+
3789
+ ### Samplers and pruners
3790
+
3791
+ | Name | Kind | Notes |
3792
+ |---|---|---|
3793
+ | `grid` | sampler | Deterministic Cartesian product; resume-safe. |
3794
+ | `random` | sampler | Uniform draws over the space. |
3795
+ | `tpe` | sampler | Tree-structured Parzen Estimator — sample-efficient Bayesian optimisation (numpy only). The default. |
3796
+ | `gpei` | sampler | Gaussian-Process Expected Improvement (registered only when `scipy` is installed; `pip install "genml_kit[tune]"`). |
3797
+ | `none` | pruner | Run every trial to its full budget. |
3798
+ | `median` | pruner | Stop trials below the running median at a rung. |
3799
+ | `asha` | pruner | Asynchronous Successive Halving; keeps the top `1/reduction_factor` at each rung. |
3800
+ | `hyperband` | pruner | Multiple ASHA brackets over `min_resource * rf^s` budgets; trials are assigned round-robin so every resource level gets explored. |
3801
+
3802
+ Pruning is delivered as a clean `SIGTERM` to the trial subprocess, which the
3803
+ trainer's existing signal guard turns into a consistent checkpoint before exit.
3804
+
3805
+ ### Concurrency, GPUs and resume
3806
+
3807
+ By default studies run **one trial at a time**. `--workers N` runs `N` trials
3808
+ concurrently, and `--gpus 0,1,2,3` pins each trial to a slot from that pool via
3809
+ Interrupt a study at any time and continue it later (the second command
3810
+ prints the leaderboard of the same study):
3811
+
3812
+ <!-- genml-cli-test
3813
+ name: tune-resume-report
3814
+ override:
3815
+ - --max_trials 2
3816
+ - -.--model
3817
+ - -.--dataset
3818
+ - -- --model convvit --dataset {tmp}/img --image_size 32 --max_items 16
3819
+ -->
3820
+ ```bash
3821
+ # First invocation creates the study; a later one resumes it (the budget
3822
+ # already spent is subtracted, so only the remaining trials run).
3823
+ genml-kit-tune run --study_dir runs/study-20250101-120000 \
3824
+ --space opt.lr=log(1e-4:1e-1) --objective val/macro_f1 --max_trials 50 \
3825
+ -- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
3826
+ genml-kit-tune run --study_dir runs/study-20250101-120000 --resume \
3827
+ --space opt.lr=log(1e-4:1e-1) --objective val/macro_f1 --max_trials 50 \
3828
+ -- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
3829
+ genml-kit-tune report runs/study-20250101-120000 --top 10
3830
+ ```
3831
+
3832
+ ### Extending
3833
+
3834
+ Both algorithm families are registries, exactly like `METHODS`/`PIPELINES`.
3835
+ A new sampler subclasses `Sampler` (implementing `suggest(observed, n)`) and
3836
+ decorates itself with `@SEARCHERS.register`; a new pruner subclasses `Pruner`
3837
+ (implementing `_decide(rung, value)`) and uses `@PRUNERS.register`. The
3838
+ subsystem stays zero-dependency: `tpe`, `grid` and `random` need only numpy,
3839
+ and the scipy-backed `gpei` registers itself only when scipy is importable
3840
+ (`pip install "genml_kit[tune]"`).
3841
+
1963
3842
  ## References And Further Reading
1964
3843
 
1965
3844
  - Xie et al., [SimMIM: A Simple Framework for Masked Image Modeling](https://arxiv.org/abs/2111.09886), CVPR 2022.
@@ -2000,6 +3879,42 @@ instead of scrolling past. Code is formatted with
2000
3879
  (Google style, 2-space indent) and linted with
2001
3880
  [Ruff](https://docs.astral.sh/ruff/) (`ruff check .`).
2002
3881
 
3882
+ ### Verifying README commands
3883
+
3884
+ Command blocks preceded by a `<!-- genml-cli-test ... -->` HTML comment are
3885
+ executed by `tests/test_readme_cli.py`. The comment is invisible on
3886
+ GitHub; its YAML body controls limiting args, network/token markers, and
3887
+ substitutions. Run them with `pytest -m network` (needs internet, no
3888
+ credentials) — the offline suite excludes them by default.
3889
+
3890
+ A fence marked with `launcher: torchrun` is rewritten to a multi-process
3891
+ `torch.distributed.run` invocation and executed that way (run it with
3892
+ `pytest -m distributed`); this is opt-in per fence, so every other example
3893
+ keeps running single-process.
3894
+
3895
+ A fence that talks to an external service names it with `service: <name>`
3896
+ (`hf`, `s3`, `r2`, `gcs`, `github`, `wandb`). Such a fence is run when the
3897
+ service is reachable either through its credentials **or** through an
3898
+ endpoint-override proxy/emulator, because the underlying client libraries
3899
+ honour their own override variables natively:
3900
+
3901
+ | `service:` | credentials | endpoint override |
3902
+ |---|---|---|
3903
+ | `hf` | `HF_TOKEN`, `HUGGING_FACE_HUB_TOKEN` | `HF_ENDPOINT` |
3904
+ | `s3` | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL` |
3905
+ | `r2` | `R2_ACCESS_KEY_ID` + `R2_SECRET_ACCESS_KEY` | `R2_ENDPOINT_URL`, `AWS_ENDPOINT_URL_S3` |
3906
+ | `gcs` | `GOOGLE_APPLICATION_CREDENTIALS` | `STORAGE_EMULATOR_HOST` |
3907
+ | `github` | `GITHUB_TOKEN`, `GH_TOKEN` | `GITHUB_API_URL` |
3908
+ | `wandb` | `WANDB_API_KEY` | `WANDB_BASE_URL` |
3909
+
3910
+ Select them with `pytest -m service` (all services) or a per-service marker
3911
+ such as `pytest -m hf`. `service:` implies `network: true`.
3912
+
3913
+ Every `bash`/`sh` block that contains a `genml-kit-*` command must either be
3914
+ marked for execution as above or explicitly opted out with a
3915
+ `<!-- genml-cli-test-exclude: <reason> -->` comment; a test enforces this so a
3916
+ new example cannot be added untested by accident.
3917
+
2003
3918
  ### Naming Conventions For Class Members
2004
3919
 
2005
3920
  - Anything that PyTorch registers — child `nn.Module`, `nn.Parameter`,
@@ -2011,7 +3926,7 @@ instead of scrolling past. Code is formatted with
2011
3926
  leading underscore. A member called from other classes, overridden as
2012
3927
  an extension point, or documented as API stays public.
2013
3928
  - Extension-point methods that *callers* invoke (e.g.
2014
- `PretrainMethod.build_transform`) are public; same-named hooks the
3929
+ `Method.build_transform`) are public; same-named hooks the
2015
3930
  base class *itself* dispatches (e.g. `BaseImageProcessor._build_transform`)
2016
3931
  are private.
2017
3932