genml-kit 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- genml_kit-0.2.0/PKG-INFO +2021 -0
- genml_kit-0.2.0/README.md +1982 -0
- genml_kit-0.2.0/genml_kit/__init__.py +0 -0
- genml_kit-0.2.0/genml_kit/augmentations/__init__.py +7 -0
- {genml_kit-0.1.0/genml_kit/pretrain → genml_kit-0.2.0/genml_kit}/augmentations/dual_view.py +5 -2
- genml_kit-0.2.0/genml_kit/augmentations/multicrop.py +123 -0
- genml_kit-0.2.0/genml_kit/datasets/__init__.py +23 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/datasets/balanced_sampler.py +40 -11
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/datasets/ensemble.py +4 -0
- genml_kit-0.2.0/genml_kit/datasets/factory.py +58 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/datasets/image_folder.py +8 -3
- genml_kit-0.2.0/genml_kit/datasets/replay_buffer.py +394 -0
- genml_kit-0.2.0/genml_kit/datasets/rollout_buffer.py +358 -0
- genml_kit-0.2.0/genml_kit/datasets/transforms.py +60 -0
- genml_kit-0.2.0/genml_kit/datasets/vo_pairs.py +329 -0
- genml_kit-0.2.0/genml_kit/geometry/__init__.py +28 -0
- genml_kit-0.2.0/genml_kit/geometry/similarity.py +190 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/io/checkpointing.py +34 -9
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/io/storage_utils.py +5 -10
- genml_kit-0.2.0/genml_kit/losses/__init__.py +52 -0
- genml_kit-0.2.0/genml_kit/losses/builtin.py +106 -0
- {genml_kit-0.1.0/genml_kit/pretrain → genml_kit-0.2.0/genml_kit}/losses/dino.py +5 -0
- {genml_kit-0.1.0/genml_kit/pretrain → genml_kit-0.2.0/genml_kit}/losses/focal.py +8 -6
- genml_kit-0.2.0/genml_kit/losses/registry.py +103 -0
- genml_kit-0.2.0/genml_kit/losses/rl.py +293 -0
- genml_kit-0.2.0/genml_kit/methods/__init__.py +26 -0
- genml_kit-0.2.0/genml_kit/methods/base.py +295 -0
- genml_kit-0.2.0/genml_kit/methods/byol.py +167 -0
- genml_kit-0.2.0/genml_kit/methods/classification.py +239 -0
- genml_kit-0.2.0/genml_kit/methods/dino.py +256 -0
- {genml_kit-0.1.0/genml_kit/pretrain → genml_kit-0.2.0/genml_kit}/methods/ijepa.py +86 -43
- genml_kit-0.2.0/genml_kit/methods/registry.py +22 -0
- genml_kit-0.2.0/genml_kit/methods/rl_dqn.py +292 -0
- genml_kit-0.2.0/genml_kit/methods/rl_ppo.py +375 -0
- genml_kit-0.2.0/genml_kit/methods/rl_sac.py +510 -0
- genml_kit-0.2.0/genml_kit/methods/rl_utils.py +69 -0
- {genml_kit-0.1.0/genml_kit/pretrain → genml_kit-0.2.0/genml_kit}/methods/simmim.py +74 -30
- genml_kit-0.2.0/genml_kit/methods/supcon.py +91 -0
- genml_kit-0.2.0/genml_kit/methods/vo_pair.py +158 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/__init__.py +12 -10
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/attention_pooling.py +15 -8
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/byol.py +3 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/cls_model_wrapper/loader.py +16 -8
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/cls_model_wrapper/model.py +7 -2
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/contrastive.py +1 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/convvit/loader.py +5 -4
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/convvit/model.py +47 -16
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/dino.py +31 -11
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/encoder_utils.py +10 -4
- genml_kit-0.2.0/genml_kit/models/init.py +45 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/registry.py +74 -64
- genml_kit-0.2.0/genml_kit/models/rl/__init__.py +20 -0
- genml_kit-0.2.0/genml_kit/models/rl/actor_critic.py +456 -0
- genml_kit-0.2.0/genml_kit/models/rl/qnetwork.py +204 -0
- genml_kit-0.2.0/genml_kit/models/rl/sac_critic.py +93 -0
- genml_kit-0.2.0/genml_kit/models/rl/sac_model.py +27 -0
- genml_kit-0.2.0/genml_kit/models/rl/spaces.py +104 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/simmim.py +14 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/timm/loader.py +9 -12
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/timm/model.py +1 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/transformer_utils.py +11 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/uvito/loader.py +4 -3
- genml_kit-0.2.0/genml_kit/models/uvito/model.py +238 -0
- genml_kit-0.2.0/genml_kit/models/vo/__init__.py +25 -0
- genml_kit-0.2.0/genml_kit/models/vo/loader.py +57 -0
- genml_kit-0.2.0/genml_kit/models/vo/processor.py +13 -0
- genml_kit-0.2.0/genml_kit/models/vo/vo_similar.py +165 -0
- genml_kit-0.2.0/genml_kit/pipelines/__init__.py +25 -0
- genml_kit-0.2.0/genml_kit/pipelines/base.py +62 -0
- genml_kit-0.2.0/genml_kit/pipelines/contracts.py +21 -0
- genml_kit-0.2.0/genml_kit/pipelines/images.py +441 -0
- genml_kit-0.2.0/genml_kit/pipelines/registry.py +18 -0
- genml_kit-0.2.0/genml_kit/pipelines/rl.py +628 -0
- genml_kit-0.2.0/genml_kit/pipelines/rl_normalize.py +208 -0
- genml_kit-0.2.0/genml_kit/pipelines/vo_pair.py +136 -0
- genml_kit-0.2.0/genml_kit/training/classifiers/__init__.py +113 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/classifiers/cls_attention.py +8 -3
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/classifiers/mlp.py +6 -4
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/eval.py +26 -8
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/infer.py +11 -7
- genml_kit-0.2.0/genml_kit/training/labels.py +110 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/model_utils.py +10 -5
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/optim_factory.py +55 -6
- genml_kit-0.2.0/genml_kit/training/rl_trainer.py +495 -0
- genml_kit-0.2.0/genml_kit/training/train.py +493 -0
- genml_kit-0.2.0/genml_kit/training/train_compat.py +161 -0
- genml_kit-0.2.0/genml_kit/training/train_reporting.py +447 -0
- genml_kit-0.2.0/genml_kit/training/trainer.py +318 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/tta.py +2 -1
- genml_kit-0.2.0/genml_kit/training/video_utils.py +98 -0
- genml_kit-0.2.0/genml_kit/training/vo/__init__.py +20 -0
- genml_kit-0.2.0/genml_kit/training/vo/eval_vo.py +108 -0
- genml_kit-0.2.0/genml_kit/training/vo/train_vo.py +156 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/args.py +62 -14
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/cli.py +12 -10
- genml_kit-0.2.0/genml_kit/utils/env.py +47 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/gpu.py +15 -3
- genml_kit-0.2.0/genml_kit/utils/registry.py +107 -0
- genml_kit-0.2.0/genml_kit/utils/seed.py +119 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/signal.py +21 -0
- genml_kit-0.2.0/genml_kit.egg-info/PKG-INFO +2021 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit.egg-info/SOURCES.txt +94 -19
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit.egg-info/entry_points.txt +0 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit.egg-info/requires.txt +6 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/pyproject.toml +13 -6
- genml_kit-0.2.0/tests/test_actor_critic.py +217 -0
- genml_kit-0.2.0/tests/test_args_frozen.py +77 -0
- genml_kit-0.2.0/tests/test_attention_pooling.py +37 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_attr_utils.py +2 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_balanced_sampler.py +36 -4
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_byol.py +40 -30
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_checkpointing.py +101 -3
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_classifiers.py +15 -16
- genml_kit-0.2.0/tests/test_cli_help.py +49 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_contrastive_models.py +2 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_custom_models.py +152 -13
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_datasets_ensemble.py +32 -2
- genml_kit-0.2.0/tests/test_dino.py +307 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_encoder_utils.py +54 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_ensemble_collate.py +36 -19
- genml_kit-0.2.0/tests/test_env.py +49 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_evaluation_metrics.py +8 -10
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_extract_features.py +6 -3
- genml_kit-0.2.0/tests/test_geometry_similarity.py +301 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_grad_monitor.py +8 -4
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_headless_models.py +11 -6
- genml_kit-0.2.0/tests/test_init_orthogonal.py +101 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_log_format.py +9 -9
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_logging_utils.py +2 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_lora.py +2 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_loss_and_multipliers.py +2 -1
- genml_kit-0.2.0/tests/test_loss_registry.py +127 -0
- genml_kit-0.2.0/tests/test_method_lifecycle.py +334 -0
- genml_kit-0.2.0/tests/test_model_script_loading.py +94 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_optim_factory.py +26 -3
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_param_align.py +8 -4
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_pipeline.py +11 -12
- genml_kit-0.2.0/tests/test_pipeline_method_registry.py +413 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_pretrain_methods.py +15 -9
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_pretrain_smoke.py +67 -114
- genml_kit-0.2.0/tests/test_pretrain_vis.py +127 -0
- genml_kit-0.2.0/tests/test_replay_buffer.py +217 -0
- genml_kit-0.2.0/tests/test_rl_cli.py +149 -0
- genml_kit-0.2.0/tests/test_rl_learning.py +238 -0
- genml_kit-0.2.0/tests/test_rl_losses.py +154 -0
- genml_kit-0.2.0/tests/test_rl_losses_phase2.py +230 -0
- genml_kit-0.2.0/tests/test_rl_method.py +177 -0
- genml_kit-0.2.0/tests/test_rl_model_selector.py +232 -0
- genml_kit-0.2.0/tests/test_rl_models.py +148 -0
- genml_kit-0.2.0/tests/test_rl_normalize.py +105 -0
- genml_kit-0.2.0/tests/test_rl_pipeline.py +270 -0
- genml_kit-0.2.0/tests/test_rl_ppo.py +341 -0
- genml_kit-0.2.0/tests/test_rl_sac.py +241 -0
- genml_kit-0.2.0/tests/test_rl_spaces.py +79 -0
- genml_kit-0.2.0/tests/test_rl_trainer.py +1302 -0
- genml_kit-0.2.0/tests/test_rl_video.py +337 -0
- genml_kit-0.2.0/tests/test_rollout_buffer.py +143 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_seed_utils.py +66 -21
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_set_train_mode.py +2 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_signal_utils.py +38 -6
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_supcon_loss.py +19 -10
- genml_kit-0.2.0/tests/test_supcon_method.py +114 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_table_utils.py +6 -3
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_timm_models.py +2 -1
- genml_kit-0.2.0/tests/test_train_reporting.py +416 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_train_smoke.py +19 -7
- genml_kit-0.2.0/tests/test_trainer.py +278 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_transformer_encoder_init.py +1 -1
- genml_kit-0.2.0/tests/test_utils_registry.py +151 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_uvito.py +64 -1
- genml_kit-0.2.0/tests/test_vo_e2e.py +73 -0
- genml_kit-0.2.0/tests/test_vo_pairs.py +153 -0
- genml_kit-0.2.0/tests/test_vo_similar.py +124 -0
- genml_kit-0.2.0/tests/test_vo_training.py +113 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_xgb_pipeline.py +2 -1
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_xgb_utils.py +6 -3
- genml_kit-0.1.0/PKG-INFO +0 -1209
- genml_kit-0.1.0/README.md +0 -1174
- genml_kit-0.1.0/genml_kit/__init__.py +0 -11
- genml_kit-0.1.0/genml_kit/datasets/__init__.py +0 -11
- genml_kit-0.1.0/genml_kit/models/uvito/model.py +0 -150
- genml_kit-0.1.0/genml_kit/pretrain/__init__.py +0 -1
- genml_kit-0.1.0/genml_kit/pretrain/augmentations/__init__.py +0 -1
- genml_kit-0.1.0/genml_kit/pretrain/augmentations/multicrop.py +0 -105
- genml_kit-0.1.0/genml_kit/pretrain/cli.py +0 -884
- genml_kit-0.1.0/genml_kit/pretrain/losses/__init__.py +0 -8
- genml_kit-0.1.0/genml_kit/pretrain/methods/__init__.py +0 -11
- genml_kit-0.1.0/genml_kit/pretrain/methods/base.py +0 -100
- genml_kit-0.1.0/genml_kit/pretrain/methods/byol.py +0 -105
- genml_kit-0.1.0/genml_kit/pretrain/methods/dino.py +0 -179
- genml_kit-0.1.0/genml_kit/pretrain/methods/registry.py +0 -33
- genml_kit-0.1.0/genml_kit/pretrain/methods/supcon.py +0 -72
- genml_kit-0.1.0/genml_kit/training/classifiers/__init__.py +0 -80
- genml_kit-0.1.0/genml_kit/training/train.py +0 -1381
- genml_kit-0.1.0/genml_kit/training/train_reporting.py +0 -197
- genml_kit-0.1.0/genml_kit/utils/seed.py +0 -86
- genml_kit-0.1.0/genml_kit.egg-info/PKG-INFO +0 -1209
- genml_kit-0.1.0/tests/test_dino.py +0 -194
- genml_kit-0.1.0/tests/test_pretrain_vis.py +0 -188
- genml_kit-0.1.0/tests/test_restructure.py +0 -87
- genml_kit-0.1.0/tests/test_supcon_method.py +0 -100
- genml_kit-0.1.0/tests/test_train_reporting.py +0 -289
- {genml_kit-0.1.0 → genml_kit-0.2.0}/LICENSE +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/datasets/field_dataset.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/datasets/hf_proxy.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/datasets/retry.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/datasets/weighted_sampler.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/io/__init__.py +0 -0
- {genml_kit-0.1.0/genml_kit/pretrain → genml_kit-0.2.0/genml_kit}/losses/byol.py +0 -0
- {genml_kit-0.1.0/genml_kit/pretrain → genml_kit-0.2.0/genml_kit}/losses/contrastive.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/cls_model_wrapper/__init__.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/convvit/__init__.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/convvit/masked_encoder.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/convvit/processor.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/masked_encoder.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/processors/__init__.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/processors/base.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/timm/__init__.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/timm/processor.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/uvito/__init__.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/models/uvito/processor.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/__init__.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/classifiers/base.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/grad_monitor.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/metrics.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/param_align.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/xgb_pipeline.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/training/xgb_utils.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/__init__.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/attr.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/image_dump.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/label.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/logging.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/script.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/table.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit/utils/transformer.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit.egg-info/dependency_links.txt +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/genml_kit.egg-info/top_level.txt +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/setup.cfg +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_cli_utils.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_ensemble_ensure_label_space.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_ensemble_labels.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_ensemble_mixed_labels.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_image_dump.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_infer_topk.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_metrics.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_retry.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_s3r2util.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_script_utils.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_simmim.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_storage_utils.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_tta.py +0 -0
- {genml_kit-0.1.0 → genml_kit-0.2.0}/tests/test_weighted_sampler.py +0 -0
genml_kit-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,2021 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: genml_kit
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: General-purpose ML toolkit: image-classification, visual-odometry and self-supervised pre-training through one unified harness.
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: numpy
|
|
10
|
+
Requires-Dist: torch
|
|
11
|
+
Requires-Dist: torchvision
|
|
12
|
+
Requires-Dist: transformers
|
|
13
|
+
Requires-Dist: datasets
|
|
14
|
+
Requires-Dist: xgboost>=2.0
|
|
15
|
+
Requires-Dist: scikit-learn
|
|
16
|
+
Requires-Dist: pillow
|
|
17
|
+
Requires-Dist: tensorboard
|
|
18
|
+
Provides-Extra: gcs
|
|
19
|
+
Requires-Dist: google-cloud-storage; extra == "gcs"
|
|
20
|
+
Provides-Extra: s3
|
|
21
|
+
Requires-Dist: boto3>=1.28; extra == "s3"
|
|
22
|
+
Provides-Extra: lora
|
|
23
|
+
Requires-Dist: peft>=0.7.0; extra == "lora"
|
|
24
|
+
Provides-Extra: timm
|
|
25
|
+
Requires-Dist: timm>=1.0; extra == "timm"
|
|
26
|
+
Provides-Extra: uvito
|
|
27
|
+
Requires-Dist: segmentation_models_pytorch>=0.3.0; extra == "uvito"
|
|
28
|
+
Provides-Extra: rl
|
|
29
|
+
Requires-Dist: gymnasium>=0.29; extra == "rl"
|
|
30
|
+
Requires-Dist: pygame>=2.1; extra == "rl"
|
|
31
|
+
Requires-Dist: imageio[ffmpeg]>=2.31; extra == "rl"
|
|
32
|
+
Provides-Extra: all
|
|
33
|
+
Requires-Dist: genml_kit[gcs,lora,rl,s3,timm,uvito]; extra == "all"
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: pytest; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff; extra == "dev"
|
|
37
|
+
Requires-Dist: yapf; extra == "dev"
|
|
38
|
+
Dynamic: license-file
|
|
39
|
+
|
|
40
|
+
# genml_kit
|
|
41
|
+
|
|
42
|
+
[](https://github.com/davidel/genml_kit/actions/workflows/ci.yml)
|
|
43
|
+
|
|
44
|
+
A general-purpose toolkit for image classification. Supports self-supervised
|
|
45
|
+
pre-training, supervised fine-tuning, and XGBoost ensemble inference — all
|
|
46
|
+
from the command line.
|
|
47
|
+
|
|
48
|
+
`genml_kit` is domain-agnostic: point it at any HuggingFace dataset, local
|
|
49
|
+
`ImageFolder` tree, or timm/HuggingFace backbone. It grew out of a
|
|
50
|
+
skin-lesion classification project; the dermoscopy-specific workflow (dataset
|
|
51
|
+
preparation, tuned recipes) is documented separately in
|
|
52
|
+
[`scdiag/README.md`](scdiag/README.md). The drone visual-odometry front-end
|
|
53
|
+
(similarity estimation between oblique aerial frames) is documented separately
|
|
54
|
+
in [`vo/README.md`](vo/README.md).
|
|
55
|
+
|
|
56
|
+
## Contents
|
|
57
|
+
|
|
58
|
+
- [Why genml_kit?](#why-genml_kit)
|
|
59
|
+
- [How It Works](#how-it-works)
|
|
60
|
+
- [The Method Lifecycle](#the-method-lifecycle)
|
|
61
|
+
- [Installation](#installation)
|
|
62
|
+
- [Quick Start](#quick-start)
|
|
63
|
+
- [Pre-Training Guide](#pre-training-guide)
|
|
64
|
+
- [Fine-Tuning Guide](#fine-tuning-guide)
|
|
65
|
+
- [Inference Guide](#inference-guide)
|
|
66
|
+
- [Tips & Pitfalls](#tips--pitfalls)
|
|
67
|
+
- [Gradient Monitor](#gradient-monitor)
|
|
68
|
+
- [Custom Losses](#custom-losses)
|
|
69
|
+
- [Custom Models](#custom-models)
|
|
70
|
+
- [References And Further Reading](#references-and-further-reading)
|
|
71
|
+
- [Development](#development)
|
|
72
|
+
- [License](#license)
|
|
73
|
+
|
|
74
|
+
## Why genml_kit?
|
|
75
|
+
|
|
76
|
+
Practitioners face two recurring problems: **labeled data is scarce** and
|
|
77
|
+
**off-the-shelf models are not domain-specific**. A ViT pre-trained on
|
|
78
|
+
ImageNet can classify cats and dogs, but specialized imagery — medical,
|
|
79
|
+
satellite, industrial, scientific — looks nothing like natural photos, and
|
|
80
|
+
the feature distributions are fundamentally different.
|
|
81
|
+
|
|
82
|
+
genml_kit solves this with a two-stage pipeline:
|
|
83
|
+
|
|
84
|
+
1. **Pre-train** on large, often unlabeled image collections (or a labeled
|
|
85
|
+
superset) to learn domain-appropriate visual features.
|
|
86
|
+
2. **Fine-tune** on your smaller labeled dataset, starting from those
|
|
87
|
+
pre-trained features instead of random initialization.
|
|
88
|
+
|
|
89
|
+
This consistently outperforms training from scratch, especially when your
|
|
90
|
+
labeled dataset has fewer than ~5 000 images. The tool also supports ensemble
|
|
91
|
+
inference with XGBoost on top of the learned features, which can squeeze out
|
|
92
|
+
additional performance for deployment.
|
|
93
|
+
|
|
94
|
+
> **Concrete example:** the dermoscopy workflow this toolkit was extracted
|
|
95
|
+
> from — preparing HAM10000 / ISIC / Derm1M corpora, per-method
|
|
96
|
+
> hyperparameters, and a worked pre-train → fine-tune run — is documented in
|
|
97
|
+
> [`scdiag/README.md`](scdiag/README.md).
|
|
98
|
+
|
|
99
|
+
## How It Works
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
103
|
+
│ Pre-Training │
|
|
104
|
+
│ Unlabeled/labeled images │
|
|
105
|
+
│ ──────────────────────────────────► Encoder with learned │
|
|
106
|
+
│ SimMIM / I-JEPA / SupCon visual features │
|
|
107
|
+
└────────────────────────────┬────────────────────────────────────┘
|
|
108
|
+
│ encoder weights
|
|
109
|
+
▼
|
|
110
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
111
|
+
│ Fine-Tuning │
|
|
112
|
+
│ Small labeled dataset ──────────► Trained classifier │
|
|
113
|
+
│ + pre-trained encoder for your task │
|
|
114
|
+
└────────────────────────────┬────────────────────────────────────┘
|
|
115
|
+
│ backbone features
|
|
116
|
+
▼
|
|
117
|
+
┌─────────────────────────────────────────────────────────────────┐
|
|
118
|
+
│ (Optional) Ensemble │
|
|
119
|
+
│ Backbone features ──────────► XGBoost on top of the │
|
|
120
|
+
│ learned representations │
|
|
121
|
+
└─────────────────────────────────────────────────────────────────┘
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
**Pre-training** teaches the model to understand the target imagery — textures,
|
|
125
|
+
boundaries, colour patterns, and spatial relationships. **Fine-tuning** adapts
|
|
126
|
+
that understanding to your specific classification task (e.g. disease vs.
|
|
127
|
+
healthy, defective vs. passing). **Ensemble inference** (optional) trains a
|
|
128
|
+
tree-based model on the same features, which sometimes generalises better
|
|
129
|
+
than a linear head for small datasets.
|
|
130
|
+
|
|
131
|
+
### A Useful Mental Model
|
|
132
|
+
|
|
133
|
+
The encoder turns an image into a vector of features. During pre-training, we
|
|
134
|
+
choose an artificial task whose answer can be obtained from the images (or,
|
|
135
|
+
for SupCon, from their labels). The encoder learns parameters $`\theta`$ that
|
|
136
|
+
make this task easy. During fine-tuning, a classifier is attached to the
|
|
137
|
+
encoder and the whole model, or a selected part of it, is adapted to the real
|
|
138
|
+
labels:
|
|
139
|
+
|
|
140
|
+
```text
|
|
141
|
+
image x ──► encoder f_theta(x) ──► classifier g_phi ──► class probabilities
|
|
142
|
+
│ │
|
|
143
|
+
reusable features task-specific boundary
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Here $`x`$ is the input image, $`f_\theta(x)`$ the feature vector computed by
|
|
147
|
+
the encoder with parameters $`\theta`$, and $`g_\phi`$ the classifier head with
|
|
148
|
+
parameters $`\phi`$.
|
|
149
|
+
|
|
150
|
+
Pre-training and fine-tuning are not two names for the same job. Pre-training
|
|
151
|
+
shapes a useful representation; fine-tuning decides how that representation
|
|
152
|
+
should be used for the target labels. If the pre-training data is visually
|
|
153
|
+
related to the target data, this gives the classifier a much better starting
|
|
154
|
+
point than random initialization. If the domains are very different, use a
|
|
155
|
+
smaller learning rate for the backbone and validate carefully.
|
|
156
|
+
|
|
157
|
+
The pre-training methods are introduced above with links to their papers (see
|
|
158
|
+
the [Pre-Training Guide](#pre-training-guide)). The
|
|
159
|
+
[timm documentation](https://huggingface.co/docs/timm/index) and the
|
|
160
|
+
[Hugging Face image classification guide](https://huggingface.co/docs/transformers/tasks/image_classification)
|
|
161
|
+
are useful references when selecting a backbone or processor.
|
|
162
|
+
|
|
163
|
+
### The Method Lifecycle
|
|
164
|
+
|
|
165
|
+
`genml-kit-train` is a *branchless* driver: it never inspects which
|
|
166
|
+
`--method` or `--pipeline` it is running. Instead, every objective registers
|
|
167
|
+
itself through the `Method` registry and declares the same four lifecycle
|
|
168
|
+
hooks, which the driver calls in one fixed order:
|
|
169
|
+
|
|
170
|
+
| # | Hook | Purpose |
|
|
171
|
+
|---|---|---|
|
|
172
|
+
| 1 | `prepare_transforms(args, device)` | Resolve any transforms the method needs (e.g. the HF processor's normalization) **before** the loaders are built. |
|
|
173
|
+
| 2 | `wire_data(args, pipeline)` | Consume pipeline-built data attributes: label space, class weights, criterion. Runs **before** the model so the HF head can be sized from the label space. |
|
|
174
|
+
| 3 | `build_model(args, device)` | Construct the model. Every method applies LoRA (`--lora`), source-checkpoint weights (`--source_checkpoint`), freeze patterns (`--freeze`) and gradient checkpointing (`--grad_checkpoint`) through the same `_apply_model_extras` step — so all four flags work for **every** objective, not just classification. |
|
|
175
|
+
| 4 | `post_train(args, pipeline, device, result)` | Optional post-training stage, called after `BaseTrainer.run()` returns (skipped when the run was interrupted). |
|
|
176
|
+
|
|
177
|
+
Two ordering invariants are enforced by the hook order itself, not by
|
|
178
|
+
convention: transforms resolve before loaders consume them, and the label
|
|
179
|
+
space exists before the classification head is constructed.
|
|
180
|
+
|
|
181
|
+
Adding a new objective means implementing the hooks you need (only
|
|
182
|
+
`build_model` and `train_step` are abstract); the driver never changes.
|
|
183
|
+
`ClassificationMethod` additionally implements `post_train` to run the
|
|
184
|
+
optional XGBoost shallow-head stage; any future classifier-style method can
|
|
185
|
+
reuse the same hook.
|
|
186
|
+
|
|
187
|
+
## Package Layout
|
|
188
|
+
|
|
189
|
+
| Module | Contents |
|
|
190
|
+
|---|---|
|
|
191
|
+
| `genml_kit.training` | `train.py` and `infer.py` CLI harnesses, `optim_factory.py` (optimizers, LLRD, schedulers), `model_utils.py` (loading, freezing, feature extraction), `param_align.py`, `eval.py` + `metrics.py`, `grad_monitor.py`, `train_reporting.py`, `tta.py`, `xgb_utils.py` + `xgb_pipeline.py`, `classifiers/` (pluggable heads) |
|
|
192
|
+
| `genml_kit.methods` | objective-side registry (classification, vo_pair, SimMIM, I-JEPA, DINO, BYOL, SupCon) |
|
|
193
|
+
| `genml_kit.losses` | shared loss library (focal, NT-Xent contrastive, DINO self-distillation, BYOL) |
|
|
194
|
+
| `genml_kit.augmentations` | objective-level transforms (dual-view, multi-crop) |
|
|
195
|
+
| `genml_kit.models` | model/processor registry; `timm/`, `convvit/`, `uvito/`, `cls_model_wrapper/` backends; `processors/base.py` |
|
|
196
|
+
| `genml_kit.datasets` | `hf_proxy.py` (HuggingFace → PyTorch bridge), `image_folder.py`, `ensemble.py`, `field_dataset.py`, `balanced_sampler.py`, `weighted_sampler.py`, `retry.py`, `vo_pairs.py` (synthetic VO pairs) |
|
|
197
|
+
| `genml_kit.geometry` | `similarity.py` (2-D similarity algebra, batched differentiable Umeyama, backward-map warp) — the math behind the visual-odometry front-end lives in [`vo/README.md`](vo/README.md) |
|
|
198
|
+
| `genml_kit.io` | `checkpointing.py` (atomic saves, LoRA state, remote fetch), `storage_utils.py` (S3 / GCS / R2) |
|
|
199
|
+
| `genml_kit.utils` | glog-style logging, CLI arg groups, seeding, signal handling, GPU info, tables, external `.py` script loading, `image_dump`, transformer init helpers |
|
|
200
|
+
|
|
201
|
+
## Installation
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
pip install genml_kit
|
|
205
|
+
|
|
206
|
+
# With timm model support:
|
|
207
|
+
pip install "genml_kit[timm]"
|
|
208
|
+
|
|
209
|
+
# With GCS checkpoint sync:
|
|
210
|
+
pip install "genml_kit[gcs]"
|
|
211
|
+
|
|
212
|
+
# With AWS S3 / Cloudflare R2 checkpoint sync (both use boto3):
|
|
213
|
+
pip install "genml_kit[s3]"
|
|
214
|
+
|
|
215
|
+
# With LoRA fine-tuning:
|
|
216
|
+
pip install "genml_kit[lora]"
|
|
217
|
+
|
|
218
|
+
# With UVito model support:
|
|
219
|
+
pip install "genml_kit[uvito]"
|
|
220
|
+
|
|
221
|
+
# Everything above in one shot (gcs, s3, lora, timm, uvito):
|
|
222
|
+
pip install "genml_kit[all]"
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
**Requirements:** Python ≥ 3.9, PyTorch, torchvision, transformers, datasets,
|
|
226
|
+
NumPy, scikit-learn ≥ 1.3, XGBoost ≥ 2.0, Pillow, tensorboard.
|
|
227
|
+
|
|
228
|
+
## Quick Start
|
|
229
|
+
|
|
230
|
+
The fastest way to get started:
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
# Fine-tune a ViT on an image-classification dataset (5 epochs, ~2 minutes on GPU)
|
|
234
|
+
genml-kit-train --model google/vit-base-patch16-224 \
|
|
235
|
+
--dataset cifar10 \
|
|
236
|
+
--label_column label \
|
|
237
|
+
--epochs 5 \
|
|
238
|
+
--batch_size 32 \
|
|
239
|
+
--lr 3e-5 \
|
|
240
|
+
--image_size 224
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
HuggingFace ViT backbones use fixed 224x224 position embeddings and do not
|
|
244
|
+
interpolate them, so they require `--image_size 224`. The toolkit default of
|
|
245
|
+
448 targets size-flexible backbones such as ConvNeXt, timm EVA, or ConvViT,
|
|
246
|
+
which pool or interpolate the token sequence to the input size.
|
|
247
|
+
|
|
248
|
+
This downloads the model and dataset from HuggingFace, trains for 5 epochs,
|
|
249
|
+
and saves `genml_kit_latest.pt` and `genml_kit_best.pt`. See
|
|
250
|
+
[Pre-Training Guide](#pre-training-guide) below for the full pipeline
|
|
251
|
+
(starting with pre-training before fine-tuning).
|
|
252
|
+
|
|
253
|
+
---
|
|
254
|
+
|
|
255
|
+
## Pre-Training Guide
|
|
256
|
+
|
|
257
|
+
Pre-training learns general visual features from large datasets *before* you
|
|
258
|
+
fine-tune on your specific task. This is especially valuable wherever labeled
|
|
259
|
+
data is expensive to obtain but raw images are available in bulk — medical
|
|
260
|
+
imaging, remote sensing, industrial inspection, scientific imaging.
|
|
261
|
+
|
|
262
|
+
genml_kit supports five pre-training methods, each with different strengths:
|
|
263
|
+
|
|
264
|
+
### Choosing Your Method
|
|
265
|
+
|
|
266
|
+
| Method | Needs labels? | Best when… | Key idea |
|
|
267
|
+
|---|---|---|---|
|
|
268
|
+
| **SimMIM** | No | You have large unlabeled datasets; want a simple, proven approach | Mask 60% of image patches, train the model to reconstruct the raw pixels |
|
|
269
|
+
| **I-JEPA** | No | You want faster training and better downstream transfer than SimMIM | Predict *representations* of masked regions, not raw pixels — avoids learning noise |
|
|
270
|
+
| **SupCon** | Yes | You have labels and want representations that cluster by class | Pull same-class images together, push different classes apart in feature space |
|
|
271
|
+
| **DINO** | No | You want strong features from a modest amount of data; proven, robust default | Self-distillation: a student matches the output distribution of its own EMA copy (the teacher) |
|
|
272
|
+
| **BYOL** | No | Similar goals to DINO with a simpler pipeline (two views, no multi-crop) | Online network predicts the output of its own EMA target network |
|
|
273
|
+
|
|
274
|
+
### How Each Method Works
|
|
275
|
+
|
|
276
|
+
The methods differ mainly in what they call a correct answer. SimMIM asks for
|
|
277
|
+
pixels, I-JEPA asks for features, DINO and BYOL ask a network to predict the
|
|
278
|
+
output of its own slowly-moving copy, and SupCon asks for relative positions in
|
|
279
|
+
feature space. That distinction matters: pixel reconstruction can spend effort
|
|
280
|
+
reproducing colour and high-frequency detail, while the other methods spend
|
|
281
|
+
effort building invariant, separable representations.
|
|
282
|
+
|
|
283
|
+
**SimMIM** (Masked Image Modelling): Randomly masks ~60% of image patches and
|
|
284
|
+
trains a lightweight decoder to reconstruct the original pixels. The encoder
|
|
285
|
+
must learn to understand textures, boundaries, and spatial context from just
|
|
286
|
+
40% of the image. Think of it as a "fill in the blanks" exercise for vision
|
|
287
|
+
models. Good default choice when you have lots of unlabeled images.
|
|
288
|
+
Reference: Xie et al., [SimMIM: A Simple Framework for Masked Image Modeling](https://arxiv.org/abs/2111.09886), CVPR 2022.
|
|
289
|
+
|
|
290
|
+
For an image split into patches $`x_1, \dots, x_N`$, let $`\mathcal{M}`$ be
|
|
291
|
+
the set of masked patch indices and $`\hat{x}_i`$ the decoder prediction.
|
|
292
|
+
SimMIM minimizes the mean squared reconstruction error over masked patches:
|
|
293
|
+
|
|
294
|
+
$$
|
|
295
|
+
\large
|
|
296
|
+
\mathcal{L}_{\text{MIM}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\| \hat{x}_i - x_i \right\|_2^2
|
|
297
|
+
$$
|
|
298
|
+
|
|
299
|
+
where:
|
|
300
|
+
|
|
301
|
+
- $`x_i`$ — the pixels of the $`i`$-th original patch;
|
|
302
|
+
- $`\hat{x}_i`$ — the decoder's reconstruction of that patch;
|
|
303
|
+
- $`\mathcal{M}`$ — the set of masked patch indices, and $`|\mathcal{M}|`$ its
|
|
304
|
+
size (how many patches are masked);
|
|
305
|
+
- $`\lVert \cdot \rVert_2^2`$ — the squared Euclidean ($`L_2`$) norm, i.e. the
|
|
306
|
+
sum of squared pixel differences over the patch.
|
|
307
|
+
|
|
308
|
+
Only masked patches contribute to the loss; otherwise copying visible pixels
|
|
309
|
+
would make the task too easy. A higher
|
|
310
|
+
`--mask_ratio` supplies less context and creates a harder task, but an
|
|
311
|
+
excessively high ratio can make reconstruction ambiguous.
|
|
312
|
+
|
|
313
|
+
**I-JEPA** (Joint-Embedding Predictive Architecture): Also masks patches, but
|
|
314
|
+
instead of reconstructing pixels, it predicts the *latent representation* of
|
|
315
|
+
the masked region from the visible context. This avoids wasting capacity on
|
|
316
|
+
pixel-level noise (e.g. exact JPEG compression artifacts) and learns more
|
|
317
|
+
transferable features. Uses a teacher–student setup with EMA momentum ramping.
|
|
318
|
+
Reference: Assran et al., [Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture](https://arxiv.org/abs/2301.08243), CVPR 2023.
|
|
319
|
+
|
|
320
|
+
Let $`f_\theta`$ be the student encoder and $`f_\xi`$ the teacher encoder;
|
|
321
|
+
$`\theta`$ and $`\xi`$ are their respective parameter vectors (the subscripts
|
|
322
|
+
name the network a symbol belongs to). The predictor $`q_\theta`$ receives the
|
|
323
|
+
visible context and predicts a target representation $`z_j = f_\xi(x_j)`$ for a
|
|
324
|
+
masked region. The objective is regression in representation space:
|
|
325
|
+
|
|
326
|
+
$$
|
|
327
|
+
\large
|
|
328
|
+
\mathcal{L}_{\text{IJEPA}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\| q_\theta(f_\theta(\text{context}))_i - \mathop{\mathrm{stopgrad}}\left(f_\xi(x_i)\right) \right\|_2^2
|
|
329
|
+
$$
|
|
330
|
+
|
|
331
|
+
where:
|
|
332
|
+
|
|
333
|
+
- $`x_i`$ — the $`i`$-th masked patch (the prediction target);
|
|
334
|
+
- $`f_\theta(\text{context})`$ — the student's encoding of the *visible* patches;
|
|
335
|
+
- $`q_\theta(\cdot)_i`$ — the predictor's output for the $`i`$-th masked location;
|
|
336
|
+
- $`f_\xi(x_i)`$ — the teacher's representation of the masked patch;
|
|
337
|
+
- $`\mathrm{stopgrad}(\cdot)`$ — treat the argument as a constant: gradients do
|
|
338
|
+
not flow through it during the student update;
|
|
339
|
+
- $`\lVert \cdot \rVert_2^2`$ — squared Euclidean distance in representation space;
|
|
340
|
+
- $`\mathcal{M}`$ — the set of masked patch indices.
|
|
341
|
+
|
|
342
|
+
The teacher is not optimized by backpropagation; it follows the student with
|
|
343
|
+
an exponential moving average:
|
|
344
|
+
|
|
345
|
+
$$
|
|
346
|
+
\large
|
|
347
|
+
\xi \leftarrow m \, \xi + (1 - m) \, \theta
|
|
348
|
+
$$
|
|
349
|
+
|
|
350
|
+
where $`m`$ is the momentum (`--teacher_momentum`) and the arrow denotes
|
|
351
|
+
assignment: the new teacher parameters are a convex blend of the old teacher
|
|
352
|
+
and the current student. A high $`m`$ changes the teacher slowly and gives
|
|
353
|
+
more stable targets. The asymmetric teacher update and masking are important:
|
|
354
|
+
two networks simply trained to copy each other could collapse to a constant
|
|
355
|
+
vector (the teacher-follows-student trick and $`\mathrm{stopgrad}`$ come from
|
|
356
|
+
Grill et al., [BYOL](https://arxiv.org/abs/2006.07733), NeurIPS 2020).
|
|
357
|
+
|
|
358
|
+
Three mechanism details complete the picture:
|
|
359
|
+
|
|
360
|
+
- **Block masking.** Target regions are not scattered single patches: the mask
|
|
361
|
+
is composed of a few contiguous rectangular blocks (4 blocks of 6×6 patches
|
|
362
|
+
in the current implementation). Contiguous holes are much harder to fill in
|
|
363
|
+
from context than isolated ones, so the task rewards spatial understanding
|
|
364
|
+
rather than high-frequency texture interpolation.
|
|
365
|
+
- **Two crop scales.** The *context* (source) view is a small random crop (50%
|
|
366
|
+
of the image side) and the *target* view is a larger one (85%). The predictor
|
|
367
|
+
must infer representations of a region it sees only through a narrow window,
|
|
368
|
+
which encourages semantic rather than layout-memorized features.
|
|
369
|
+
- **Momentum ramp.** $`m`$ moves from `--teacher_momentum` (0.996) toward
|
|
370
|
+
`--teacher_final_momentum` (1.0) as training progresses: as the encoder
|
|
371
|
+
matures, the teacher freezes progressively, stabilizing the targets late in
|
|
372
|
+
training.
|
|
373
|
+
|
|
374
|
+
**SupCon** (Supervised Contrastive Learning): Uses labels to define "positive"
|
|
375
|
+
pairs (same class) and "negative" pairs (different classes). The loss pulls
|
|
376
|
+
features of same-class images together and pushes different-class features
|
|
377
|
+
apart. Produces a feature space where similar images naturally cluster.
|
|
378
|
+
Requires a `ContrastiveEncoder` (backbone + projection head) and balanced
|
|
379
|
+
batch sampling to ensure each batch has enough same-class pairs.
|
|
380
|
+
Reference: Khosla et al., [Supervised Contrastive Learning](https://arxiv.org/abs/2004.11362), NeurIPS 2020.
|
|
381
|
+
|
|
382
|
+
An image $`x_i`$ is mapped by the encoder-and-projection-head
|
|
383
|
+
$`f_\theta`$ to a vector $`f_\theta(x_i)`$, which is rescaled to unit length
|
|
384
|
+
(normalized) so that comparisons are scale-invariant:
|
|
385
|
+
|
|
386
|
+
$$
|
|
387
|
+
\large
|
|
388
|
+
z_i = \frac{f_\theta(x_i)}{\lVert f_\theta(x_i) \rVert_2}
|
|
389
|
+
$$
|
|
390
|
+
|
|
391
|
+
where $`\lVert v \rVert_2 = \sqrt{\sum_d v_d^2}`$ is the Euclidean ($`L_2`$)
|
|
392
|
+
length of a vector $`v`$, so every $`z_i`$ lies on the unit hypersphere.
|
|
393
|
+
The similarity of two examples $`i`$ and $`j`$ is then the dot product of
|
|
394
|
+
their normalized projections, divided by the temperature $`\tau`$
|
|
395
|
+
(`--temperature`):
|
|
396
|
+
|
|
397
|
+
$$
|
|
398
|
+
\large
|
|
399
|
+
s_{ij} = \frac{z_i^\top z_j}{\tau}
|
|
400
|
+
$$
|
|
401
|
+
|
|
402
|
+
where $`z_i^\top z_j`$ is the dot product (cosine of the angle between the two
|
|
403
|
+
unit vectors) and $`\tau`$ rescales the similarities before the softmax.
|
|
404
|
+
|
|
405
|
+
Fix one example $`i`$, called the *anchor*. Its *positive set* is
|
|
406
|
+
|
|
407
|
+
$$
|
|
408
|
+
\large
|
|
409
|
+
\mathcal{P}(i) = \{\, j : j \neq i \text{ and } y_j = y_i \,\}
|
|
410
|
+
$$
|
|
411
|
+
|
|
412
|
+
i.e. all other examples in the batch that share the anchor's class label
|
|
413
|
+
$`y_i`$; every remaining example is a *negative*. SupCon averages the
|
|
414
|
+
log-softmax probability that the anchor assigns to each of its positives:
|
|
415
|
+
|
|
416
|
+
$$
|
|
417
|
+
\large
|
|
418
|
+
\mathcal{L}_i = -\frac{1}{|\mathcal{P}(i)|} \sum_{p \in \mathcal{P}(i)} \log \left( \frac{\exp(s_{ip})}{\sum_{a \neq i} \exp(s_{ia})} \right)
|
|
419
|
+
$$
|
|
420
|
+
|
|
421
|
+
$$
|
|
422
|
+
\large
|
|
423
|
+
\mathcal{L} = \frac{1}{B} \sum_{i=1}^{B} \mathcal{L}_i
|
|
424
|
+
$$
|
|
425
|
+
|
|
426
|
+
where:
|
|
427
|
+
|
|
428
|
+
- $`s_{ip}`$ — similarity between the anchor and positive $`p`$ (above);
|
|
429
|
+
- $`\sum_{a \neq i}`$ — sum over all other examples in the batch, positives
|
|
430
|
+
*and* negatives (the softmax denominator);
|
|
431
|
+
- $`|\mathcal{P}(i)|`$ — the number of positives for the anchor;
|
|
432
|
+
- $`B`$ — the batch size; the total loss averages the per-anchor terms.
|
|
433
|
+
|
|
434
|
+
Minimizing $`\mathcal{L}`$ pushes $`s_{ip}`$ up for same-class pairs and down
|
|
435
|
+
for different-class pairs. A lower temperature makes the distribution sharper:
|
|
436
|
+
this
|
|
437
|
+
can help separate hard negatives but can also make optimization less stable.
|
|
438
|
+
`--samples_per_class` matters because an anchor needs another example of its
|
|
439
|
+
class to have a positive. A class represented once contributes no useful
|
|
440
|
+
SupCon term for that anchor.
|
|
441
|
+
|
|
442
|
+
#### Temperature
|
|
443
|
+
|
|
444
|
+
The division by $`\tau`$ (`--temperature`, default 0.07) above, and the two
|
|
445
|
+
temperatures used by DINO below, all do the same job. Softmax over logits
|
|
446
|
+
$`s`$ becomes
|
|
447
|
+
|
|
448
|
+
$$
|
|
449
|
+
\large
|
|
450
|
+
\mathrm{softmax}(s)_j = \frac{\exp(s_j / \tau)}{\sum_k \exp(s_k / \tau)}
|
|
451
|
+
$$
|
|
452
|
+
|
|
453
|
+
A small $`\tau`$ divides every logit by a tiny number, so exponentials grow
|
|
454
|
+
very fast and the distribution concentrates on the maximum (sharp); a large
|
|
455
|
+
$`\tau`$ flattens it toward uniform. Sharpening amplifies small differences in
|
|
456
|
+
similarity, which speeds learning, but too-small $`\tau`$ makes gradients
|
|
457
|
+
vanish for non-maximum classes and destabilizes training. SupCon uses one
|
|
458
|
+
moderate value; DINO uses a small teacher temperature (sharper *targets*,
|
|
459
|
+
they are meant to be confident) and a larger student temperature (softer
|
|
460
|
+
*predictions*, easier to move).
|
|
461
|
+
|
|
462
|
+
**DINO** (self-**di**stillation with **no** labels): Caron et al.,
|
|
463
|
+
[Emerging Properties in Self-Supervised Vision Transformers](https://arxiv.org/abs/2104.14294), ICCV 2021.
|
|
464
|
+
One backbone is instantiated twice. The **student** $`f_\theta`$ is trained by
|
|
465
|
+
gradient descent; the **teacher** $`f_\xi`$ is the EMA copy of the student
|
|
466
|
+
($`\xi \leftarrow m\,\xi + (1-m)\,\theta`$, as in I-JEPA above, ramped from
|
|
467
|
+
`--dino_momentum` 0.996 to `--dino_final_momentum` 1.0). Each image is cut
|
|
468
|
+
into 2 **global** crops plus `--dino_local_num` (8) small **local** crops:
|
|
469
|
+
the teacher sees only global views, the student sees all of them, so the
|
|
470
|
+
student must produce globally consistent features from partial views.
|
|
471
|
+
|
|
472
|
+
Both towers end in an MLP projection head whose output is turned into a
|
|
473
|
+
probability vector over the projection dimension. The teacher's distribution
|
|
474
|
+
is *sharpened* by its small temperature and *centered* by a running mean
|
|
475
|
+
$`c \in \mathbb{R}^{D}`$ updated once per step,
|
|
476
|
+
|
|
477
|
+
$$
|
|
478
|
+
\large
|
|
479
|
+
c \leftarrow m_c \, c + (1 - m_c) \cdot \frac{1}{B}\sum_{i=1}^{B} f_\xi(x_i)
|
|
480
|
+
$$
|
|
481
|
+
|
|
482
|
+
(`--dino_center_momentum` 0.9), and the loss is the cross-entropy between the
|
|
483
|
+
teacher's processed distribution $`p_\xi`$ and the student's $`p_\theta`$,
|
|
484
|
+
averaged over all (teacher global, student crop) pairings:
|
|
485
|
+
|
|
486
|
+
$$
|
|
487
|
+
\large
|
|
488
|
+
\mathcal{L} = - \sum_{j} p_{\xi,j} \, \log p_{\theta,j}
|
|
489
|
+
$$
|
|
490
|
+
|
|
491
|
+
Centering and sharpening are the two anti-collapse mechanisms: centering
|
|
492
|
+
cancels any component the teacher produces for *every* input (a constant
|
|
493
|
+
output carries no information and would otherwise still lower the loss), and
|
|
494
|
+
sharpening makes the teacher confident, forcing the student to truly reproduce
|
|
495
|
+
its argmax rather than drifting toward a flat, uninformative distribution.
|
|
496
|
+
Because the targets come from the teacher and the teacher follows the student,
|
|
497
|
+
this is self-distillation — there are no external labels anywhere.
|
|
498
|
+
|
|
499
|
+
**BYOL** (**B**ootstrap **Y**our **O**wn **L**atent): Grill et al.,
|
|
500
|
+
[Bootstrap Your Own Latent](https://arxiv.org/abs/2006.07733), NeurIPS 2020.
|
|
501
|
+
The same two-tower idea as DINO but with two augmented views and no
|
|
502
|
+
multi-crop. The **online** network (encoder + projector + predictor MLP) and
|
|
503
|
+
the **target** network (encoder + projector, EMA copy) process the two views;
|
|
504
|
+
only the online tower receives gradients. With $`z_1, z_2`$ the target
|
|
505
|
+
projections of the two views and $`p_1, p_2`$ the online *predictions* of
|
|
506
|
+
those projections, the loss is the symmetric cosine regression
|
|
507
|
+
|
|
508
|
+
$$
|
|
509
|
+
\large
|
|
510
|
+
\mathcal{L} = 2 - \cos(p_1, z_2) - \cos(p_2, z_1)
|
|
511
|
+
$$
|
|
512
|
+
|
|
513
|
+
which is 0 when predictions perfectly match target projections and reaches 4
|
|
514
|
+
when they point in opposite directions; $`\cos(u,v) = u^\top v / (\lVert u
|
|
515
|
+
\rVert_2 \lVert v \rVert_2)`$ is the cosine of the angle between the two
|
|
516
|
+
vectors. Two asymmetries keep this from collapsing to a constant: the
|
|
517
|
+
predictor exists only on the online side (predicting a target is easier than
|
|
518
|
+
being one), and the targets move slowly under the EMA — exactly the
|
|
519
|
+
teacher-follows-student trick noted under I-JEPA.
|
|
520
|
+
|
|
521
|
+
### Typical Hyperparameters
|
|
522
|
+
|
|
523
|
+
These are reasonable starting points. Tune from here based on your dataset
|
|
524
|
+
size and GPU memory:
|
|
525
|
+
|
|
526
|
+
| Parameter | SimMIM | I-JEPA | DINO | BYOL | SupCon |
|
|
527
|
+
|---|---|---|---|---|---|
|
|
528
|
+
| `--image_size` | 448 | 448 | 224 | 224 | 448 |
|
|
529
|
+
| `--batch_size` | 32 | 32 | 64 | 64 | 64 |
|
|
530
|
+
| `--lr` | 1e-4 | 1e-4 | 5e-4 | 5e-4 | 1e-4 |
|
|
531
|
+
| `--epochs` | 200 | 200 | 100 | 100 | 100 |
|
|
532
|
+
| `--scheduler` | CosineAnnealingLR | CosineAnnealingLR | CosineAnnealingLR | CosineAnnealingLR | CosineAnnealingLR |
|
|
533
|
+
| `--amp_dtype` | bfloat16 | bfloat16 | bfloat16 | bfloat16 | bfloat16 |
|
|
534
|
+
| `--mask_ratio` | 0.6 | — | — | — | — |
|
|
535
|
+
| `--teacher_momentum` | — | 0.996→1.0 | — | — | — |
|
|
536
|
+
| `--dino_momentum` | — | — | 0.996→1.0 | — | — |
|
|
537
|
+
| `--byol_momentum` | — | — | — | 0.996→1.0 | — |
|
|
538
|
+
| `--temperature` | — | — | — | — | 0.07 |
|
|
539
|
+
| `--samples_per_class` | — | — | — | — | 16 |
|
|
540
|
+
|
|
541
|
+
**Tips:**
|
|
542
|
+
- Start with 200 epochs for SimMIM/I-JEPA. DINO, BYOL, and SupCon converge
|
|
543
|
+
faster (~100).
|
|
544
|
+
- `--temperature 0.07` is the standard from the original SupCon paper.
|
|
545
|
+
Lower = sharper contrastive distribution; try 0.05–0.1.
|
|
546
|
+
- DINO and BYOL train on smaller crops (224) with a larger LR because their
|
|
547
|
+
loss acts on a small projection space rather than every pixel; the defaults
|
|
548
|
+
above match the method-specific `--dino_*` / `--byol_*` flags.
|
|
549
|
+
- `--samples_per_class 16` with `--batch_size 64` gives 4 classes per batch
|
|
550
|
+
on a 7-class dataset. batch_size need not be divisible by
|
|
551
|
+
samples_per_class; the remainder is spread evenly across the per-class
|
|
552
|
+
groups so each batch still contains exactly batch_size samples.
|
|
553
|
+
- Use `--amp_dtype bfloat16` if your GPU supports it (Ampere+). Otherwise
|
|
554
|
+
`float16` with GradScaler works too.
|
|
555
|
+
|
|
556
|
+
### Example: Full Pre-Training Pipeline
|
|
557
|
+
|
|
558
|
+
```bash
|
|
559
|
+
# Step 1: Pre-train with SimMIM on two large image collections.
|
|
560
|
+
# v4.2: pre-training is a *configuration* of the unified genml-kit-train
|
|
561
|
+
# CLI (--pipeline images + --method simmim), not a separate binary.
|
|
562
|
+
genml-kit-train --pipeline images --method simmim \
|
|
563
|
+
--model convvit \
|
|
564
|
+
--datasets "imagefolder/raw-photos" "imagefolder/more-photos" \
|
|
565
|
+
--image_size 448 \
|
|
566
|
+
--batch_size 32 \
|
|
567
|
+
--epochs 200 \
|
|
568
|
+
--lr 1e-4 \
|
|
569
|
+
--scheduler CosineAnnealingLR \
|
|
570
|
+
--sched_arg T_max=200 --sched_arg eta_min=1e-6 \
|
|
571
|
+
--amp_dtype bfloat16 \
|
|
572
|
+
--checkpoint ./checkpoints/convvit_simmim
|
|
573
|
+
|
|
574
|
+
# Step 2: Fine-tune on your labeled dataset
|
|
575
|
+
genml-kit-train --model convvit \
|
|
576
|
+
--dataset my-org/labeled-photos \
|
|
577
|
+
--label_column category \
|
|
578
|
+
--source_checkpoint ./checkpoints/convvit_simmim_latest.pt \
|
|
579
|
+
--epochs 100 \
|
|
580
|
+
--lr 3e-5 \
|
|
581
|
+
--batch_size 32 \
|
|
582
|
+
--amp_dtype bfloat16
|
|
583
|
+
```
|
|
584
|
+
|
|
585
|
+
### Example: Supervised Contrastive Pre-Training
|
|
586
|
+
|
|
587
|
+
```bash
|
|
588
|
+
genml-kit-train --pipeline images --method supcon \
|
|
589
|
+
--model convvit \
|
|
590
|
+
--datasets my-org/labeled-photos \
|
|
591
|
+
--label_column category \
|
|
592
|
+
--image_size 448 \
|
|
593
|
+
--batch_size 64 \
|
|
594
|
+
--samples_per_class 16 \
|
|
595
|
+
--proj_dim 128 \
|
|
596
|
+
--temperature 0.07 \
|
|
597
|
+
--epochs 100 \
|
|
598
|
+
--lr 1e-4 \
|
|
599
|
+
--amp_dtype bfloat16 \
|
|
600
|
+
--checkpoint ./checkpoints/convvit_supcon
|
|
601
|
+
|
|
602
|
+
# Then fine-tune as above with --source_checkpoint ./checkpoints/convvit_supcon_latest.pt
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
### Pre-Training CLI Reference
|
|
606
|
+
|
|
607
|
+
| Argument | Default | Description |
|
|
608
|
+
|---|---|---|
|
|
609
|
+
| `--method` | `classification` | Objective. The pre-training examples below pass `--method simmim` explicitly. Choices: `classification`, `simmim`, `ijepa`, `dino`, `byol`, `supcon`, `vo_pair`. |
|
|
610
|
+
| `--model` | `convvit` | Model name registered in genml_kit or HuggingFace model ID. |
|
|
611
|
+
| `--datasets` | (required) | Space-separated dataset names or local paths. |
|
|
612
|
+
| `--cache_dir` | `None` | HuggingFace cache directory for downloads. |
|
|
613
|
+
| `--remote_checkpoint` | `None` | Remote URI for checkpoint sync (`gs://BUCKET/PREFIX`, `r2://BUCKET/PREFIX`, or `s3://BUCKET/PREFIX`). |
|
|
614
|
+
| `--hf_token` | `None` | HuggingFace token for gated datasets (or set `HF_TOKEN` env var). |
|
|
615
|
+
| `--image_column` | auto-detected | Explicit HF image column name. |
|
|
616
|
+
| `--label_column` | `None` | Explicit HF label column name. Required by `--method supcon` if non-standard. |
|
|
617
|
+
| `STRICT_DATASETS` (env) | `False` | Abort on first dataset-loading failure instead of skipping. |
|
|
618
|
+
| `--image_size` | `448` | Input image size (square). |
|
|
619
|
+
| `--batch_size` | `32` | Per-GPU batch size. |
|
|
620
|
+
| `--seed` | `None` | Explicit RNG seed: pins the shuffling and enables full determinism (cuDNN deterministic kernels, benchmark off). Omit (default) to keep runs fast — RNG streams are still seeded from `$GENML_KIT_SEED` (default 42). See [Reproducibility](#reproducibility). |
|
|
621
|
+
| `--epochs` | `5` | Number of epochs. The recipes below override this for pre-training. |
|
|
622
|
+
| `--lr` | `3e-5` | Peak learning rate (base). The recipes below override this. |
|
|
623
|
+
| `--amp_dtype` | `None` | Mixed precision: `float16` or `bfloat16`. Omit to disable. |
|
|
624
|
+
| `--num_workers` | `4` | DataLoader worker processes. |
|
|
625
|
+
| `--device` | auto-detect | Device: `cpu`, `cuda`, or `cuda:INDEX`. |
|
|
626
|
+
| `--resume` | `True` | Auto-resume from latest checkpoint. Use `--no-resume` to disable. |
|
|
627
|
+
| `--state_save` | `opt,sched,amp` | States to save: `opt`, `sched`, `amp`, `none`. |
|
|
628
|
+
| `--state_load` | `opt,sched,amp` | States to restore on resume: `opt`, `sched`, `amp`, `none`. |
|
|
629
|
+
| `--checkpoint` | (required) | Checkpoint path prefix (saves `_latest.pt` and `_best.pt`). |
|
|
630
|
+
| `--log_level` | `INFO` | Minimum logging level. |
|
|
631
|
+
| `--log_targets` | `STDERR` | Comma-separated log destinations. `STDERR` logs to standard error; any other entry is a log file path (appended). Example: `STDERR,/tmp/train.log` logs to both. |
|
|
632
|
+
| `--grad_monitor` | `-1` | Log gradient statistics every N steps; `-1` disables. See [Gradient Monitor](#gradient-monitor). |
|
|
633
|
+
| `--norm_history` | `0` | Keep last N norm snapshots per parameter for trend analysis. |
|
|
634
|
+
| `--trend_top_n` | `10` | Show top N params in trend table by abs change %. `0` = show all. |
|
|
635
|
+
| `--grad_clip` | `1.0` | Maximum gradient norm for clipping. `0` disables. |
|
|
636
|
+
| `--lr_group` | `None` | Per-parameter-group learning rates (repeatable). Format: `"REGEX=LR"`. |
|
|
637
|
+
| `--llrd_decay` | `None` | Layer-wise learning rate decay factor. |
|
|
638
|
+
| `--vis_every` | `0` | Log reconstruction visualisation every N steps (SimMIM only). |
|
|
639
|
+
| `--save_every` | `500` | Save checkpoint every N optimizer steps. 0 disables. |
|
|
640
|
+
| `--model_arg` | `{}` | Override model configuration (repeatable). For `--model cls_model_wrapper:<hf_name>` the classifier head travels here too: `--model_arg classifier=mlp:hidden=512,dropout=0.3`. |
|
|
641
|
+
| `--proc_arg` | `{}` | Override processor configuration (repeatable). |
|
|
642
|
+
| `--optimizer` | `AdamW` | `torch.optim` optimizer class name or `.py` script path. |
|
|
643
|
+
| `--opt_arg` | `{}` | Extra optimizer kwargs (repeatable). |
|
|
644
|
+
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path. |
|
|
645
|
+
| `--sched_arg` | `{}` | Extra scheduler kwargs (repeatable). |
|
|
646
|
+
| `--source_checkpoint` | `None` | Path to source checkpoint to absorb parameters from. |
|
|
647
|
+
| `--param_rename` | `None` | Regex-based key rename patterns (`SEARCH;REPLACE`). |
|
|
648
|
+
| `--grad_checkpoint` | `False` | Enable gradient checkpointing. Reduces activation memory by ~40-50% at the cost of ~25-35% more compute per step. Enables larger batch sizes. See [Gradient Checkpointing](#gradient-checkpointing). |
|
|
649
|
+
|
|
650
|
+
**Defaults shown are the generic `genml-kit-train` defaults; the
|
|
651
|
+
pre-training recipes below override `--method`, `--epochs`, `--lr` for their
|
|
652
|
+
specific objective.** The pre-training examples invoke
|
|
653
|
+
`genml-kit-train --pipeline images --method simmim ...` explicitly.
|
|
654
|
+
|
|
655
|
+
**SupCon-specific arguments:**
|
|
656
|
+
|
|
657
|
+
| Argument | Default | Description |
|
|
658
|
+
|---|---|---|
|
|
659
|
+
| `--proj_dim` | `128` | Output dimensionality of the projection head. |
|
|
660
|
+
| `--proj_hidden` | `None` | Hidden layer size of the projection MLP. `None` = single linear layer. |
|
|
661
|
+
| `--temperature` | `0.07` | NT-Xent temperature. Lower = sharper contrastive distribution. |
|
|
662
|
+
| `--samples_per_class` | `16` | Samples per class in each batch. Batch size need not be divisible by this; the remainder is spread evenly across the per-class groups. |
|
|
663
|
+
|
|
664
|
+
### Dataset Ensemble
|
|
665
|
+
|
|
666
|
+
With `--pipeline images`, `genml-kit-train` stitches multiple datasets into
|
|
667
|
+
a single pre-training corpus (via `--datasets`). This is useful because no
|
|
668
|
+
single dataset is large enough for effective pre-training on its own.
|
|
669
|
+
|
|
670
|
+
Supported dataset types:
|
|
671
|
+
- **HuggingFace datasets** — any HF dataset ID that returns decoded image
|
|
672
|
+
data (e.g. `cifar10`, `food101`). Gated datasets require `--hf_token` or
|
|
673
|
+
`HF_TOKEN`.
|
|
674
|
+
- **Local image directories** — pass a path to a folder of images
|
|
675
|
+
(ImageFolder format).
|
|
676
|
+
|
|
677
|
+
Datasets are loaded lazily (only when first accessed). By default, datasets
|
|
678
|
+
that fail to load are logged and skipped (best-effort mode). Set the
|
|
679
|
+
`STRICT_DATASETS` environment variable to a truthy value (`1`, `true`,
|
|
680
|
+
`yes`, `on`) to abort on the first failure instead.
|
|
681
|
+
|
|
682
|
+
Images that cannot be decoded are skipped with a warning — this prevents a
|
|
683
|
+
single corrupted file from blocking an entire pre-training run.
|
|
684
|
+
|
|
685
|
+
#### Label Validation
|
|
686
|
+
|
|
687
|
+
When using a label-aware method (`--method supcon`), the ensemble validates
|
|
688
|
+
that every dataset supports labels. Datasets without a label column cause a
|
|
689
|
+
clear error *before* training begins, not a cryptic runtime failure mid-epoch.
|
|
690
|
+
|
|
691
|
+
Labels are automatically remapped to a shared global label space across all
|
|
692
|
+
datasets, so mixing datasets with overlapping but differently-named classes
|
|
693
|
+
works transparently.
|
|
694
|
+
|
|
695
|
+
### Preparing Datasets
|
|
696
|
+
|
|
697
|
+
Some datasets store images inside zip archives or need custom preprocessing
|
|
698
|
+
before they can be used for pre-training. The toolkit consumes any local
|
|
699
|
+
ImageFolder directory, so a small preparation script is all it takes:
|
|
700
|
+
|
|
701
|
+
```bash
|
|
702
|
+
python my_prepare_script.py --output_dir ./prepared_images
|
|
703
|
+
```
|
|
704
|
+
|
|
705
|
+
Then use the extracted directory as a local dataset:
|
|
706
|
+
|
|
707
|
+
```bash
|
|
708
|
+
genml-kit-train --pipeline images --method simmim \
|
|
709
|
+
--datasets ./prepared_images ./other-images \
|
|
710
|
+
--image_size 448 --batch_size 32 ...
|
|
711
|
+
```
|
|
712
|
+
|
|
713
|
+
For a concrete worked example of such a preparation script, see
|
|
714
|
+
`scdiag/scripts/prepare_derm1m.py` and `scdiag/scripts/prepare_ham10000.py`
|
|
715
|
+
in the repository (dermoscopy corpora).
|
|
716
|
+
|
|
717
|
+
---
|
|
718
|
+
|
|
719
|
+
## Fine-Tuning Guide
|
|
720
|
+
|
|
721
|
+
After pre-training (or directly, if you skip pre-training), fine-tune a
|
|
722
|
+
classifier on your labeled dataset.
|
|
723
|
+
|
|
724
|
+
### What Fine-Tuning Is Changing
|
|
725
|
+
|
|
726
|
+
Suppose the encoder produces the feature vector $`h = f_\theta(x)`$. A linear
|
|
727
|
+
classification head computes logits $`a = W h + b`$, and softmax turns them
|
|
728
|
+
into probabilities:
|
|
729
|
+
|
|
730
|
+
$$
|
|
731
|
+
\large
|
|
732
|
+
p(y=c \mid x) = \frac{\exp(a_c)}{\sum_k \exp(a_k)}
|
|
733
|
+
$$
|
|
734
|
+
|
|
735
|
+
where:
|
|
736
|
+
|
|
737
|
+
- $`h \in \mathbb{R}^{D}`$ — the encoder's feature vector for image $`x`$;
|
|
738
|
+
- $`W \in \mathbb{R}^{C \times D}`$, $`b \in \mathbb{R}^{C}`$ — the head's weight
|
|
739
|
+
matrix and bias ($`C`$ = number of classes, $`D`$ = feature dimension);
|
|
740
|
+
- $`a_c`$ — the logit (raw score) for class $`c`$;
|
|
741
|
+
- $`p(y=c \mid x)`$ — the model's probability that the class of $`x`$ is
|
|
742
|
+
$`c`$; the softmax exponentials make the scores positive and summing to one.
|
|
743
|
+
|
|
744
|
+
Training minimizes cross-entropy between the model's predicted distribution
|
|
745
|
+
$`p`$ and the target distribution $`q`$ over the $`C`$ classes,
|
|
746
|
+
|
|
747
|
+
$$
|
|
748
|
+
\large
|
|
749
|
+
\mathcal{L}_{\text{CE}}(p, q) = -\sum_{c=1}^{C} q_c \log p_c
|
|
750
|
+
$$
|
|
751
|
+
|
|
752
|
+
For a hard label the target $`q`$ is a one-hot vector, so only the term of the
|
|
753
|
+
true class survives and the loss reduces to the familiar $`-\log p(y \mid x)`$.
|
|
754
|
+
genml_kit generalizes this in three directions via `CombinedFocalLoss`:
|
|
755
|
+
|
|
756
|
+
- **Soft targets.** With Mixup (below) $`q`$ is the blend
|
|
757
|
+
$`\lambda\, y_i + (1-\lambda)\, y_j`$, and the sum keeps both classes'
|
|
758
|
+
terms weighted by $`\lambda`$ and $`1-\lambda`$.
|
|
759
|
+
- **Label smoothing** (via `--loss_args '{"label_smoothing": ...}'`):
|
|
760
|
+
the one-hot target is replaced by
|
|
761
|
+
$`q_c = 1 - \varepsilon + \varepsilon/C`$ for the true class and
|
|
762
|
+
$`\varepsilon/C`$ for the others. The model is no longer asked to drive
|
|
763
|
+
$`p_c \to 1`$ exactly, which reduces overconfidence and improves
|
|
764
|
+
calibration on validation data.
|
|
765
|
+
- **Class weights** (`--class_multipliers`): each term of the sum is scaled by
|
|
766
|
+
a per-class factor $`w_c`$, so rare classes contribute larger gradients.
|
|
767
|
+
|
|
768
|
+
The new classifier head is normally initialized from scratch because
|
|
769
|
+
its output
|
|
770
|
+
size depends on the target classes. When loading a pre-training checkpoint,
|
|
771
|
+
the useful part to transfer is the encoder; an unused SupCon projection head
|
|
772
|
+
should not be copied into the classifier.
|
|
773
|
+
|
|
774
|
+
The practical choice is how much of $`\theta`$ to update:
|
|
775
|
+
|
|
776
|
+
- **Full fine-tuning** updates encoder and head. It gives the model the most
|
|
777
|
+
freedom, but needs enough data and a conservative learning rate.
|
|
778
|
+
- **Frozen-backbone training** updates only the head. It is a useful baseline
|
|
779
|
+
for small datasets and shows how much information the representation holds.
|
|
780
|
+
- **LLRD** updates all layers but gives early layers smaller learning rates.
|
|
781
|
+
This is often a good compromise when the pre-training domain differs from
|
|
782
|
+
the fine-tuning domain (e.g. ImageNet to specialized imagery).
|
|
783
|
+
- **LoRA** freezes the original matrices and learns small low-rank updates.
|
|
784
|
+
It is useful when GPU memory or labeled data is limited.
|
|
785
|
+
|
|
786
|
+
Compare these strategies on the same validation split. The lowest training
|
|
787
|
+
loss is not necessarily the best model: monitor macro-F1, balanced accuracy,
|
|
788
|
+
weighted F1, and per-class precision and recall, especially for minority
|
|
789
|
+
classes.
|
|
790
|
+
|
|
791
|
+
### Evaluation Metrics
|
|
792
|
+
|
|
793
|
+
Validation reports include:
|
|
794
|
+
|
|
795
|
+
- **Top-1 accuracy:** the percentage of validation images assigned the correct
|
|
796
|
+
class by the highest-probability prediction.
|
|
797
|
+
- **Precision:** for a class, the fraction of images predicted as that class
|
|
798
|
+
that truly belong to it:
|
|
799
|
+
$`\text{precision} = TP / (TP + FP)`$.
|
|
800
|
+
- **Recall:** for a class, the fraction of images belonging to that class that
|
|
801
|
+
are predicted correctly: $`\text{recall} = TP / (TP + FN)`$.
|
|
802
|
+
Here $`TP`$ (true positives) counts images of the class that were correctly
|
|
803
|
+
predicted as that class, $`FP`$ (false positives) counts images of *other*
|
|
804
|
+
classes wrongly predicted as this class, and $`FN`$ (false negatives) counts
|
|
805
|
+
images of this class wrongly predicted as something else.
|
|
806
|
+
- **F1:** the harmonic mean of precision and recall:
|
|
807
|
+
$`F_1 = \frac{2 \cdot \text{precision} \cdot \text{recall}}{\text{precision} + \text{recall}}`$.
|
|
808
|
+
When the denominator is zero, scikit-learn's `zero_division=0` behavior
|
|
809
|
+
reports zero.
|
|
810
|
+
- **Macro F1:** the arithmetic mean of the per-class F1 scores:
|
|
811
|
+
$`F_{1}^{\text{macro}} = \frac{1}{C} \sum_{c=1}^{C} F_{1,c}`$.
|
|
812
|
+
Every class contributes equally, regardless of its validation-set size
|
|
813
|
+
($`C`$ is the number of classes, $`F_{1,c}`$ the F1 of class $`c`$).
|
|
814
|
+
This is the metric used for best-checkpoint selection.
|
|
815
|
+
- **Weighted F1:** the mean of per-class F1 scores weighted by each class's
|
|
816
|
+
validation support. It is therefore more influenced by common classes.
|
|
817
|
+
- **Balanced accuracy:** the arithmetic mean of per-class recall:
|
|
818
|
+
$`\text{balanced accuracy} = \frac{1}{C} \sum_{c=1}^{C} \text{recall}_c`$.
|
|
819
|
+
It gives each class equal weight and is useful for imbalanced datasets.
|
|
820
|
+
- **Support:** the number of true validation examples for a class.
|
|
821
|
+
|
|
822
|
+
The confusion matrix uses rows for true classes and columns for predicted
|
|
823
|
+
classes. Diagonal entries are correct predictions. The compact confusion
|
|
824
|
+
summary reports each class's recall and its largest confusion destinations.
|
|
825
|
+
|
|
826
|
+
|
|
827
|
+
When TTA is enabled, validation loss is computed from the original image only,
|
|
828
|
+
while Top-1 accuracy, balanced accuracy, macro F1, weighted F1, and per-class
|
|
829
|
+
metrics use probabilities averaged over the original image and augmented
|
|
830
|
+
views. The log also reports original-view metrics and the change produced by
|
|
831
|
+
TTA.
|
|
832
|
+
|
|
833
|
+
Training metrics may be measured on an augmented or weighted-sampler stream.
|
|
834
|
+
They should not be compared directly with validation metrics unless they use
|
|
835
|
+
the same sampling and preprocessing scheme.
|
|
836
|
+
|
|
837
|
+
### Basic Fine-Tuning
|
|
838
|
+
|
|
839
|
+
```bash
|
|
840
|
+
genml-kit-train --model google/vit-base-patch16-224 \
|
|
841
|
+
--dataset my-org/my-labeled-images \
|
|
842
|
+
--label_column category \
|
|
843
|
+
--epochs 5 \
|
|
844
|
+
--batch_size 32 \
|
|
845
|
+
--lr 3e-5 \
|
|
846
|
+
--image_size 448
|
|
847
|
+
```
|
|
848
|
+
|
|
849
|
+
### With A Custom Classifier Head
|
|
850
|
+
|
|
851
|
+
Replace the default linear head with a custom MLP or attention-based
|
|
852
|
+
classifier:
|
|
853
|
+
|
|
854
|
+
```bash
|
|
855
|
+
# Freeze backbone, train only the custom head
|
|
856
|
+
genml-kit-train --model cls_model_wrapper:google/vit-base-patch16-224 \
|
|
857
|
+
--dataset my-org/my-labeled-images \
|
|
858
|
+
--label_column category \
|
|
859
|
+
--model_arg classifier=mlp:hidden=512,dropout=0.3 \
|
|
860
|
+
--freeze ".*\.(head|pool)"
|
|
861
|
+
```
|
|
862
|
+
|
|
863
|
+
### With LoRA (Parameter-Efficient Fine-Tuning)
|
|
864
|
+
|
|
865
|
+
Freeze the entire backbone and train only small low-rank adapter matrices.
|
|
866
|
+
Reduces trainable parameters by ~97% while often matching full fine-tuning:
|
|
867
|
+
|
|
868
|
+
```bash
|
|
869
|
+
genml-kit-train \
|
|
870
|
+
--model cls_model_wrapper:facebook/dinov2-with-registers-large \
|
|
871
|
+
--lora --lora_r 16 --lora_alpha 32 \
|
|
872
|
+
--lora_target_modules "query,key,value" \
|
|
873
|
+
--freeze "classifier\.(head|pool|encoder)" \
|
|
874
|
+
--lr 3e-5 \
|
|
875
|
+
--dataset my-org/my-labeled-images \
|
|
876
|
+
--label_column category \
|
|
877
|
+
--epochs 20
|
|
878
|
+
```
|
|
879
|
+
|
|
880
|
+
### From A Pre-Trained Checkpoint
|
|
881
|
+
|
|
882
|
+
Load encoder weights from a pre-training run (SimMIM, I-JEPA, or SupCon):
|
|
883
|
+
|
|
884
|
+
```bash
|
|
885
|
+
genml-kit-train --model convvit \
|
|
886
|
+
--dataset my-org/my-labeled-images \
|
|
887
|
+
--label_column category \
|
|
888
|
+
--source_checkpoint ./checkpoints/convvit_simmim_latest.pt \
|
|
889
|
+
--epochs 100
|
|
890
|
+
```
|
|
891
|
+
|
|
892
|
+
The backbone weights are loaded automatically; the classifier head is
|
|
893
|
+
reinitialised (different `num_classes`). Use `--state_load none` to avoid
|
|
894
|
+
carrying over old optimizer/scheduler states.
|
|
895
|
+
|
|
896
|
+
### Layer-Wise Learning Rate Decay (LLRD)
|
|
897
|
+
|
|
898
|
+
LLRD is a compromise between freezing the backbone and updating every layer at
|
|
899
|
+
the same speed. If layers are indexed from shallow 0 to deep $`L`$, a common
|
|
900
|
+
schedule is (introduced by Howard & Ruder, [Universal Language Model Fine-tuning](https://arxiv.org/abs/1801.06146), ACL 2018):
|
|
901
|
+
|
|
902
|
+
$$
|
|
903
|
+
\large
|
|
904
|
+
\text{lr}(\text{layer}) = \text{lr}_{\text{base}} \cdot d^{\,L - \text{layer}}
|
|
905
|
+
$$
|
|
906
|
+
|
|
907
|
+
where:
|
|
908
|
+
|
|
909
|
+
- $`\text{lr}_{\text{base}}`$ — the `--lr` you pass on the command line;
|
|
910
|
+
- $`d`$ — the decay factor (`--llrd_decay`), usually between 0.8 and 1.0;
|
|
911
|
+
- $`L`$ — the index of the deepest layer, $`\text{layer}`$ the current layer's
|
|
912
|
+
index, both counted from 0 at the input side.
|
|
913
|
+
|
|
914
|
+
The deepest layer
|
|
915
|
+
receives the base rate while earlier layers receive smaller updates. Early
|
|
916
|
+
layers tend to represent general edges and textures; later layers are more
|
|
917
|
+
task-specific. This is a useful heuristic, not a law, so validate it on your
|
|
918
|
+
dataset. A very small decay factor can effectively freeze the shallow network.
|
|
919
|
+
|
|
920
|
+
### Cosine Annealing
|
|
921
|
+
|
|
922
|
+
`--scheduler CosineAnnealingLR` (the default in every example above) decays
|
|
923
|
+
the learning rate from its initial value $`\eta_{\max}`$ toward $`\eta_{\min}`$
|
|
924
|
+
(0 by default) following a half cosine wave over `--epochs` epochs $`T`$:
|
|
925
|
+
|
|
926
|
+
$$
|
|
927
|
+
\large
|
|
928
|
+
\eta(t) = \eta_{\min} + \frac{1}{2}\left(\eta_{\max} - \eta_{\min}\right)
|
|
929
|
+
\left(1 + \cos\left(\frac{t \pi}{T}\right)\right)
|
|
930
|
+
$$
|
|
931
|
+
|
|
932
|
+
where $`t`$ is the current epoch. The cosine shape spends more time at high
|
|
933
|
+
learning rates than a linear decay — useful early, when large steps explore
|
|
934
|
+
the loss surface — and slows down smoothly as it approaches zero, which lets
|
|
935
|
+
the model settle into a minimum instead of bouncing around it. Late-training
|
|
936
|
+
low learning rates are also what make the EMA teacher of DINO/BYOL
|
|
937
|
+
effective: the teacher tracks a student that changes gently.
|
|
938
|
+
|
|
939
|
+
### Mixup, Label Smoothing, And Imbalance
|
|
940
|
+
|
|
941
|
+
Mixup forms a virtual example from two training examples.
|
|
942
|
+
Reference: Zhang et al., [mixup: Beyond Empirical Risk Minimization](https://arxiv.org/abs/1710.09412), ICLR 2018.
|
|
943
|
+
|
|
944
|
+
$$
|
|
945
|
+
\large
|
|
946
|
+
\tilde{x} = \lambda x_i + (1 - \lambda) x_j
|
|
947
|
+
$$
|
|
948
|
+
|
|
949
|
+
$$
|
|
950
|
+
\large
|
|
951
|
+
\tilde{y} = \lambda y_i + (1 - \lambda) y_j
|
|
952
|
+
$$
|
|
953
|
+
|
|
954
|
+
where $`(x_i, y_i)`$ and $`(x_j, y_j)`$ are two randomly paired training
|
|
955
|
+
examples and the mixing coefficient $`\lambda`$ is drawn from the Beta
|
|
956
|
+
distribution, $`\lambda \sim \mathrm{Beta}(\alpha, \alpha)`$, with $`\alpha`$
|
|
957
|
+
set by `--mixup_alpha` (typical values 0.2–0.4). The parameter $`\alpha`$
|
|
958
|
+
shapes the distribution: small $`\alpha`$ puts most weight near 0 or 1,
|
|
959
|
+
large $`\alpha`$ near 1/2. The labels are probability vectors, not
|
|
960
|
+
class indices. Mixup smooths the decision boundary and can help on small
|
|
961
|
+
datasets, but strong Mixup can obscure fine-grained image details. Label
|
|
962
|
+
smoothing similarly replaces a one-hot label with a mostly-correct
|
|
963
|
+
distribution.
|
|
964
|
+
|
|
965
|
+
#### Focal Loss
|
|
966
|
+
|
|
967
|
+
Focal loss instead changes the emphasis of cross-entropy. Reference: Lin et
|
|
968
|
+
al., [Focal Loss for Dense Object Detection](https://arxiv.org/abs/1708.02002), ICCV 2017.
|
|
969
|
+
With a hard label and $`p_t`$ the predicted probability of the correct class,
|
|
970
|
+
its basic form is
|
|
971
|
+
|
|
972
|
+
$$
|
|
973
|
+
\large
|
|
974
|
+
\mathcal{L}_{\text{focal}} = -(1 - p_t)^{\gamma} \log(p_t)
|
|
975
|
+
$$
|
|
976
|
+
|
|
977
|
+
where $`\gamma`$ (via `--loss_args '{"gamma": ...}'`) down-weights easy examples: when
|
|
978
|
+
$`p_t \to 1`$ the factor $`(1-p_t)^{\gamma} \to 0`$, so already-confident,
|
|
979
|
+
easy examples receive almost no weight and training focuses on hard ones.
|
|
980
|
+
With $`\gamma = 0`$ the factor is 1 for every example and the loss is plain
|
|
981
|
+
cross-entropy; typical values are 1–3, and 2.0 is the usual default.
|
|
982
|
+
|
|
983
|
+
genml_kit implements the general form (`CombinedFocalLoss` in
|
|
984
|
+
`genml_kit/losses/focal.py`), which composes cleanly with the other
|
|
985
|
+
tools in this section. Let $`q`$ be the target distribution from the previous
|
|
986
|
+
step (one-hot, Mixup-blended, or label-smoothed), $`w_c`$ the per-class
|
|
987
|
+
multipliers (`--class_multipliers`), and $`p`$ the model's softmax output.
|
|
988
|
+
The loss for one example is
|
|
989
|
+
|
|
990
|
+
$$
|
|
991
|
+
\large
|
|
992
|
+
\mathcal{L} = \left(1 - p_t\right)^{\gamma} \sum_{c=1}^{C} w_c\, q_c
|
|
993
|
+
\left(-\log p_c\right), \qquad p_t = \sum_{c=1}^{C} q_c\, p_c
|
|
994
|
+
$$
|
|
995
|
+
|
|
996
|
+
Two details matter for the reader combining these flags:
|
|
997
|
+
|
|
998
|
+
- **Unified $`p_t`.** The modulating factor uses the *expected* probability
|
|
999
|
+
under the target distribution, $`p_t = \sum_c q_c p_c`$, rather than the
|
|
1000
|
+
probability of a single class. For a one-hot target $`q`$ this reduces
|
|
1001
|
+
exactly to the basic form above ($`p_t`$ becomes the correct-class
|
|
1002
|
+
probability); with Mixup soft targets it blends the two classes'
|
|
1003
|
+
probabilities, giving one smooth modulation per example instead of an
|
|
1004
|
+
arbitrary single-class choice.
|
|
1005
|
+
- **One modulation per example.** The factor $`(1-p_t)^{\gamma}`$ scales the
|
|
1006
|
+
whole weighted cross-entropy sum — weights and soft labels act *inside* the
|
|
1007
|
+
sum, focal acts *outside* it. Each term is therefore attenuated
|
|
1008
|
+
proportionally to how confident the model already is about that example, no
|
|
1009
|
+
matter which class the probability mass sits on.
|
|
1010
|
+
|
|
1011
|
+
Use these tools deliberately; combining every regularizer is not
|
|
1012
|
+
automatically better.
|
|
1013
|
+
|
|
1014
|
+
### Hyperparameter Guidance
|
|
1015
|
+
|
|
1016
|
+
| Scenario | `--lr` | `--epochs` | `--batch_size` | Notes |
|
|
1017
|
+
|---|---|---|---|---|
|
|
1018
|
+
| Large dataset (>10k images) | 3e-5 | 20–50 | 32–64 | Standard fine-tuning |
|
|
1019
|
+
| Small dataset (<1k images) | 1e-5 | 50–100 | 16–32 | Consider LoRA, stronger augmentation |
|
|
1020
|
+
| From pre-trained checkpoint | 3e-5 | 50–100 | 32 | Lower LR than from scratch |
|
|
1021
|
+
| Custom classifier head only | 1e-3 | 50–200 | 32 | Higher LR since only head trains |
|
|
1022
|
+
|
|
1023
|
+
**Tips:**
|
|
1024
|
+
- Start with `--lr 3e-5` for full fine-tuning, `1e-3` for head-only training.
|
|
1025
|
+
- Use `--mixup_alpha 0.2` for small datasets — it helps prevent overfitting.
|
|
1026
|
+
- `--loss_args '{"gamma": 2.0}'` down-weights easy examples, useful when
|
|
1027
|
+
classes are imbalanced.
|
|
1028
|
+
- `--class_multipliers "rare_class=3.0"` increases the loss weight for
|
|
1029
|
+
safety-critical or otherwise priority classes.
|
|
1030
|
+
|
|
1031
|
+
### Reproducibility
|
|
1032
|
+
|
|
1033
|
+
RNG streams are always seeded. By default the seed comes from the
|
|
1034
|
+
`GENML_KIT_SEED` environment variable (default `42`), and it drives
|
|
1035
|
+
DataLoader shuffling, per-worker augmentation randomness, mixup, and
|
|
1036
|
+
dropout — the user not caring about determinism does not mean the
|
|
1037
|
+
toolkit runs on entropy. To compare hyperparameters under a different
|
|
1038
|
+
random draw, change `GENML_KIT_SEED` or pass an explicit seed.
|
|
1039
|
+
|
|
1040
|
+
Two levels of determinism:
|
|
1041
|
+
|
|
1042
|
+
| Invocation | Seeding | CUDA kernels | Reproducible? |
|
|
1043
|
+
|---|---|---|---|
|
|
1044
|
+
| default (no `--seed`) | `GENML_KIT_SEED` or 42 | `cudnn.benchmark=True` (fastest algorithms) | RNG streams identical; arithmetic not bit-exact |
|
|
1045
|
+
| `--seed 42` | 42 everywhere | cuDNN deterministic mode, benchmark off, deterministic algorithms | bit-exact |
|
|
1046
|
+
|
|
1047
|
+
Passing an explicit `--seed` therefore buys full determinism at a
|
|
1048
|
+
throughput cost (typically 10–20%); omit it for maximum speed:
|
|
1049
|
+
|
|
1050
|
+
```bash
|
|
1051
|
+
genml-kit-train --model convvit --seed 42 ...
|
|
1052
|
+
```
|
|
1053
|
+
|
|
1054
|
+
The train/val split and the XGBoost stage follow the user's seed: with
|
|
1055
|
+
`--seed` given they are pinned; without it they draw fresh entropy each
|
|
1056
|
+
run (HF `train_test_split(seed=None)`, XGBoost `random_state=None`).
|
|
1057
|
+
|
|
1058
|
+
Two caveats:
|
|
1059
|
+
|
|
1060
|
+
- Some CUDA ops have no deterministic kernel. Instead of aborting a
|
|
1061
|
+
long-running job, genml_kit logs a warning and proceeds (the op falls
|
|
1062
|
+
back to a non-deterministic kernel).
|
|
1063
|
+
- `float16` AMP with `GradScaler` involves non-associative reductions
|
|
1064
|
+
that can still differ run-to-run; use `bfloat16` (default on
|
|
1065
|
+
Ampere+) or disable AMP for strictly repeatable arithmetic.
|
|
1066
|
+
|
|
1067
|
+
Checkpointing is atomic: `_latest.pt` is written to a temporary file
|
|
1068
|
+
and renamed into place, so an interrupted run never leaves a truncated
|
|
1069
|
+
resume point.
|
|
1070
|
+
|
|
1071
|
+
### Fine-Tuning CLI Reference
|
|
1072
|
+
|
|
1073
|
+
| Argument | Default | Description |
|
|
1074
|
+
|---|---|---|
|
|
1075
|
+
| `--model` | `google/vit-base-patch16-224` | HuggingFace model name, local path, or custom model (e.g. `convvit`, `timm:<name>`). |
|
|
1076
|
+
| `--dataset` | required | HuggingFace dataset name or `imagefolder/PATH` for local data. |
|
|
1077
|
+
| `--image_column` | auto-detected | Explicit HF image column name. |
|
|
1078
|
+
| `--label_column` | auto-detected | Explicit HF label column name. |
|
|
1079
|
+
| `--image_size` | `448` | Augmentation crop size (processor handles final resize). |
|
|
1080
|
+
| `--epochs` | `5` | Number of training epochs. |
|
|
1081
|
+
| `--batch_size` | `32` | Batch size. |
|
|
1082
|
+
| `--lr` | `3e-5` | Peak learning rate. |
|
|
1083
|
+
| `--weight_decay` | `0.01` | Weight decay. |
|
|
1084
|
+
| `--loss` | `None` | Loss spec: registered name (`focal`, `supcon`, or the RL bundles `ppo`/`sac`/`dqn`) or a path/URL to a `.py` script defining `build_loss(**kwargs)`. `None` (default) selects the method's built-in loss. |
|
|
1085
|
+
| `--loss_args` | `{}` | JSON kwargs forwarded to the loss builder. Example: `'{"gamma": 1.0, "label_smoothing": 0.1}'`. |
|
|
1086
|
+
| `--class_multipliers` | `""` | Per-class priority multipliers. Example: `"cat=3.0,dog=1.0"`. |
|
|
1087
|
+
| `--sampler` | `none` | Training sampler: `none` (shuffle), `weighted` (WeightedRandomSampler for class imbalance), or `balanced` (equal samples per class per batch; batch_size need not divide evenly). |
|
|
1088
|
+
| `--sampler_weights` | `frequency` | Weight mode for `--sampler weighted`: `frequency` (inverse-freq), `multipliers` (--class_multipliers), or `combined` (freq × multipliers). |
|
|
1089
|
+
| `--mixup_alpha` | `0.0` | Mixup alpha (`0` = disabled; recommended: `0.2`). |
|
|
1090
|
+
| `--seed` | `None` | Explicit RNG seed: pins the train/val split and the XGBoost stage and enables full determinism (cuDNN deterministic kernels, benchmark off). Omit (default) to keep runs fast — shuffling/mixup/dropout are still seeded from `$GENML_KIT_SEED` (default 42). See [Reproducibility](#reproducibility). |
|
|
1091
|
+
| `--grad_accum_steps` | `1` | Gradient accumulation steps (effective batch = batch_size × steps). |
|
|
1092
|
+
| `--amp_dtype` | `None` | Mixed precision: `float16` or `bfloat16`. |
|
|
1093
|
+
| `--device` | auto-detect | Device: `cpu`, `cuda`, or `cuda:INDEX`. |
|
|
1094
|
+
| `--lr_group` | `None` | Per-parameter-group learning rates (repeatable). Format: `"REGEX=LR"`. |
|
|
1095
|
+
| `--llrd_decay` | `None` | Layer-wise LR decay factor per depth level. Example: `--llrd_decay 0.85`. |
|
|
1096
|
+
| `--checkpoint` | `genml_kit` | Checkpoint base path (`_latest.pt` / `_best.pt` appended). |
|
|
1097
|
+
| `--log_every` | `20` | Log every N steps. |
|
|
1098
|
+
| `--grad_monitor` | `-1` | Log gradient statistics every N steps. See [Gradient Monitor](#gradient-monitor). |
|
|
1099
|
+
| `--norm_history` | `0` | Keep last N norm snapshots for trend analysis. |
|
|
1100
|
+
| `--trend_top_n` | `10` | Show top N params in trend table. `0` = show all. |
|
|
1101
|
+
| `--grad_clip` | `1.0` | Max gradient norm for clipping. `0` disables. |
|
|
1102
|
+
| `--save_every` | `500` | Save checkpoint every N optimizer steps. 0 disables. |
|
|
1103
|
+
| `--num_workers` | `2` | DataLoader worker processes. |
|
|
1104
|
+
| `--log_level` | `INFO` | Logging level. |
|
|
1105
|
+
| `--log_targets` | `STDERR` | Comma-separated log destinations. `STDERR` logs to standard error; any other entry is a log file path (appended). Example: `STDERR,/tmp/train.log` logs to both. |
|
|
1106
|
+
| `--log_dir` | `None` | TensorBoard log directory (default: `<checkpoint_dir>/logs`). Requires the `tensorboard` package (installed by the `dev` extra). |
|
|
1107
|
+
| `--cache_dir` | `None` | HuggingFace cache directory. |
|
|
1108
|
+
| `--remote_checkpoint` | `None` | Remote URI for checkpoint sync (`gs://BUCKET/PREFIX`, `r2://BUCKET/PREFIX`, or `s3://BUCKET/PREFIX`). |
|
|
1109
|
+
| `--source_checkpoint` | `None` | Path to source checkpoint to absorb parameters from. |
|
|
1110
|
+
| `--param_rename` | `None` | Regex-based key rename patterns (`SEARCH;REPLACE`). |
|
|
1111
|
+
| `--freeze` | `None` | Regex patterns for parameters to keep trainable. All others frozen. |
|
|
1112
|
+
| `--lora` | `False` | Enable LoRA via PEFT. Requires `pip install "genml_kit[lora]"`. |
|
|
1113
|
+
| `--lora_r` | `8` | LoRA rank. |
|
|
1114
|
+
| `--lora_alpha` | `16` | LoRA alpha (scaling = `alpha / r`). |
|
|
1115
|
+
| `--lora_dropout` | `0.0` | Dropout on LoRA layers. |
|
|
1116
|
+
| `--lora_target_modules` | `None` | Comma-separated module names for LoRA (e.g. `"query,key,value"`). |
|
|
1117
|
+
| `--optimizer` | `AdamW` | `torch.optim` optimizer class name or `.py` script path. |
|
|
1118
|
+
| `--opt_arg` | `{}` | Extra optimizer kwargs (repeatable). |
|
|
1119
|
+
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path. |
|
|
1120
|
+
| `--sched_arg` | `{}` | Extra scheduler kwargs (repeatable). |
|
|
1121
|
+
| `--state_save` | `opt,sched,amp` | States to save: `opt`, `sched`, `amp`, `none`. |
|
|
1122
|
+
| `--state_load` | `opt,sched,amp` | States to restore on resume. |
|
|
1123
|
+
| `--xgboost_model` | `None` | Output path for XGBoost model (trains after PyTorch). |
|
|
1124
|
+
| `--xgb_*` | various | XGBoost hyperparameters (see `--help` for full list). |
|
|
1125
|
+
| `--model_arg` | `{}` | Override model configuration (repeatable). |
|
|
1126
|
+
| `--proc_arg` | `{}` | Override processor configuration (repeatable). |
|
|
1127
|
+
| `--train_augmentation_script` | `None` | Custom augmentation script. Must define `create_train_transform()`. |
|
|
1128
|
+
| `--grad_checkpoint` | `False` | Enable gradient checkpointing. Reduces activation memory by ~40-50% at the cost of ~25-35% more compute per step. Enables larger batch sizes. See [Gradient Checkpointing](#gradient-checkpointing). |
|
|
1129
|
+
| `--tta` | `None` | Test-Time Augmentation. `default` uses built-in 4-view transform (identity + flips). A path/URL loads an external script defining `create_tta_transform()`. Omit to disable. |
|
|
1130
|
+
|
|
1131
|
+
Training automatically resumes from an existing `_latest.pt` or `_best.pt`
|
|
1132
|
+
checkpoint if one exists at the `--checkpoint` path.
|
|
1133
|
+
|
|
1134
|
+
### Remote Checkpoint Sync (GCS / R2 / S3)
|
|
1135
|
+
|
|
1136
|
+
`--remote_checkpoint` uploads each saved checkpoint to cloud storage.
|
|
1137
|
+
Requires `pip install "genml_kit[s3]"` for `s3://` and `r2://` URIs (both
|
|
1138
|
+
use boto3), or `genml_kit[gcs]` for `gs://`. The unified `genml-kit-train`
|
|
1139
|
+
CLI accepts the flag for every pipeline/method configuration.
|
|
1140
|
+
|
|
1141
|
+
The sync also works in reverse at startup: before auto-resume, any missing
|
|
1142
|
+
`_latest.pt` / `_best.pt` is downloaded from the remote prefix (latest is
|
|
1143
|
+
tried first, matching resume precedence). A locally present checkpoint is
|
|
1144
|
+
never overwritten — the local copy always wins — and connection or
|
|
1145
|
+
credential failures degrade to a warning instead of aborting startup.
|
|
1146
|
+
|
|
1147
|
+
**AWS S3** — credentials come from the standard environment variables:
|
|
1148
|
+
|
|
1149
|
+
```bash
|
|
1150
|
+
%env AWS_ACCESS_KEY_ID=AKIA...
|
|
1151
|
+
%env AWS_SECRET_ACCESS_KEY=...
|
|
1152
|
+
%env AWS_SESSION_TOKEN=... # only for temporary (STS/SSO) credentials
|
|
1153
|
+
%env AWS_DEFAULT_REGION=us-east-1
|
|
1154
|
+
|
|
1155
|
+
--remote_checkpoint s3://my-bucket/genml_kit/convvit_ijepa
|
|
1156
|
+
```
|
|
1157
|
+
|
|
1158
|
+
Permanent IAM-user keys need only the access/secret pair; the session
|
|
1159
|
+
token is picked up automatically when present. When the key variables
|
|
1160
|
+
are unset, boto3's default credential chain applies (IAM instance role,
|
|
1161
|
+
`~/.aws/credentials`, SSO cache).
|
|
1162
|
+
|
|
1163
|
+
**Cloudflare R2** — the S3-compatible API with R2 credentials:
|
|
1164
|
+
|
|
1165
|
+
```bash
|
|
1166
|
+
%env R2_ENDPOINT_URL=https://<account_id>.r2.cloudflarestorage.com
|
|
1167
|
+
%env R2_ACCESS_KEY_ID=...
|
|
1168
|
+
%env R2_SECRET_ACCESS_KEY=...
|
|
1169
|
+
|
|
1170
|
+
--remote_checkpoint r2://my-bucket/genml_kit/convvit_ijepa
|
|
1171
|
+
```
|
|
1172
|
+
|
|
1173
|
+
### LoRA Details
|
|
1174
|
+
|
|
1175
|
+
Low-Rank Adaptation freezes the pre-trained backbone and injects small
|
|
1176
|
+
trainable low-rank matrices into attention layers.
|
|
1177
|
+
Reference: Hu et al., [LoRA: Low-Rank Adaptation of Large Language Models](https://arxiv.org/abs/2106.09685), ICLR 2022.
|
|
1178
|
+
|
|
1179
|
+
Instead of updating a weight matrix $`W`$ directly, LoRA keeps it frozen and
|
|
1180
|
+
adds a low-rank product to its output:
|
|
1181
|
+
|
|
1182
|
+
$$
|
|
1183
|
+
\large
|
|
1184
|
+
W' = W + \Delta W, \qquad \Delta W = \frac{\alpha}{r} \cdot B A
|
|
1185
|
+
$$
|
|
1186
|
+
|
|
1187
|
+
where:
|
|
1188
|
+
|
|
1189
|
+
- $`W \in \mathbb{R}^{d \times k}`$ — the frozen pre-trained weight matrix;
|
|
1190
|
+
- $`A \in \mathbb{R}^{r \times k}`$, $`B \in \mathbb{R}^{d \times r}`$ — the two
|
|
1191
|
+
small trainable matrices ($`BA`$ is a matrix product);
|
|
1192
|
+
- $`r`$ — the rank (`--lora_r`), much smaller than $`d`$ or $`k`$, which is
|
|
1193
|
+
what makes $`\Delta W`$ cheap: instead of $`d \times k`$ weights it trains
|
|
1194
|
+
only $`r(d + k)`$;
|
|
1195
|
+
- $`\alpha`$ — the scale factor (`--lora_alpha`); the effective step size of
|
|
1196
|
+
the update is $`\alpha / r`$.
|
|
1197
|
+
|
|
1198
|
+
Only $`A`$, $`B`$ (and the classifier head) receive gradients.
|
|
1199
|
+
|
|
1200
|
+
| r | alpha | alpha/r | Use case |
|
|
1201
|
+
|---|---|---|---|
|
|
1202
|
+
| 8 | 16 | 2.0 | Conservative, very few params |
|
|
1203
|
+
| 16 | 32 | 2.0 | Good default for medium datasets |
|
|
1204
|
+
| 16 | 64 | 4.0 | Larger updates (helpful under bfloat16) |
|
|
1205
|
+
| 32 | 64 | 2.0 | More capacity, same scaling |
|
|
1206
|
+
|
|
1207
|
+
LoRA can be combined with a custom classifier:
|
|
1208
|
+
|
|
1209
|
+
```bash
|
|
1210
|
+
genml-kit-train \
|
|
1211
|
+
--model cls_model_wrapper:facebook/dinov2-with-registers-large \
|
|
1212
|
+
--model_arg classifier=cls_attention:num_encoder_layers=2 \
|
|
1213
|
+
--lora --lora_r 16 --lora_alpha 32 \
|
|
1214
|
+
--freeze 'classifier\.(head|pool|encoder)' \
|
|
1215
|
+
--lr_group 'backbone.*=1e-5' 'classifier.*=3e-4' \
|
|
1216
|
+
...
|
|
1217
|
+
```
|
|
1218
|
+
|
|
1219
|
+
### Cross-Dataset Resume
|
|
1220
|
+
|
|
1221
|
+
Switch from one dataset to another while keeping backbone weights:
|
|
1222
|
+
|
|
1223
|
+
```bash
|
|
1224
|
+
genml-kit-train --model facebook/convnextv2-base-22k-224 \
|
|
1225
|
+
--dataset my-org/other-labeled-images \
|
|
1226
|
+
--label_column category \
|
|
1227
|
+
--checkpoint genml_kit \
|
|
1228
|
+
--state_load none \
|
|
1229
|
+
--epochs 10 \
|
|
1230
|
+
--batch_size 16 \
|
|
1231
|
+
--lr 3e-5 \
|
|
1232
|
+
--mixup_alpha 0.2 \
|
|
1233
|
+
--amp_dtype bfloat16
|
|
1234
|
+
```
|
|
1235
|
+
|
|
1236
|
+
Backbone weights load via `strict=False`; the classifier head (different
|
|
1237
|
+
`num_classes`) is reinitialised. `--state_load none` prevents carrying over
|
|
1238
|
+
old optimizer/scheduler states.
|
|
1239
|
+
|
|
1240
|
+
### Gradient Checkpointing
|
|
1241
|
+
|
|
1242
|
+
Training deep vision transformers is memory-intensive: the attention maps for
|
|
1243
|
+
EVA02-base at 448×448 resolution consume ~4.6 GB across 12 layers, which
|
|
1244
|
+
limits the maximum batch size even on a 24 GB GPU. Gradient checkpointing
|
|
1245
|
+
trades compute for memory by discarding intermediate activations during the
|
|
1246
|
+
forward pass and recomputing them during the backward pass.
|
|
1247
|
+
|
|
1248
|
+
```bash
|
|
1249
|
+
genml-kit-train \
|
|
1250
|
+
--model timm:eva02_base_patch14_448.mim_in22k_ft_in22k_in1k \
|
|
1251
|
+
--batch_size 32 \
|
|
1252
|
+
--grad_accum_steps 2 \
|
|
1253
|
+
--grad_checkpoint \
|
|
1254
|
+
--amp_dtype bfloat16 \
|
|
1255
|
+
...
|
|
1256
|
+
```
|
|
1257
|
+
|
|
1258
|
+
**What it does:** Each transformer block stores only its input and output.
|
|
1259
|
+
The intra-block activations (attention maps, FFN intermediates) are
|
|
1260
|
+
recomputed on-demand during the backward pass. This reduces peak activation
|
|
1261
|
+
memory by ~40-50%.
|
|
1262
|
+
|
|
1263
|
+
**What it costs:** The recomputation adds ~25-35% more compute per training
|
|
1264
|
+
step. Whether this translates to a net throughput gain or loss depends on
|
|
1265
|
+
your GPU:
|
|
1266
|
+
- **Memory-bound GPU** (VRAM-limited, compute underutilized): larger batches
|
|
1267
|
+
fill the GPU's compute capacity → net throughput **increase** of ~50-60%.
|
|
1268
|
+
- **Compute-saturated GPU** (TFLOPS at 100%): every extra FLOP adds to step
|
|
1269
|
+
time → net throughput **decrease** of ~20-25%, though the larger batch can
|
|
1270
|
+
still improve training stability and final accuracy.
|
|
1271
|
+
|
|
1272
|
+
**Backend support:** Enabled automatically for all model backends — timm,
|
|
1273
|
+
HuggingFace, ConvViT, and UVito — via native APIs or per-block
|
|
1274
|
+
`torch.utils.checkpoint` in the transformer loops.
|
|
1275
|
+
|
|
1276
|
+
---
|
|
1277
|
+
|
|
1278
|
+
## Inference Guide
|
|
1279
|
+
|
|
1280
|
+
Run inference on individual images:
|
|
1281
|
+
|
|
1282
|
+
```bash
|
|
1283
|
+
genml-kit-infer --model facebook/convnextv2-base-22k-224 \
|
|
1284
|
+
--checkpoint genml_kit_best.pt \
|
|
1285
|
+
path/to/image.jpg path/to/other_image.png
|
|
1286
|
+
```
|
|
1287
|
+
|
|
1288
|
+
Output is JSON with per-class probabilities:
|
|
1289
|
+
|
|
1290
|
+
```json
|
|
1291
|
+
{
|
|
1292
|
+
"source": "image.jpg",
|
|
1293
|
+
"predictions": [
|
|
1294
|
+
{"label": "golden_retriever", "probability": 0.435},
|
|
1295
|
+
{"label": "labrador", "probability": 0.281}
|
|
1296
|
+
]
|
|
1297
|
+
}
|
|
1298
|
+
```
|
|
1299
|
+
|
|
1300
|
+
### XGBoost And Test-Time Augmentation
|
|
1301
|
+
|
|
1302
|
+
The neural classifier makes decisions through its head. The XGBoost option
|
|
1303
|
+
takes the encoder representation $`h = f_\theta(x)`$ instead and fits an
|
|
1304
|
+
ensemble of decision trees to those vectors.
|
|
1305
|
+
Reference: Chen & Guestrin, [XGBoost: A Scalable Tree Boosting System](https://arxiv.org/abs/1603.02754), KDD 2016.
|
|
1306
|
+
A tree partitions feature space with rules such as $`h_{37} < t`$ (comparing
|
|
1307
|
+
the 37th feature of $`h`$ against a learned threshold $`t`$); boosting adds
|
|
1308
|
+
trees sequentially so each new tree focuses on errors left by previous trees.
|
|
1309
|
+
This can work well when the dataset is small
|
|
1310
|
+
and the representation is already useful, but it is not guaranteed to beat the
|
|
1311
|
+
neural head. Use validation data to choose tree depth, number of rounds, and
|
|
1312
|
+
ensemble weight.
|
|
1313
|
+
|
|
1314
|
+
Test-time augmentation (TTA) runs the same image through several plausible
|
|
1315
|
+
views and averages the probability vectors:
|
|
1316
|
+
|
|
1317
|
+
$$
|
|
1318
|
+
\large
|
|
1319
|
+
\bar{p}(y \mid x) = \frac{1}{K} \sum_{k=1}^{K} p(y \mid T_k(x))
|
|
1320
|
+
$$
|
|
1321
|
+
|
|
1322
|
+
where $`T_1, \dots, T_K`$ are the $`K`$ test-time transformations (e.g. the
|
|
1323
|
+
original image plus flipped copies) and $`\bar{p}`$ is their averaged
|
|
1324
|
+
probability vector: each transformed copy is classified independently, and the
|
|
1325
|
+
final prediction averages the per-view class probabilities. The
|
|
1326
|
+
transformations should preserve the label semantics. Horizontal flips
|
|
1327
|
+
are usually safer than orientation-specific crops; verify that an augmentation
|
|
1328
|
+
does not erase the cue that distinguishes the classes.
|
|
1329
|
+
|
|
1330
|
+
### Inference CLI Reference
|
|
1331
|
+
|
|
1332
|
+
| Flag | Default | Description |
|
|
1333
|
+
|---|---|---|
|
|
1334
|
+
| `--model` | (required) | HuggingFace model name or custom model. |
|
|
1335
|
+
| `--checkpoint` | (required) | Path to state dict or wrapped checkpoint. |
|
|
1336
|
+
| `--top_k` | `None` | Show top-K predictions; omit for all classes. |
|
|
1337
|
+
| `--output` | `None` | Write JSON results to file. |
|
|
1338
|
+
| `--device` | `None` | Force PyTorch device (`cuda`, `cpu`). Auto-detected if omitted. |
|
|
1339
|
+
| `--cache_dir` | `None` | HuggingFace cache directory. |
|
|
1340
|
+
| `--xgboost_model` | `None` | XGBoost model path. If provided, runs XGBoost alongside PyTorch. |
|
|
1341
|
+
|
|
1342
|
+
Wrapped checkpoints (containing `model_state_dict` and metadata) are preferred
|
|
1343
|
+
over raw state dictionaries, which produce a metadata warning.
|
|
1344
|
+
|
|
1345
|
+
### XGBoost Inference
|
|
1346
|
+
|
|
1347
|
+
When `--xgboost_model` is provided, the output includes both predictions:
|
|
1348
|
+
|
|
1349
|
+
```json
|
|
1350
|
+
{
|
|
1351
|
+
"source": "image.jpg",
|
|
1352
|
+
"predictions": [
|
|
1353
|
+
{"label": "golden_retriever", "probability": 0.435}
|
|
1354
|
+
],
|
|
1355
|
+
"xgboost_predictions": [
|
|
1356
|
+
{"label": "golden_retriever", "probability": 0.612}
|
|
1357
|
+
]
|
|
1358
|
+
}
|
|
1359
|
+
```
|
|
1360
|
+
|
|
1361
|
+
---
|
|
1362
|
+
|
|
1363
|
+
## Tips & Pitfalls
|
|
1364
|
+
|
|
1365
|
+
### `--checkpoint` And `--source_checkpoint` Are Different
|
|
1366
|
+
|
|
1367
|
+
`--checkpoint` names the output prefix used for saving and resuming the current
|
|
1368
|
+
training run. `--source_checkpoint` imports weights from another run before
|
|
1369
|
+
the new optimizer is created. When moving a SupCon encoder into classification,
|
|
1370
|
+
use:
|
|
1371
|
+
|
|
1372
|
+
```bash
|
|
1373
|
+
genml-kit-train \
|
|
1374
|
+
--checkpoint /content/eva02_finetune \
|
|
1375
|
+
--source_checkpoint /content/eva02_supcon_latest.pt \
|
|
1376
|
+
--param_rename 'encoder\\.model\\.(.*);model.$1' \
|
|
1377
|
+
...
|
|
1378
|
+
```
|
|
1379
|
+
|
|
1380
|
+
The rename is needed because the pre-training wrapper stores backbone keys
|
|
1381
|
+
under `encoder.model.*`, whereas the fine-tuning model stores them under
|
|
1382
|
+
`model.*`. The SupCon `projection.*` keys are expected to remain unused, and
|
|
1383
|
+
new classifier-head keys are expected to be missing from the source checkpoint.
|
|
1384
|
+
Those messages indicate a successful partial transfer if the backbone keys
|
|
1385
|
+
are matched.
|
|
1386
|
+
|
|
1387
|
+
### Interpreting A SupCon Plateau
|
|
1388
|
+
|
|
1389
|
+
SupCon loss is not expected to approach zero. If an anchor has k-1 positives and
|
|
1390
|
+
positives become much more similar than negatives, a useful reference value is
|
|
1391
|
+
approximately `-log(k-1)`. This is only a diagnostic approximation: class
|
|
1392
|
+
counts may vary, the sampler may repeat examples, and temperature affects
|
|
1393
|
+
optimization. Judge the loss together with embedding quality and downstream
|
|
1394
|
+
validation metrics. A plateau near this reference can mean convergence; a
|
|
1395
|
+
plateau well above it can mean too few positives, a learning-rate problem, or
|
|
1396
|
+
labels that are not being passed correctly.
|
|
1397
|
+
|
|
1398
|
+
### A Practical Debugging Order
|
|
1399
|
+
|
|
1400
|
+
1. Confirm that images and labels are valid and that each SupCon batch has
|
|
1401
|
+
repeated classes.
|
|
1402
|
+
2. Confirm that the loss changes when the learning rate changes and inspect
|
|
1403
|
+
`--grad_monitor` for zero or exploding gradients.
|
|
1404
|
+
3. Check whether pre-trained weights loaded by reading the alignment report,
|
|
1405
|
+
not just the final epoch number.
|
|
1406
|
+
4. Compare against a simple head-only or full-fine-tuning baseline before
|
|
1407
|
+
adding LLRD, LoRA, Mixup, focal loss, and class multipliers together.
|
|
1408
|
+
5. Select the checkpoint using a validation metric appropriate to the
|
|
1409
|
+
objective, rather than training loss alone.
|
|
1410
|
+
|
|
1411
|
+
### Low GPU Utilization (`util=0` In The Training Log)
|
|
1412
|
+
|
|
1413
|
+
The periodic `GPU:` line reports memory in MiB and SM utilization as a
|
|
1414
|
+
percentage sampled at log time (`mem=allocated/totalMiB (pct) res=…MiB
|
|
1415
|
+
util=…%`). `mem` counts live tensors, `res` is the caching allocator's
|
|
1416
|
+
pool, and `util` is one instant — not an average — so an isolated 0 is
|
|
1417
|
+
normal. A *persistent* pattern of `util=0` at every log line, with
|
|
1418
|
+
`mem`/`res` byte-identical between samples, means the GPU finished the
|
|
1419
|
+
last batch and is idle waiting for the next one: the run is **input
|
|
1420
|
+
bound**, not compute bound. The throughput line (`img/s=`) is the ground
|
|
1421
|
+
truth for step time.
|
|
1422
|
+
|
|
1423
|
+
Common causes and fixes, in order of likelihood:
|
|
1424
|
+
|
|
1425
|
+
- **Too few DataLoader workers.** JPEG decode + augmentation is CPU work;
|
|
1426
|
+
with `--num_workers 4` a 4090 can easily outrun the pipeline at
|
|
1427
|
+
448². Check `nproc` on the machine and raise `--num_workers` toward
|
|
1428
|
+
the core count — `img/s` should scale until the GPU saturates
|
|
1429
|
+
(a `util` that now sits consistently above ~90% means you are done).
|
|
1430
|
+
- **Expensive decode.** Full-resolution JPEGs dominate decode cost.
|
|
1431
|
+
Pre-resizing the dataset offline (e.g. ~512px long side) cuts decode
|
|
1432
|
+
work several-fold and often doubles throughput on small VRAM budgets.
|
|
1433
|
+
- **Slow storage.** A network volume (S3/R2 mount, NFS) adds latency per
|
|
1434
|
+
image; prefer local NVMe for the dataset.
|
|
1435
|
+
- **`--seed`.** It enables cuDNN deterministic mode and disables
|
|
1436
|
+
benchmarking; expect slower steps by design. Omit it when you do not
|
|
1437
|
+
need bit-exact runs.
|
|
1438
|
+
|
|
1439
|
+
Fine-tuning runs on frozen-backbone models need very little GPU memory
|
|
1440
|
+
(`mem` is typically a few hundred MiB), so a low `mem` number is normal
|
|
1441
|
+
there and not a sign of a problem. Loader prefetch (`prefetch_factor=4`
|
|
1442
|
+
per worker) and `cudnn.benchmark=True` (default; disabled by an explicit
|
|
1443
|
+
`--seed`) already help absorb jitter and speed up the GPU side.
|
|
1444
|
+
|
|
1445
|
+
---
|
|
1446
|
+
|
|
1447
|
+
## Gradient Monitor
|
|
1448
|
+
|
|
1449
|
+
`--grad_monitor N` logs a per-parameter gradient report every *N* training
|
|
1450
|
+
steps. This helps diagnose training instability (exploding / vanishing
|
|
1451
|
+
gradients, imbalanced parameter updates) before it shows up in the loss.
|
|
1452
|
+
|
|
1453
|
+
### Summary Line
|
|
1454
|
+
|
|
1455
|
+
```
|
|
1456
|
+
[Step 29400] Gradient Report: 202 params | grad_rms: mean=5.68e-01 max=3.41e+00 min=1.66e-05 | grad/param: mean=2.94e-01
|
|
1457
|
+
```
|
|
1458
|
+
|
|
1459
|
+
All norms are **RMS** (root mean square): the $`L_2`$ norm of the tensor
|
|
1460
|
+
divided by $`\sqrt{N}`$, where $`N`$ is the number of elements (`numel`):
|
|
1461
|
+
|
|
1462
|
+
$$
|
|
1463
|
+
\large
|
|
1464
|
+
\mathrm{RMS}(v) = \frac{\lVert v \rVert_2}{\sqrt{N}} = \sqrt{\frac{1}{N} \sum_{n=1}^{N} v_n^2}
|
|
1465
|
+
$$
|
|
1466
|
+
|
|
1467
|
+
Here $`v`$ is the tensor (a parameter or its gradient) flattened to $`N`$
|
|
1468
|
+
values $`v_1, \dots, v_N`$, and $`\lVert v \rVert_2 = \sqrt{\sum_n v_n^2}`$ is
|
|
1469
|
+
its Euclidean norm. Dividing by $`\sqrt{N}`$ makes the measure independent of
|
|
1470
|
+
tensor shape and directly comparable across parameters of different sizes.
|
|
1471
|
+
|
|
1472
|
+
| Field | Meaning |
|
|
1473
|
+
|---|---|
|
|
1474
|
+
| `params` | Total number of trainable parameters. |
|
|
1475
|
+
| `grad_rms: mean/max/min` | RMS of the gradient tensor for each parameter, then aggregated. `max` is the single most aggressive gradient — the one most likely to cause instability. |
|
|
1476
|
+
| `grad/param: mean` | Average gradient-to-parameter ratio. Healthy: < 0.1. Concerning: > 1.0. |
|
|
1477
|
+
|
|
1478
|
+
### Per-Parameter Columns
|
|
1479
|
+
|
|
1480
|
+
| Column | Symbol | What to look for |
|
|
1481
|
+
|---|---|---|
|
|
1482
|
+
| **g_rms** | $`\mathrm{RMS}(\nabla_W \mathcal{L})`$ | Compare across params. One param with g_rms 100× higher is a problem. |
|
|
1483
|
+
| **p_rms** | $`\mathrm{RMS}(W)`$ | Per-element scale context. With `std=0.02` init, expect ~0.02. |
|
|
1484
|
+
| **g/p** | $`\lVert \nabla_W \mathcal{L} \rVert_2 / (\lVert W \rVert_2 + \varepsilon)`$ | **Most useful column.** Healthy: < 0.1. Concerning: > 1.0 (update overshoots). Dangerous: > 5.0. |
|
|
1485
|
+
| **g_max** | $`\max_n \lvert \partial \mathcal{L} / \partial W_n \rvert`$ | Highlights individual neurons with extreme gradients. |
|
|
1486
|
+
| **sparse** | `% zero` | High sparsity (> 50%) = most neurons not receiving signal. |
|
|
1487
|
+
| **status** | | `OK` = healthy. `STL` = stalled. `OVF` = exploding. `IMB` = imbalanced. `GPR` = high g/p ratio. |
|
|
1488
|
+
|
|
1489
|
+
In this table $`W`$ denotes one parameter tensor, $`\nabla_W \mathcal{L}`$ its
|
|
1490
|
+
gradient (the vector of partial derivatives of the loss with respect to each
|
|
1491
|
+
element of $`W`$), and $`\varepsilon`$ a tiny constant that only prevents
|
|
1492
|
+
division by zero for parameters that are exactly zero.
|
|
1493
|
+
|
|
1494
|
+
### Reading The Report
|
|
1495
|
+
|
|
1496
|
+
**Healthy training:** g/p < 0.1, `grad/param: mean` in 0.01–0.1 range, g_rms within ~10× across layers.
|
|
1497
|
+
|
|
1498
|
+
**Exploding gradients:** One or more params with `OVF`, g/p > 5.0, `grad_rms: max` >> `mean`. Fix: lower LR, add `--grad_clip 1.0`, or use warmup.
|
|
1499
|
+
|
|
1500
|
+
**Vanishing gradients:** Many params with `STL`, g_rms near 1e-7, high sparsity. Fix: increase LR, check for dead neurons.
|
|
1501
|
+
|
|
1502
|
+
**Imbalanced updates:** Some params `IMB`, large g/p disparity between layers. Fix: use `--lr_group` for different rates, or freeze the dominant component.
|
|
1503
|
+
|
|
1504
|
+
### Norm Trend History
|
|
1505
|
+
|
|
1506
|
+
`--norm_history N` (requires `--grad_monitor`) keeps the last N snapshots
|
|
1507
|
+
per parameter. A trend summary table is appended to each report showing
|
|
1508
|
+
direction (`UP`/`DOWN`/`---`), percentage change, and min/max values.
|
|
1509
|
+
|
|
1510
|
+
---
|
|
1511
|
+
|
|
1512
|
+
## Custom Losses
|
|
1513
|
+
|
|
1514
|
+
Losses are resolved through the `LOSSES` registry
|
|
1515
|
+
(`genml_kit.losses.registry.load_loss`). A loss spec is either a
|
|
1516
|
+
registered name (`focal`, `supcon`, or the RL bundles `ppo`, `sac`,
|
|
1517
|
+
`dqn`) or a path/URL to a `.py` script defining `build_loss(**kwargs)`:
|
|
1518
|
+
|
|
1519
|
+
```python
|
|
1520
|
+
# my_loss.py
|
|
1521
|
+
import torch.nn as nn
|
|
1522
|
+
|
|
1523
|
+
class MyLoss(nn.Module):
|
|
1524
|
+
def __init__(self, alpha=1.0):
|
|
1525
|
+
super().__init__()
|
|
1526
|
+
self.alpha = alpha
|
|
1527
|
+
|
|
1528
|
+
def forward(self, logits, targets):
|
|
1529
|
+
return self.alpha * nn.functional.cross_entropy(logits, targets)
|
|
1530
|
+
|
|
1531
|
+
def build_loss(alpha=1.0, **kwargs):
|
|
1532
|
+
"""Protocol: return a callable (logits, targets) -> tensor."""
|
|
1533
|
+
return MyLoss(alpha=alpha)
|
|
1534
|
+
```
|
|
1535
|
+
|
|
1536
|
+
```bash
|
|
1537
|
+
genml-kit-train --model google/vit-base-patch16-224 \
|
|
1538
|
+
--loss ./my_loss.py --loss_args '{"alpha": 2.0}' ...
|
|
1539
|
+
```
|
|
1540
|
+
|
|
1541
|
+
**RL losses** work the same way but return a *bundle* (dict of
|
|
1542
|
+
callables); the script's entries are merged over the method's built-in
|
|
1543
|
+
bundle (the method passes its own name as the merge base: `ppo`, `sac`,
|
|
1544
|
+
or `dqn`), so a script may override just one loss and inherit the rest:
|
|
1545
|
+
|
|
1546
|
+
```python
|
|
1547
|
+
# my_ppo_entropy.py
|
|
1548
|
+
def build_loss(**kwargs):
|
|
1549
|
+
return {"entropy": lambda ent: ent.mean() * 2.0} # override only entropy
|
|
1550
|
+
```
|
|
1551
|
+
|
|
1552
|
+
```bash
|
|
1553
|
+
genml-kit-train --method ppo --loss ./my_ppo_entropy.py ...
|
|
1554
|
+
```
|
|
1555
|
+
|
|
1556
|
+
The built-in bundles are merged over (`ppo`: `policy`/`value`/`entropy`;
|
|
1557
|
+
`sac`: `q`/`policy`/`alpha`; `dqn`: `td`). The `--loss_args` JSON
|
|
1558
|
+
kwargs are forwarded to `build_loss(**kwargs)`.
|
|
1559
|
+
|
|
1560
|
+
## Custom Models
|
|
1561
|
+
|
|
1562
|
+
genml_kit supports any HuggingFace `AutoModelForImageClassification` model,
|
|
1563
|
+
any timm model via `timm:<name>`, and custom architectures registered in
|
|
1564
|
+
`genml_kit.models`.
|
|
1565
|
+
|
|
1566
|
+
### Built-In Custom Models
|
|
1567
|
+
|
|
1568
|
+
| Name | Description | `--model` value |
|
|
1569
|
+
|---|---|---|
|
|
1570
|
+
| timm | Any model from [timm](https://github.com/huggingface/pytorch-image-models) | `timm:<model_name>` |
|
|
1571
|
+
| ConvViT | Multi-block conv stem + ViT encoder with CLS-guided attention pooling | `convvit` |
|
|
1572
|
+
| UVito | Frozen SMP encoder + learnable patch projection + Transformer encoder | `uvito` |
|
|
1573
|
+
| ClsModelWrapper | HuggingFace backbone + custom classifier head | `cls_model_wrapper:<hf_name>` |
|
|
1574
|
+
| ContrastiveEncoder | Backbone + projection head for contrastive pre-training | `contrastive_encoder:<hf_name>` |
|
|
1575
|
+
|
|
1576
|
+
### Adding A Custom Model
|
|
1577
|
+
|
|
1578
|
+
1. Create `genml_kit/models/{name}/` with `model.py`, `processor.py`,
|
|
1579
|
+
`loader.py`, and `__init__.py`.
|
|
1580
|
+
2. Add the import to `genml_kit/models/__init__.py`.
|
|
1581
|
+
3. The model must expose `.forward(pixel_values=images)` → object with `.logits`,
|
|
1582
|
+
and `config.id2label` / `config.label2id`.
|
|
1583
|
+
4. CLI overrides via `--model_arg KEY=VALUE` are forwarded to the loader.
|
|
1584
|
+
|
|
1585
|
+
### External Model Scripts
|
|
1586
|
+
|
|
1587
|
+
Instead of registering a model inside the source tree, you can point
|
|
1588
|
+
`--model` at a `.py` file (or HTTP/HTTPS URL). The script must define
|
|
1589
|
+
`create_model`:
|
|
1590
|
+
|
|
1591
|
+
```python
|
|
1592
|
+
# my_model.py
|
|
1593
|
+
import torch.nn as nn
|
|
1594
|
+
|
|
1595
|
+
class Tiny(nn.Module):
|
|
1596
|
+
def __init__(self, num_labels, hidden=128):
|
|
1597
|
+
super().__init__()
|
|
1598
|
+
self.net = nn.Sequential(nn.Linear(784, hidden), nn.ReLU(),
|
|
1599
|
+
nn.Linear(hidden, num_labels))
|
|
1600
|
+
|
|
1601
|
+
def forward(self, pixel_values):
|
|
1602
|
+
return pixel_values
|
|
1603
|
+
|
|
1604
|
+
def create_model(num_labels, image_size=224, id2label=None, label2id=None,
|
|
1605
|
+
checkpoint_path=None, device="cpu", **kwargs):
|
|
1606
|
+
"""Protocol: return a torch.nn.Module."""
|
|
1607
|
+
return Tiny(num_labels=num_labels, hidden=kwargs.get("hidden", 128))
|
|
1608
|
+
```
|
|
1609
|
+
|
|
1610
|
+
```bash
|
|
1611
|
+
genml-kit-train --model ./my_model.py --loss focal ...
|
|
1612
|
+
```
|
|
1613
|
+
|
|
1614
|
+
The script is **fully responsible for weight loading**: when
|
|
1615
|
+
`checkpoint_path` is provided (inference), call
|
|
1616
|
+
`genml_kit.io.checkpointing.load_checkpoint_weights(checkpoint_path, model, ...)`
|
|
1617
|
+
inside `create_model`, mirroring the ConvViT/UViTO loaders. In training
|
|
1618
|
+
`checkpoint_path` is `None` (random init) and the global `--source_checkpoint`
|
|
1619
|
+
/ `--resume` paths apply uniformly after `create_model` returns.
|
|
1620
|
+
|
|
1621
|
+
An optional `create_processor(image_size=224, **kwargs)` may be defined
|
|
1622
|
+
for the processor; it must return an object exposing `image_mean` /
|
|
1623
|
+
`image_std`.
|
|
1624
|
+
|
|
1625
|
+
### Custom Classifiers
|
|
1626
|
+
|
|
1627
|
+
`ClsModelWrapper` lets you replace the default HF classification head:
|
|
1628
|
+
|
|
1629
|
+
```python
|
|
1630
|
+
class Classifier(nn.Module):
|
|
1631
|
+
def __init__(self, num_labels, hidden_size, **kwargs):
|
|
1632
|
+
super().__init__()
|
|
1633
|
+
self.head = nn.Linear(hidden_size, num_labels)
|
|
1634
|
+
|
|
1635
|
+
def forward(self, hidden_states): # (B, N, D) tensor
|
|
1636
|
+
features = hidden_states[:, 0] # CLS token
|
|
1637
|
+
return self.head(features)
|
|
1638
|
+
|
|
1639
|
+
def extract_features(self, hidden_states): # (B, N, D) → (B, D)
|
|
1640
|
+
return hidden_states[:, 0]
|
|
1641
|
+
```
|
|
1642
|
+
|
|
1643
|
+
The `extract_features` method is used by `--xgboost_model` for XGBoost
|
|
1644
|
+
training on backbone features.
|
|
1645
|
+
|
|
1646
|
+
The head is selected through `--model_arg`, not a dedicated flag:
|
|
1647
|
+
|
|
1648
|
+
```bash
|
|
1649
|
+
--model cls_model_wrapper:<hf_name> --model_arg classifier=mlp:hidden=512,dropout=0.3
|
|
1650
|
+
```
|
|
1651
|
+
|
|
1652
|
+
The `classifier` value is an inline spec: the head name (a registered
|
|
1653
|
+
built-in — `mlp`, `cls_attention` — or a path to a `.py` file defining a
|
|
1654
|
+
`Classifier` class), optionally followed by `:key=value,...` kwargs. Values
|
|
1655
|
+
are parsed as int/float/bool, and bracketed lists such as
|
|
1656
|
+
`cls_slice=(0, 1)` work. A head is required: `cls_model_wrapper` fails fast
|
|
1657
|
+
with a clear message when `classifier` is missing. Models that embed their
|
|
1658
|
+
own head (`convvit`, `uvito`, `timm:<name>`) do not accept this kwarg — for
|
|
1659
|
+
a custom head on a HuggingFace backbone use `cls_model_wrapper` instead.
|
|
1660
|
+
`genml-kit-infer` needs the same `--model_arg classifier=...` to rebuild the
|
|
1661
|
+
head before loading the checkpoint weights.
|
|
1662
|
+
|
|
1663
|
+
---
|
|
1664
|
+
|
|
1665
|
+
## Reinforcement Learning
|
|
1666
|
+
|
|
1667
|
+
genml_kit ships a self-contained reinforcement-learning (RL) toolkit built on
|
|
1668
|
+
Gymnasium. The companion document
|
|
1669
|
+
[`rl/README.md`](rl/README.md) is a from-the-ground-up *mathematics* tutorial
|
|
1670
|
+
(Bellman equations, TD, Q-learning, GAE, PPO, SAC); this section is the
|
|
1671
|
+
practical *usage* guide.
|
|
1672
|
+
|
|
1673
|
+
### Installation
|
|
1674
|
+
|
|
1675
|
+
RL support is an optional extra:
|
|
1676
|
+
|
|
1677
|
+
```
|
|
1678
|
+
pip install 'genml_kit[rl]'
|
|
1679
|
+
```
|
|
1680
|
+
|
|
1681
|
+
### Supported algorithms
|
|
1682
|
+
|
|
1683
|
+
| `--method` | Algorithm | Action space | Key files |
|
|
1684
|
+
|---|---|---|---|
|
|
1685
|
+
| `dqn` | Double-DQN with epsilon-greedy, optional dueling + n-step + PER | Discrete | `genml_kit/methods/rl_dqn.py` |
|
|
1686
|
+
| `ppo` | On-policy PPO with GAE, clipped surrogate, entropy bonus | Discrete or continuous | `genml_kit/methods/rl_ppo.py` |
|
|
1687
|
+
| `sac` | Soft Actor-Critic, twin critics, reparameterized policy, auto-alpha | Continuous only | `genml_kit/methods/rl_sac.py` |
|
|
1688
|
+
|
|
1689
|
+
### Quick start
|
|
1690
|
+
|
|
1691
|
+
```
|
|
1692
|
+
# DQN on CartPole (discrete)
|
|
1693
|
+
genml-kit-train --pipeline rl --method dqn --env_id CartPole-v1 \
|
|
1694
|
+
--epochs 20 --batch_size 64
|
|
1695
|
+
|
|
1696
|
+
# PPO on CartPole (discrete)
|
|
1697
|
+
genml-kit-train --pipeline rl --method ppo --env_id CartPole-v1 \
|
|
1698
|
+
--epochs 20 --ppo_rollout_len 2048
|
|
1699
|
+
|
|
1700
|
+
# SAC on a continuous env (e.g. LunarLanderContinuous-v3)
|
|
1701
|
+
genml-kit-train --pipeline rl --method sac --env_id LunarLanderContinuous-v3 \
|
|
1702
|
+
--epochs 20
|
|
1703
|
+
```
|
|
1704
|
+
|
|
1705
|
+
All RL methods use `eval_return` (mean undiscounted return over
|
|
1706
|
+
`--eval_episodes` episodes, default 5) as the checkpoint-selection metric.
|
|
1707
|
+
The trainer bypasses the DataLoader: it steps the environment directly and
|
|
1708
|
+
samples from a replay buffer (DQN/SAC) or re-uses a rollout buffer (PPO).
|
|
1709
|
+
|
|
1710
|
+
### Custom RL models
|
|
1711
|
+
|
|
1712
|
+
Every RL method resolves its network through the same model registry as the
|
|
1713
|
+
rest of the framework: `--model` selects a registered factory and
|
|
1714
|
+
`--model_arg key=value` forwards constructor kwargs.
|
|
1715
|
+
|
|
1716
|
+
```
|
|
1717
|
+
# DQN: dueling Q-head (also the default for --dqn_dueling)
|
|
1718
|
+
genml-kit-train --pipeline rl --method dqn --model rl/qnet_dueling
|
|
1719
|
+
|
|
1720
|
+
# DQN: configure the MLP width via --model_arg
|
|
1721
|
+
genml-kit-train --pipeline rl --method dqn \
|
|
1722
|
+
--model rl/qnet --model_arg hidden_dims=[512,256]
|
|
1723
|
+
|
|
1724
|
+
# PPO: wider actor-critic trunk
|
|
1725
|
+
genml-kit-train --pipeline rl --method ppo \
|
|
1726
|
+
--model rl/actor_critic --model_arg hidden_dims=[64,64]
|
|
1727
|
+
|
|
1728
|
+
# SAC: critic head width (the actor defaults to rl/actor_critic)
|
|
1729
|
+
genml-kit-train --pipeline rl --method sac --env_id Pendulum-v1 \
|
|
1730
|
+
--model rl/sac_critic --model_arg hidden_dims=[256,256]
|
|
1731
|
+
```
|
|
1732
|
+
|
|
1733
|
+
Registered RL factories (auto-discovered; see `MODELS.list_names()`):
|
|
1734
|
+
|
|
1735
|
+
| Name | Returns | Notes |
|
|
1736
|
+
|---|---|---|
|
|
1737
|
+
| `rl/qnet` | `QNetwork` (flat Q-head) | DQN default |
|
|
1738
|
+
| `rl/qnet_dueling` | `QNetwork` (dueling Q-head) | `--dqn_dueling` shortcut |
|
|
1739
|
+
| `rl/actor_critic` | `ActorCritic` (shared backbone) | PPO default |
|
|
1740
|
+
| `rl/sac_critic` | `SACCritic` (single critic head) | SAC default |
|
|
1741
|
+
|
|
1742
|
+
The methods pass a `SpaceSpec` (obs/action sizes derived from the env) to
|
|
1743
|
+
the factory, so a custom registered factory can accept the standard scalar
|
|
1744
|
+
kwargs (`obs_dim`, `n_actions`, `action_dim`, `hidden_dims`, ...) or the
|
|
1745
|
+
`space=` spec directly. SAC additionally supports a *factory-return
|
|
1746
|
+
contract*: a factory may return a plain dict `{"actor": ..., "critic": ...}`
|
|
1747
|
+
to supply both networks (the method composes the twin critic via a deep
|
|
1748
|
+
copy). As elsewhere in the framework there is **no** model-validation
|
|
1749
|
+
layer: an incompatible network fails loudly at first use (`act()` /
|
|
1750
|
+
`forward()`), and an unknown `--model` name falls through `load_model` to
|
|
1751
|
+
Hugging Face and fails at load time.
|
|
1752
|
+
|
|
1753
|
+
### Evaluation videos
|
|
1754
|
+
|
|
1755
|
+
Pass `--record_eval_video` to record **one video per evaluation episode**
|
|
1756
|
+
during validation (all RL methods: DQN / PPO / SAC).
|
|
1757
|
+
`<checkpoint_dir>/videos/` as `eval_episode_000.mp4` etc., or as
|
|
1758
|
+
`eval_episode_000.gif` when the MP4 writer is unavailable (e.g. no
|
|
1759
|
+
`imageio[ffmpeg]`). Recording is **only** attempted when the environment can
|
|
1760
|
+
produce `rgb_array` frames: scripted/custom environments without a renderer
|
|
1761
|
+
are detected at validation time and skipped without crashing.
|
|
1762
|
+
|
|
1763
|
+
### How a PPO training epoch works
|
|
1764
|
+
|
|
1765
|
+
PPO is *on-policy*: every update must be computed from transitions
|
|
1766
|
+
collected by the **current** policy. To keep that guarantee, the rollout
|
|
1767
|
+
buffer is **refilled from scratch each epoch** — nothing survives from the
|
|
1768
|
+
previous epoch.
|
|
1769
|
+
|
|
1770
|
+
**One training epoch** (the `=== Epoch N/500 ===` line in the logs) is
|
|
1771
|
+
exactly:
|
|
1772
|
+
|
|
1773
|
+
```
|
|
1774
|
+
1. rollout.reset() # clear the on-policy buffer
|
|
1775
|
+
2. collect rollout_len env steps # --ppo_rollout_len (default 2048)
|
|
1776
|
+
3. GAE (gamma, lam) over the rollout # advantages + returns
|
|
1777
|
+
4. normalize advantages once # over the whole rollout
|
|
1778
|
+
5. ppo_epochs SGD passes over it # --ppo_epochs (default 4)
|
|
1779
|
+
each pass: rollout_len / mini_batch_size minibatches
|
|
1780
|
+
6. next epoch -> back to 1
|
|
1781
|
+
```
|
|
1782
|
+
|
|
1783
|
+
So the arithmetic of a run is:
|
|
1784
|
+
|
|
1785
|
+
| Quantity | Formula | Default (Pendulum) |
|
|
1786
|
+
|---|---|---|
|
|
1787
|
+
| env steps per epoch | `rollout_len` | 2048 |
|
|
1788
|
+
| gradient steps per epoch | `ppo_epochs * (rollout_len / mini_batch_size)` | `4 * 2048/64 = 128` |
|
|
1789
|
+
| episodes per epoch | `rollout_len / episode_len` | `2048/200 ≈ 10` |
|
|
1790
|
+
| total env steps | `epochs * rollout_len` | 500 × 2048 = 1.02M |
|
|
1791
|
+
|
|
1792
|
+
The **two "epochs"** that can be confused:
|
|
1793
|
+
|
|
1794
|
+
- **Training epoch** (`--epochs`) — one rollout + its updates. This is
|
|
1795
|
+
what the log lines and checkpoints count.
|
|
1796
|
+
- **PPO SGD epoch** (`--ppo_epochs`) — the number of *passes over the
|
|
1797
|
+
same* rollout during the update phase. Raising it increases data
|
|
1798
|
+
reuse of a single rollout; lowering it (or raising `rollout_len`)
|
|
1799
|
+
increases data freshness.
|
|
1800
|
+
|
|
1801
|
+
The buffer itself is wiped cheaply: `rollout.reset()` just rewrites the
|
|
1802
|
+
pointer, so the tensor storage is reused and the per-epoch cost is one
|
|
1803
|
+
`rollout_len` collection, not an allocation. On resume, the rollout
|
|
1804
|
+
buffer state is saved/restored with the checkpoint and the previous
|
|
1805
|
+
epoch's last observation carries over, so an interrupted run continues
|
|
1806
|
+
mid-environment rather than restarting the episode.
|
|
1807
|
+
|
|
1808
|
+
Changing `--ppo_rollout_len` therefore changes the *epoch length in
|
|
1809
|
+
steps*, not the memory footprint (memory is `O(rollout_len)` but tiny —
|
|
1810
|
+
~115 KB for Pendulum at 2048) and not the total step budget
|
|
1811
|
+
(`epochs × rollout_len`). A larger rollout gives the GAE a longer
|
|
1812
|
+
horizon before bootstrap and more data per update, but also means more
|
|
1813
|
+
gradient steps per rollout: at 50000 you'd run `4 × 50000/64 = 3125`
|
|
1814
|
+
updates on one batch, which pushes PPO toward overfitting a single
|
|
1815
|
+
rollout.
|
|
1816
|
+
|
|
1817
|
+
### PPO flag reference
|
|
1818
|
+
|
|
1819
|
+
| Flag | Default | Meaning |
|
|
1820
|
+
|---|---|---|
|
|
1821
|
+
| `--ppo_gamma` | 0.99 | Discount factor for GAE / returns |
|
|
1822
|
+
| `--ppo_lam` | 0.95 | GAE trace-decay (bias-variance tradeoff) |
|
|
1823
|
+
| `--ppo_clip_eps` | 0.2 | Clipping epsilon of the clipped surrogate |
|
|
1824
|
+
| `--ppo_epochs` | 4 | SGD passes over each rollout |
|
|
1825
|
+
| `--ppo_mini_batch_size` | 64 | Mini-batch size for PPO updates |
|
|
1826
|
+
| `--ppo_entropy_coef` | 0.01 | Entropy bonus coefficient (see SB3-tricks section) |
|
|
1827
|
+
| `--ppo_value_coef` | 0.5 | Value-loss coefficient |
|
|
1828
|
+
| `--ppo_vf_clip_eps` | None | Value-function clipping (None = unclipped) |
|
|
1829
|
+
| `--ppo_target_kl` | None | Early-stop a PPO epoch when approx-KL exceeds this (see SB3-tricks section) |
|
|
1830
|
+
| `--ppo_rollout_len` | 2048 | Env steps collected per training epoch |
|
|
1831
|
+
| `--ppo_discrete` / `--ppo_continuous` | discrete | Action-space selection |
|
|
1832
|
+
| `--ppo_log_std_init` | log(0.5) ≈ −0.693 | Initial log_std of the continuous Gaussian policy (σ₀ = 0.5) |
|
|
1833
|
+
| `--ppo_log_std_min` | −10.0 | Lower safety clamp of log_std (σ floor ≈ 4.5e-5). Kept far below a converged policy's operating range so the entropy gradient never dies at the boundary (the old −2.0 default froze exploration at σ = 0.135) |
|
|
1834
|
+
| `--ppo_log_std_max` | 2.0 | Upper safety clamp of log_std (σ ceiling ≈ 7.4; bounds entropy) |
|
|
1835
|
+
|
|
1836
|
+
The `log_std_*` knobs control the **continuous** Gaussian policy's
|
|
1837
|
+
standard deviation. `log_std_init` sets where exploration starts;
|
|
1838
|
+
`log_std_min`/`log_std_max` are wide safety rails (SOTA pattern: SB3
|
|
1839
|
+
unbounded, rsl_rl `std_range=(1e-6, 1e6)`, jaxrl `LOG_STD_MIN=-10`) that
|
|
1840
|
+
prevent σ collapse/explosion while leaving entropy freely trainable.
|
|
1841
|
+
|
|
1842
|
+
### Stable-Baselines3-style tricks
|
|
1843
|
+
|
|
1844
|
+
The PPO implementation supports three opt-in techniques that
|
|
1845
|
+
Stable-Baselines3 bakes into its default algorithms. All three are
|
|
1846
|
+
**off by default** — every existing flag combination behaves exactly as
|
|
1847
|
+
before — and each is documented with *when it helps* and *when it
|
|
1848
|
+
doesn't*.
|
|
1849
|
+
|
|
1850
|
+
#### Reward normalization (`--reward_normalize`)
|
|
1851
|
+
|
|
1852
|
+
Normalizes rewards by the running standard deviation of discounted
|
|
1853
|
+
returns, matching SB3 `VecNormalize(norm_reward=True)`. For each step,
|
|
1854
|
+
`running_return = gamma * running_return + reward` is fed to a Welford
|
|
1855
|
+
`RunningMeanStd`; the reward is then divided by `sqrt(var + eps)` and
|
|
1856
|
+
clamped to `[-reward_norm_clip, reward_norm_clip]` (default 10.0).
|
|
1857
|
+
|
|
1858
|
+
- **When it helps:** environments with large or unnormalized reward
|
|
1859
|
+
scales (Pendulum-v1, MuJoCo) where an O(1000) value loss otherwise
|
|
1860
|
+
dominates the total loss and swamps the entropy/policy signal.
|
|
1861
|
+
- **When it doesn't:** rewards already O(1) (CartPole, most toy envs) —
|
|
1862
|
+
normalization is harmless but unnecessary.
|
|
1863
|
+
- **Eval always uses raw rewards** (SB3 `VecNormalize.train(False)`):
|
|
1864
|
+
the running-return stats are not updated during evaluation, so
|
|
1865
|
+
`eval_return` stays comparable across runs. The running stats travel
|
|
1866
|
+
with the checkpoint and are restored on resume.
|
|
1867
|
+
|
|
1868
|
+
#### Orthogonal initialization (`--param_init ortho`)
|
|
1869
|
+
|
|
1870
|
+
Applies SB3-style orthogonal initialization to the model after
|
|
1871
|
+
construction: gain `sqrt(2)` for policy/hidden layers, `1.0` for
|
|
1872
|
+
value/head layers, zero biases — matching the original PPO
|
|
1873
|
+
implementation. `log_std` is left untouched. Works for any method
|
|
1874
|
+
(PPO, SAC, and non-RL MLPs alike).
|
|
1875
|
+
|
|
1876
|
+
- **When it helps:** continuous RL policies, where the default PyTorch
|
|
1877
|
+
init can leave the policy near-degenerate at start and slow the
|
|
1878
|
+
critical early phase.
|
|
1879
|
+
- **When it doesn't:** fine-tuned / checkpoint-loaded models (init is
|
|
1880
|
+
skipped when weights are loaded from a checkpoint) or discrete
|
|
1881
|
+
policies that already train fine with defaults.
|
|
1882
|
+
|
|
1883
|
+
#### `target_kl` early stopping (`--ppo_target_kl <float>`)
|
|
1884
|
+
|
|
1885
|
+
SB3 stops a PPO epoch early when the approximate KL divergence (the
|
|
1886
|
+
Schulman **k1** estimator, `0.5 * mean(log_ratio^2)`, always >= 0)
|
|
1887
|
+
exceeds `target_kl`. This guards against destructive updates on
|
|
1888
|
+
high-variance minibatches.
|
|
1889
|
+
|
|
1890
|
+
- **Suggested starting point:** `--ppo_target_kl 0.03`.
|
|
1891
|
+
- **Too low** (e.g. < 0.005) stalls learning; **too high** (e.g. > 0.1)
|
|
1892
|
+
disables the safety. `None` (default) disables early stopping.
|
|
1893
|
+
- Only affects PPO epochs after the first rollout, so it is a pure
|
|
1894
|
+
safety valve for the on-policy update phase.
|
|
1895
|
+
|
|
1896
|
+
#### Entropy coefficient (`--ppo_entropy_coef <float>`)
|
|
1897
|
+
|
|
1898
|
+
Coefficient of the entropy bonus (default 0.01). SB3 uses **0.0** for
|
|
1899
|
+
continuous PPO, relying on init + normalization instead; genml_kit keeps
|
|
1900
|
+
a small positive default as light regularization.
|
|
1901
|
+
|
|
1902
|
+
- **Higher** (0.05–0.1) encourages broader exploration — may help hard
|
|
1903
|
+
exploration tasks.
|
|
1904
|
+
- **Lower** (toward 0.0) sharpens the policy — may help fine-tuning late
|
|
1905
|
+
in training.
|
|
1906
|
+
- No single value is best for all tasks: sweep 0.0–0.1 on your task.
|
|
1907
|
+
|
|
1908
|
+
A typical "serious" continuous-PPO setup (Pendulum-style) looks like:
|
|
1909
|
+
|
|
1910
|
+
```
|
|
1911
|
+
genml-kit-train --pipeline rl --method ppo --env_id Pendulum-v1 \
|
|
1912
|
+
--ppo_continuous --obs_normalize --reward_normalize --param_init ortho \
|
|
1913
|
+
--ppo_target_kl 0.03 --epochs 500
|
|
1914
|
+
```
|
|
1915
|
+
|
|
1916
|
+
### Custom environments
|
|
1917
|
+
|
|
1918
|
+
Pass `--env_script /path/to/env.py` to load a custom environment. The
|
|
1919
|
+
script must expose a `make_env()` factory returning a Gymnasium-style env
|
|
1920
|
+
(`reset()`, `step(action)`, `close()`; `observation_space` and
|
|
1921
|
+
`action_space` attributes, which the pipeline introspects to size the
|
|
1922
|
+
networks). `make_env()` may take a single `seed` keyword argument if your
|
|
1923
|
+
environment wants reproducible initialization.
|
|
1924
|
+
|
|
1925
|
+
### Environment & replay options
|
|
1926
|
+
|
|
1927
|
+
Relevant flags:
|
|
1928
|
+
- `--env_id`, `--obs_dim`, `--env_script` -- environment selection.
|
|
1929
|
+
- `--replay_capacity`, `--batch_size`, `--n_step`, `--prioritized`,
|
|
1930
|
+
`--per_alpha`, `--per_beta_start`, `--per_beta_frames` -- replay tuning.
|
|
1931
|
+
- `--obs_normalize`, `--obs_norm_clip` -- running observation
|
|
1932
|
+
normalization (Welford). Both train and eval observations are
|
|
1933
|
+
normalized with the same RMS statistics.
|
|
1934
|
+
- `--reward_normalize`, `--reward_norm_clip` -- running-return reward
|
|
1935
|
+
normalization (SB3 `VecNormalize(norm_reward=True)`). See
|
|
1936
|
+
"Stable-Baselines3-style tricks" below.
|
|
1937
|
+
- `--param_init ortho` -- SB3-style orthogonal parameter initialization.
|
|
1938
|
+
See "Stable-Baselines3-style tricks" below.
|
|
1939
|
+
- `--ppo_gamma`, `--ppo_lam`, `--ppo_clip_eps`, `--ppo_epochs`,
|
|
1940
|
+
`--ppo_mini_batch_size`, `--ppo_value_coef`, `--ppo_vf_clip_eps`,
|
|
1941
|
+
`--ppo_rollout_len`, `--ppo_discrete`/`--ppo_continuous`,
|
|
1942
|
+
`--ppo_log_std_init/min/max` -- PPO update and policy knobs. See the
|
|
1943
|
+
"PPO flag reference" table above.
|
|
1944
|
+
- `--ppo_target_kl` -- early-stop a PPO epoch when the k1 approximate-KL
|
|
1945
|
+
exceeds this threshold (SB3 `target_kl`). See below.
|
|
1946
|
+
- `--ppo_entropy_coef` -- entropy bonus coefficient (default 0.01).
|
|
1947
|
+
See below.
|
|
1948
|
+
- `--eval_episodes`, `--env_seed` -- evaluation.
|
|
1949
|
+
- `--record_eval_video` -- record one MP4 (or GIF fallback) per evaluation
|
|
1950
|
+
episode under `<checkpoint_dir>/videos/`; only when the environment
|
|
1951
|
+
supports rendering (see "Evaluation videos" below).
|
|
1952
|
+
|
|
1953
|
+
### Episode-end semantics (terminated vs truncated)
|
|
1954
|
+
|
|
1955
|
+
Gymnasium distinguishes *terminated* (the MDP truly ended, e.g. the pole
|
|
1956
|
+
fell) from *truncated* (the episode was cut short, e.g. by a time limit).
|
|
1957
|
+
genml_kit threads the true `terminated` flag through the replay buffer,
|
|
1958
|
+
rollout buffer, GAE, and the TD targets: a truncated final step still
|
|
1959
|
+
bootstraps `γ V(s′)`, while a terminated one does not. Environments using
|
|
1960
|
+
the legacy 4-tuple gym API (no `terminated`/`truncated` split) fall back to
|
|
1961
|
+
treating `done` as `terminated`.
|
|
1962
|
+
|
|
1963
|
+
## References And Further Reading
|
|
1964
|
+
|
|
1965
|
+
- Xie et al., [SimMIM: A Simple Framework for Masked Image Modeling](https://arxiv.org/abs/2111.09886), CVPR 2022.
|
|
1966
|
+
- Assran et al., [Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture](https://arxiv.org/abs/2301.08243), CVPR 2023.
|
|
1967
|
+
- Grill et al., [Bootstrap Your Own Latent: A New Approach to Self-Supervised Learning](https://arxiv.org/abs/2006.07733), NeurIPS 2020.
|
|
1968
|
+
The EMA teacher and stop-gradient design used by I-JEPA-style methods.
|
|
1969
|
+
- Caron et al., [Emerging Properties in Self-Supervised Vision Transformers](https://arxiv.org/abs/2104.14294), ICCV 2021.
|
|
1970
|
+
The DINO paper — self-distillation with an EMA teacher and multi-crop.
|
|
1971
|
+
The centering + sharpening trick to avoid representation collapse.
|
|
1972
|
+
- Khosla et al., [Supervised Contrastive Learning](https://arxiv.org/abs/2004.11362), NeurIPS 2020.
|
|
1973
|
+
- He et al., [Masked Autoencoders Are Scalable Vision Learners](https://arxiv.org/abs/2111.06377), CVPR 2022.
|
|
1974
|
+
A useful comparison for masked-image pre-training.
|
|
1975
|
+
- Hu et al., [LoRA: Low-Rank Adaptation of Large Language Models](https://arxiv.org/abs/2106.09685), ICLR 2022.
|
|
1976
|
+
- Howard & Ruder, [Universal Language Model Fine-tuning for Text Classification](https://arxiv.org/abs/1801.06146), ACL 2018.
|
|
1977
|
+
Introduced layer-wise learning rate decay.
|
|
1978
|
+
- Zhang et al., [mixup: Beyond Empirical Risk Minimization](https://arxiv.org/abs/1710.09412), ICLR 2018.
|
|
1979
|
+
- Lin et al., [Focal Loss for Dense Object Detection](https://arxiv.org/abs/1708.02002), ICCV 2017.
|
|
1980
|
+
Introduced focal loss and the $`(1-p_t)^\gamma`$ easy-example down-weighting.
|
|
1981
|
+
- Chen & Guestrin, [XGBoost: A Scalable Tree Boosting System](https://arxiv.org/abs/1603.02754), KDD 2016.
|
|
1982
|
+
- The [PyTorch optimization documentation](https://pytorch.org/docs/stable/optim.html)
|
|
1983
|
+
explains AdamW, schedulers, and gradient clipping.
|
|
1984
|
+
- The [scikit-learn metrics documentation](https://scikit-learn.org/stable/modules/model_evaluation.html)
|
|
1985
|
+
is useful when choosing metrics for imbalanced classification.
|
|
1986
|
+
|
|
1987
|
+
## Development
|
|
1988
|
+
|
|
1989
|
+
```bash
|
|
1990
|
+
git clone https://github.com/davidel/genml_kit
|
|
1991
|
+
cd genml_kit
|
|
1992
|
+
pip install -e ".[dev,all]"
|
|
1993
|
+
pytest
|
|
1994
|
+
```
|
|
1995
|
+
|
|
1996
|
+
Tests run with warnings-as-errors (`filterwarnings = ["error"]` in
|
|
1997
|
+
`pyproject.toml`), so a new dependency deprecation warning fails the suite
|
|
1998
|
+
instead of scrolling past. Code is formatted with
|
|
1999
|
+
[yapf](https://github.com/google/yapf) using the project's `.style.yapf`
|
|
2000
|
+
(Google style, 2-space indent) and linted with
|
|
2001
|
+
[Ruff](https://docs.astral.sh/ruff/) (`ruff check .`).
|
|
2002
|
+
|
|
2003
|
+
### Naming Conventions For Class Members
|
|
2004
|
+
|
|
2005
|
+
- Anything that PyTorch registers — child `nn.Module`, `nn.Parameter`,
|
|
2006
|
+
registered buffer — is **never** underscore-prefixed, regardless of
|
|
2007
|
+
intended visibility. A leading underscore would change `state_dict()`
|
|
2008
|
+
keys, optimizer param groups, and DDP behavior.
|
|
2009
|
+
- Everything else that is not part of a class's public contract
|
|
2010
|
+
(internal state, helper hooks invoked only by the class itself) gets a
|
|
2011
|
+
leading underscore. A member called from other classes, overridden as
|
|
2012
|
+
an extension point, or documented as API stays public.
|
|
2013
|
+
- Extension-point methods that *callers* invoke (e.g.
|
|
2014
|
+
`PretrainMethod.build_transform`) are public; same-named hooks the
|
|
2015
|
+
base class *itself* dispatches (e.g. `BaseImageProcessor._build_transform`)
|
|
2016
|
+
are private.
|
|
2017
|
+
|
|
2018
|
+
## License
|
|
2019
|
+
|
|
2020
|
+
Apache-2.0
|
|
2021
|
+
|