genml-kit 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {genml_kit-0.2.0 → genml_kit-0.3.0}/PKG-INFO +1983 -68
- {genml_kit-0.2.0 → genml_kit-0.3.0}/README.md +1961 -66
- genml_kit-0.3.0/genml_kit/augmentations/multicrop.py +113 -0
- genml_kit-0.3.0/genml_kit/cli/__init__.py +12 -0
- {genml_kit-0.2.0/genml_kit/training → genml_kit-0.3.0/genml_kit/cli}/infer.py +43 -79
- genml_kit-0.3.0/genml_kit/cli/llm.py +53 -0
- genml_kit-0.3.0/genml_kit/cli/tokenizer.py +146 -0
- genml_kit-0.3.0/genml_kit/cli/train.py +526 -0
- genml_kit-0.3.0/genml_kit/cli/tune.py +303 -0
- genml_kit-0.3.0/genml_kit/datasets/__init__.py +49 -0
- genml_kit-0.3.0/genml_kit/datasets/base.py +156 -0
- genml_kit-0.3.0/genml_kit/datasets/batch_policy.py +139 -0
- genml_kit-0.3.0/genml_kit/datasets/capped.py +125 -0
- genml_kit-0.3.0/genml_kit/datasets/counting.py +784 -0
- genml_kit-0.3.0/genml_kit/datasets/det_folder.py +285 -0
- genml_kit-0.3.0/genml_kit/datasets/det_transforms.py +198 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/ensemble.py +216 -28
- genml_kit-0.3.0/genml_kit/datasets/factory.py +85 -0
- genml_kit-0.3.0/genml_kit/datasets/field_selector.py +124 -0
- genml_kit-0.3.0/genml_kit/datasets/hf_proxy.py +489 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/image_folder.py +3 -3
- genml_kit-0.3.0/genml_kit/datasets/iterable.py +282 -0
- genml_kit-0.3.0/genml_kit/datasets/length_bucket_sampler.py +92 -0
- genml_kit-0.3.0/genml_kit/datasets/packing_sampler.py +108 -0
- genml_kit-0.3.0/genml_kit/datasets/preference_proxy.py +138 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/replay_buffer.py +11 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/retry.py +10 -3
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/rollout_buffer.py +109 -64
- genml_kit-0.3.0/genml_kit/datasets/seg_folder.py +206 -0
- genml_kit-0.3.0/genml_kit/datasets/split.py +213 -0
- genml_kit-0.3.0/genml_kit/datasets/stream_batching.py +212 -0
- genml_kit-0.3.0/genml_kit/datasets/text_proxy.py +267 -0
- genml_kit-0.3.0/genml_kit/datasets/transforms.py +223 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/vo_pairs.py +41 -10
- genml_kit-0.3.0/genml_kit/datasets/webdataset_source.py +180 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/geometry/similarity.py +5 -2
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/io/checkpointing.py +141 -32
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/io/storage_utils.py +33 -32
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/__init__.py +28 -0
- genml_kit-0.3.0/genml_kit/losses/alignment.py +104 -0
- genml_kit-0.3.0/genml_kit/losses/builtin.py +208 -0
- genml_kit-0.3.0/genml_kit/losses/detection.py +342 -0
- genml_kit-0.3.0/genml_kit/losses/distillation.py +241 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/registry.py +2 -8
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/rl.py +26 -19
- genml_kit-0.3.0/genml_kit/losses/segmentation.py +131 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/__init__.py +4 -5
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/base.py +169 -49
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/byol.py +29 -24
- genml_kit-0.3.0/genml_kit/methods/causal_lm.py +370 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/classification.py +141 -31
- genml_kit-0.3.0/genml_kit/methods/detection.py +326 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/dino.py +46 -27
- genml_kit-0.3.0/genml_kit/methods/distillation.py +577 -0
- genml_kit-0.3.0/genml_kit/methods/dpo.py +183 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/ijepa.py +138 -44
- genml_kit-0.3.0/genml_kit/methods/reward_model.py +187 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/rl_dqn.py +27 -25
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/rl_ppo.py +31 -39
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/rl_sac.py +83 -58
- genml_kit-0.3.0/genml_kit/methods/rl_utils.py +130 -0
- genml_kit-0.3.0/genml_kit/methods/segmentation.py +358 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/simmim.py +13 -7
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/supcon.py +24 -8
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/vo_pair.py +55 -35
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/__init__.py +12 -5
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/byol.py +4 -5
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/loader.py +6 -4
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/model.py +20 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/loader.py +25 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/masked_encoder.py +16 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/model.py +44 -7
- genml_kit-0.3.0/genml_kit/models/decoders/__init__.py +109 -0
- genml_kit-0.3.0/genml_kit/models/decoders/base.py +133 -0
- genml_kit-0.3.0/genml_kit/models/decoders/fcos_head.py +170 -0
- genml_kit-0.3.0/genml_kit/models/decoders/fpn.py +90 -0
- genml_kit-0.3.0/genml_kit/models/decoders/linear.py +60 -0
- genml_kit-0.3.0/genml_kit/models/decoders/unet.py +105 -0
- genml_kit-0.3.0/genml_kit/models/det_model_wrapper/__init__.py +7 -0
- genml_kit-0.3.0/genml_kit/models/det_model_wrapper/loader.py +147 -0
- genml_kit-0.3.0/genml_kit/models/det_model_wrapper/model.py +122 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/dino.py +3 -4
- genml_kit-0.3.0/genml_kit/models/ema.py +276 -0
- genml_kit-0.3.0/genml_kit/models/encoder_utils.py +157 -0
- genml_kit-0.3.0/genml_kit/models/generative.py +91 -0
- genml_kit-0.3.0/genml_kit/models/layer_match.py +203 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/registry.py +57 -22
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/actor_critic.py +13 -2
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/qnetwork.py +11 -3
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/sac_critic.py +7 -7
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/spaces.py +5 -2
- genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/__init__.py +7 -0
- genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/encoder.py +119 -0
- genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/loader.py +184 -0
- genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/model.py +118 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/timm/loader.py +3 -2
- genml_kit-0.3.0/genml_kit/models/tokenizer.py +291 -0
- genml_kit-0.3.0/genml_kit/models/tokenizer_train.py +346 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/uvito/loader.py +7 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/uvito/model.py +14 -10
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/vo/__init__.py +2 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/vo/loader.py +16 -3
- genml_kit-0.3.0/genml_kit/models/vo/vo_similar.py +236 -0
- genml_kit-0.3.0/genml_kit/pipelines/__init__.py +34 -0
- genml_kit-0.3.0/genml_kit/pipelines/base.py +282 -0
- genml_kit-0.3.0/genml_kit/pipelines/contracts.py +91 -0
- genml_kit-0.3.0/genml_kit/pipelines/detection.py +225 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/images.py +187 -86
- genml_kit-0.3.0/genml_kit/pipelines/preference.py +197 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/rl.py +87 -137
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/rl_normalize.py +3 -2
- genml_kit-0.3.0/genml_kit/pipelines/segmentation.py +421 -0
- genml_kit-0.3.0/genml_kit/pipelines/text.py +515 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/vo_pair.py +31 -16
- genml_kit-0.3.0/genml_kit/search/__init__.py +35 -0
- genml_kit-0.3.0/genml_kit/search/errors.py +17 -0
- genml_kit-0.3.0/genml_kit/search/pruners.py +242 -0
- genml_kit-0.3.0/genml_kit/search/runner.py +307 -0
- genml_kit-0.3.0/genml_kit/search/samplers.py +293 -0
- genml_kit-0.3.0/genml_kit/search/space.py +306 -0
- genml_kit-0.3.0/genml_kit/search/store.py +101 -0
- genml_kit-0.3.0/genml_kit/search/study.py +139 -0
- genml_kit-0.3.0/genml_kit/search/trial.py +120 -0
- genml_kit-0.3.0/genml_kit/training/budget.py +208 -0
- genml_kit-0.3.0/genml_kit/training/calibration.py +123 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/classifiers/__init__.py +24 -35
- genml_kit-0.3.0/genml_kit/training/det_metrics.py +224 -0
- genml_kit-0.3.0/genml_kit/training/generate.py +231 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/grad_monitor.py +9 -6
- genml_kit-0.3.0/genml_kit/training/infer.py +71 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/labels.py +11 -5
- genml_kit-0.3.0/genml_kit/training/llm_eval.py +366 -0
- genml_kit-0.3.0/genml_kit/training/llm_model.py +148 -0
- genml_kit-0.3.0/genml_kit/training/metrics_sinks.py +364 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/model_utils.py +158 -11
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/optim_factory.py +141 -13
- genml_kit-0.3.0/genml_kit/training/packed_attention.py +199 -0
- genml_kit-0.3.0/genml_kit/training/preference_collator.py +112 -0
- genml_kit-0.3.0/genml_kit/training/rl_trainer.py +712 -0
- genml_kit-0.3.0/genml_kit/training/run_manifest.py +125 -0
- genml_kit-0.3.0/genml_kit/training/seg_metrics.py +126 -0
- genml_kit-0.3.0/genml_kit/training/sequence_mode.py +103 -0
- genml_kit-0.3.0/genml_kit/training/text_collator.py +390 -0
- genml_kit-0.3.0/genml_kit/training/train.py +292 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/train_compat.py +58 -13
- genml_kit-0.3.0/genml_kit/training/train_reporting.py +676 -0
- genml_kit-0.3.0/genml_kit/training/trainer.py +747 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/video_utils.py +1 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/vo/__init__.py +2 -0
- genml_kit-0.3.0/genml_kit/training/vo/eval_vo.py +132 -0
- genml_kit-0.3.0/genml_kit/training/vo/train_vo.py +241 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/xgb_pipeline.py +16 -12
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/xgb_utils.py +7 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/args.py +71 -61
- genml_kit-0.3.0/genml_kit/utils/attr.py +113 -0
- genml_kit-0.3.0/genml_kit/utils/cli.py +237 -0
- genml_kit-0.3.0/genml_kit/utils/distributed.py +1056 -0
- genml_kit-0.3.0/genml_kit/utils/generation.py +449 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/gpu.py +18 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/logging.py +7 -2
- genml_kit-0.3.0/genml_kit/utils/script.py +213 -0
- genml_kit-0.3.0/genml_kit/utils/seg.py +236 -0
- genml_kit-0.3.0/genml_kit/utils/shared_args.py +537 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/signal.py +6 -4
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/PKG-INFO +1983 -68
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/SOURCES.txt +166 -1
- genml_kit-0.3.0/genml_kit.egg-info/entry_points.txt +6 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/requires.txt +24 -1
- genml_kit-0.3.0/pyproject.toml +97 -0
- genml_kit-0.3.0/tests/test_alignment_losses.py +101 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_args_frozen.py +1 -27
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_attr_utils.py +26 -7
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_balanced_sampler.py +4 -4
- genml_kit-0.3.0/tests/test_batch_policy.py +241 -0
- genml_kit-0.3.0/tests/test_budget.py +135 -0
- genml_kit-0.3.0/tests/test_budget_distributed.py +157 -0
- genml_kit-0.3.0/tests/test_cache_dir.py +92 -0
- genml_kit-0.3.0/tests/test_calibration.py +164 -0
- genml_kit-0.3.0/tests/test_capped_dataset.py +111 -0
- genml_kit-0.3.0/tests/test_causal_lm.py +280 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_checkpointing.py +150 -16
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_classifiers.py +4 -3
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_cli_help.py +1 -3
- genml_kit-0.3.0/tests/test_cli_utils.py +262 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_contrastive_models.py +22 -0
- genml_kit-0.3.0/tests/test_counting.py +543 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_custom_models.py +71 -9
- genml_kit-0.3.0/tests/test_dataset_arg.py +90 -0
- genml_kit-0.3.0/tests/test_det_e2e.py +213 -0
- genml_kit-0.3.0/tests/test_det_folder.py +176 -0
- genml_kit-0.3.0/tests/test_det_head.py +129 -0
- genml_kit-0.3.0/tests/test_det_loss.py +116 -0
- genml_kit-0.3.0/tests/test_det_metrics.py +114 -0
- genml_kit-0.3.0/tests/test_det_transforms.py +123 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_dino.py +2 -1
- genml_kit-0.3.0/tests/test_distillation_losses.py +154 -0
- genml_kit-0.3.0/tests/test_distillation_method.py +306 -0
- genml_kit-0.3.0/tests/test_distributed.py +832 -0
- genml_kit-0.3.0/tests/test_distributed_e2e.py +186 -0
- genml_kit-0.3.0/tests/test_dpo.py +186 -0
- genml_kit-0.3.0/tests/test_ema.py +147 -0
- genml_kit-0.3.0/tests/test_ema_weights.py +262 -0
- genml_kit-0.3.0/tests/test_ema_weights_trainer.py +154 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_encoder_utils.py +58 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_ensemble_collate.py +5 -5
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_ensemble_labels.py +9 -8
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_ensemble_mixed_labels.py +15 -4
- genml_kit-0.3.0/tests/test_generation.py +273 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_grad_monitor.py +97 -102
- genml_kit-0.3.0/tests/test_hyperband.py +97 -0
- genml_kit-0.3.0/tests/test_ijepa.py +145 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_init_orthogonal.py +12 -6
- genml_kit-0.3.0/tests/test_iterable_base.py +260 -0
- genml_kit-0.3.0/tests/test_iterable_ddp.py +343 -0
- genml_kit-0.3.0/tests/test_iterable_ensemble.py +195 -0
- genml_kit-0.3.0/tests/test_iterable_flush.py +129 -0
- genml_kit-0.3.0/tests/test_iterable_hf_proxy.py +123 -0
- genml_kit-0.3.0/tests/test_iterable_resume.py +175 -0
- genml_kit-0.3.0/tests/test_iterable_split.py +189 -0
- genml_kit-0.3.0/tests/test_iterable_webdataset.py +412 -0
- genml_kit-0.3.0/tests/test_iterable_workers.py +240 -0
- genml_kit-0.3.0/tests/test_layer_match.py +183 -0
- genml_kit-0.3.0/tests/test_length_bucket_sampler.py +49 -0
- genml_kit-0.3.0/tests/test_llm_cli.py +195 -0
- genml_kit-0.3.0/tests/test_llm_e2e.py +108 -0
- genml_kit-0.3.0/tests/test_llm_eval.py +119 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_log_format.py +35 -33
- genml_kit-0.3.0/tests/test_logging_helpers.py +49 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_lora.py +99 -1
- genml_kit-0.3.0/tests/test_lora_targets.py +171 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_loss_and_multipliers.py +1 -1
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_loss_registry.py +15 -3
- genml_kit-0.3.0/tests/test_make_tiny_coco.py +83 -0
- genml_kit-0.3.0/tests/test_md_latex.py +322 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_method_lifecycle.py +14 -7
- genml_kit-0.3.0/tests/test_metrics_sinks.py +242 -0
- genml_kit-0.3.0/tests/test_midepoch_resume.py +275 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_optim_factory.py +11 -11
- genml_kit-0.3.0/tests/test_owner_arg_defaults.py +221 -0
- genml_kit-0.3.0/tests/test_packed_attention.py +130 -0
- genml_kit-0.3.0/tests/test_packing_sampler.py +108 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pipeline.py +82 -15
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pipeline_method_registry.py +106 -41
- genml_kit-0.3.0/tests/test_preference_collator.py +109 -0
- genml_kit-0.3.0/tests/test_preference_e2e.py +90 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pretrain_methods.py +53 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pretrain_smoke.py +2 -2
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_pretrain_vis.py +14 -14
- genml_kit-0.3.0/tests/test_readme_cli.py +50 -0
- genml_kit-0.3.0/tests/test_readme_cli_coverage.py +163 -0
- genml_kit-0.3.0/tests/test_readme_cli_harness.py +394 -0
- genml_kit-0.3.0/tests/test_rl_dist_e2e.py +107 -0
- genml_kit-0.3.0/tests/test_rl_distributed.py +540 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_learning.py +18 -42
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_losses_phase2.py +19 -23
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_method.py +5 -22
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_model_selector.py +7 -48
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_normalize.py +10 -9
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_pipeline.py +50 -10
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_ppo.py +7 -24
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_sac.py +5 -21
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_trainer.py +270 -198
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_video.py +67 -23
- genml_kit-0.3.0/tests/test_run_manifest.py +98 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_s3r2util.py +1 -1
- genml_kit-0.3.0/tests/test_safe_len.py +285 -0
- genml_kit-0.3.0/tests/test_script_utils.py +229 -0
- genml_kit-0.3.0/tests/test_search_runner.py +244 -0
- genml_kit-0.3.0/tests/test_search_samplers.py +113 -0
- genml_kit-0.3.0/tests/test_search_space.py +123 -0
- genml_kit-0.3.0/tests/test_search_store_trial.py +125 -0
- genml_kit-0.3.0/tests/test_search_study_pruners.py +119 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_seed_utils.py +9 -9
- genml_kit-0.3.0/tests/test_seg_class_map.py +254 -0
- genml_kit-0.3.0/tests/test_seg_decoders.py +137 -0
- genml_kit-0.3.0/tests/test_seg_e2e.py +188 -0
- genml_kit-0.3.0/tests/test_seg_folder.py +199 -0
- genml_kit-0.3.0/tests/test_seg_hf_proxy.py +69 -0
- genml_kit-0.3.0/tests/test_seg_transforms.py +239 -0
- genml_kit-0.3.0/tests/test_seg_vis.py +174 -0
- genml_kit-0.3.0/tests/test_sequence_mode.py +71 -0
- genml_kit-0.3.0/tests/test_shared_args.py +104 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_signal_utils.py +4 -4
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_simmim.py +41 -0
- genml_kit-0.3.0/tests/test_ssl_ddp.py +203 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_storage_utils.py +4 -5
- genml_kit-0.3.0/tests/test_streaming_classification.py +238 -0
- genml_kit-0.3.0/tests/test_streaming_text_pipeline.py +206 -0
- genml_kit-0.3.0/tests/test_text_collator.py +222 -0
- genml_kit-0.3.0/tests/test_text_pipeline.py +243 -0
- genml_kit-0.3.0/tests/test_tokenizer.py +117 -0
- genml_kit-0.3.0/tests/test_tokenizer_arg.py +95 -0
- genml_kit-0.3.0/tests/test_tokenizer_train.py +259 -0
- genml_kit-0.3.0/tests/test_train_cli_integration.py +82 -0
- genml_kit-0.3.0/tests/test_train_reporting.py +640 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_train_smoke.py +2 -2
- genml_kit-0.3.0/tests/test_trainer.py +654 -0
- genml_kit-0.3.0/tests/test_trainer_ddp.py +224 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_transformer_encoder_init.py +0 -1
- genml_kit-0.3.0/tests/test_vo_e2e.py +163 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_vo_pairs.py +1 -1
- genml_kit-0.3.0/tests/test_vo_similar.py +222 -0
- genml_kit-0.3.0/tests/test_vo_training.py +247 -0
- genml_kit-0.3.0/tests/test_wandb_sink.py +160 -0
- genml_kit-0.3.0/tests/test_warmup_schedulers.py +216 -0
- genml_kit-0.3.0/tests/test_webdataset_pipeline.py +94 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_xgb_pipeline.py +4 -2
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_xgb_utils.py +7 -6
- genml_kit-0.2.0/genml_kit/augmentations/multicrop.py +0 -123
- genml_kit-0.2.0/genml_kit/datasets/__init__.py +0 -23
- genml_kit-0.2.0/genml_kit/datasets/factory.py +0 -58
- genml_kit-0.2.0/genml_kit/datasets/field_dataset.py +0 -40
- genml_kit-0.2.0/genml_kit/datasets/hf_proxy.py +0 -239
- genml_kit-0.2.0/genml_kit/datasets/transforms.py +0 -60
- genml_kit-0.2.0/genml_kit/losses/builtin.py +0 -106
- genml_kit-0.2.0/genml_kit/methods/rl_utils.py +0 -69
- genml_kit-0.2.0/genml_kit/models/encoder_utils.py +0 -89
- genml_kit-0.2.0/genml_kit/models/vo/vo_similar.py +0 -165
- genml_kit-0.2.0/genml_kit/pipelines/__init__.py +0 -25
- genml_kit-0.2.0/genml_kit/pipelines/base.py +0 -62
- genml_kit-0.2.0/genml_kit/pipelines/contracts.py +0 -21
- genml_kit-0.2.0/genml_kit/training/rl_trainer.py +0 -495
- genml_kit-0.2.0/genml_kit/training/train.py +0 -493
- genml_kit-0.2.0/genml_kit/training/train_reporting.py +0 -447
- genml_kit-0.2.0/genml_kit/training/trainer.py +0 -318
- genml_kit-0.2.0/genml_kit/training/vo/eval_vo.py +0 -108
- genml_kit-0.2.0/genml_kit/training/vo/train_vo.py +0 -156
- genml_kit-0.2.0/genml_kit/utils/attr.py +0 -141
- genml_kit-0.2.0/genml_kit/utils/cli.py +0 -110
- genml_kit-0.2.0/genml_kit/utils/script.py +0 -82
- genml_kit-0.2.0/genml_kit.egg-info/entry_points.txt +0 -3
- genml_kit-0.2.0/pyproject.toml +0 -64
- genml_kit-0.2.0/tests/test_cli_utils.py +0 -146
- genml_kit-0.2.0/tests/test_script_utils.py +0 -67
- genml_kit-0.2.0/tests/test_train_reporting.py +0 -416
- genml_kit-0.2.0/tests/test_trainer.py +0 -278
- genml_kit-0.2.0/tests/test_vo_e2e.py +0 -73
- genml_kit-0.2.0/tests/test_vo_similar.py +0 -124
- genml_kit-0.2.0/tests/test_vo_training.py +0 -113
- {genml_kit-0.2.0 → genml_kit-0.3.0}/LICENSE +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/augmentations/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/augmentations/dual_view.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/balanced_sampler.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/datasets/weighted_sampler.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/geometry/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/io/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/byol.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/contrastive.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/dino.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/losses/focal.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/methods/registry.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/attention_pooling.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/contrastive.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/convvit/processor.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/init.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/masked_encoder.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/processors/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/processors/base.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/rl/sac_model.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/simmim.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/timm/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/timm/model.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/timm/processor.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/transformer_utils.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/uvito/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/uvito/processor.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/models/vo/processor.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/pipelines/registry.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/classifiers/base.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/classifiers/cls_attention.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/classifiers/mlp.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/eval.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/metrics.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/param_align.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/training/tta.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/__init__.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/env.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/image_dump.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/label.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/registry.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/seed.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/table.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit/utils/transformer.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/dependency_links.txt +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/genml_kit.egg-info/top_level.txt +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/setup.cfg +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_actor_critic.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_attention_pooling.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_byol.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_datasets_ensemble.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_ensemble_ensure_label_space.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_env.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_evaluation_metrics.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_extract_features.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_geometry_similarity.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_headless_models.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_image_dump.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_infer_topk.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_logging_utils.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_metrics.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_model_script_loading.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_param_align.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_replay_buffer.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_retry.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_cli.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_losses.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_models.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rl_spaces.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_rollout_buffer.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_set_train_mode.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_supcon_loss.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_supcon_method.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_table_utils.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_timm_models.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_tta.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_utils_registry.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_uvito.py +0 -0
- {genml_kit-0.2.0 → genml_kit-0.3.0}/tests/test_weighted_sampler.py +0 -0
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: genml_kit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: General-purpose ML toolkit: image-classification, visual-odometry and self-supervised pre-training through one unified harness.
|
|
5
5
|
License: Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://github.com/davidel/genml_kit
|
|
7
|
+
Project-URL: Repository, https://github.com/davidel/genml_kit
|
|
8
|
+
Project-URL: Issues, https://github.com/davidel/genml_kit/issues
|
|
6
9
|
Requires-Python: >=3.9
|
|
7
10
|
Description-Content-Type: text/markdown
|
|
8
11
|
License-File: LICENSE
|
|
@@ -11,6 +14,7 @@ Requires-Dist: torch
|
|
|
11
14
|
Requires-Dist: torchvision
|
|
12
15
|
Requires-Dist: transformers
|
|
13
16
|
Requires-Dist: datasets
|
|
17
|
+
Requires-Dist: webdataset
|
|
14
18
|
Requires-Dist: xgboost>=2.0
|
|
15
19
|
Requires-Dist: scikit-learn
|
|
16
20
|
Requires-Dist: pillow
|
|
@@ -21,6 +25,10 @@ Provides-Extra: s3
|
|
|
21
25
|
Requires-Dist: boto3>=1.28; extra == "s3"
|
|
22
26
|
Provides-Extra: lora
|
|
23
27
|
Requires-Dist: peft>=0.7.0; extra == "lora"
|
|
28
|
+
Provides-Extra: llm
|
|
29
|
+
Requires-Dist: peft>=0.7.0; extra == "llm"
|
|
30
|
+
Provides-Extra: llm-spm
|
|
31
|
+
Requires-Dist: sentencepiece>=0.1.99; extra == "llm-spm"
|
|
24
32
|
Provides-Extra: timm
|
|
25
33
|
Requires-Dist: timm>=1.0; extra == "timm"
|
|
26
34
|
Provides-Extra: uvito
|
|
@@ -29,21 +37,37 @@ Provides-Extra: rl
|
|
|
29
37
|
Requires-Dist: gymnasium>=0.29; extra == "rl"
|
|
30
38
|
Requires-Dist: pygame>=2.1; extra == "rl"
|
|
31
39
|
Requires-Dist: imageio[ffmpeg]>=2.31; extra == "rl"
|
|
40
|
+
Requires-Dist: box2d>=2.3.10; extra == "rl"
|
|
41
|
+
Provides-Extra: tune
|
|
42
|
+
Requires-Dist: scipy; extra == "tune"
|
|
43
|
+
Provides-Extra: gpu
|
|
44
|
+
Requires-Dist: nvidia-ml-py; extra == "gpu"
|
|
45
|
+
Provides-Extra: flash
|
|
46
|
+
Requires-Dist: flash-attn; extra == "flash"
|
|
47
|
+
Provides-Extra: wandb
|
|
48
|
+
Requires-Dist: wandb>=0.16; extra == "wandb"
|
|
32
49
|
Provides-Extra: all
|
|
33
|
-
Requires-Dist: genml_kit[gcs,lora,rl,s3,timm,uvito]; extra == "all"
|
|
50
|
+
Requires-Dist: genml_kit[gcs,gpu,llm,llm-spm,lora,rl,s3,timm,tune,uvito,wandb]; extra == "all"
|
|
34
51
|
Provides-Extra: dev
|
|
35
52
|
Requires-Dist: pytest; extra == "dev"
|
|
53
|
+
Requires-Dist: pytest-timeout; extra == "dev"
|
|
54
|
+
Requires-Dist: pytest-split; extra == "dev"
|
|
36
55
|
Requires-Dist: ruff; extra == "dev"
|
|
37
56
|
Requires-Dist: yapf; extra == "dev"
|
|
57
|
+
Requires-Dist: pyyaml; extra == "dev"
|
|
38
58
|
Dynamic: license-file
|
|
39
59
|
|
|
40
60
|
# genml_kit
|
|
41
61
|
|
|
42
62
|
[](https://github.com/davidel/genml_kit/actions/workflows/ci.yml)
|
|
63
|
+
[](https://pypi.org/project/genml_kit/)
|
|
64
|
+
[](https://pypi.org/project/genml_kit/)
|
|
65
|
+
[](LICENSE)
|
|
43
66
|
|
|
44
|
-
A general-purpose toolkit for image classification
|
|
45
|
-
pre-training, supervised fine-tuning,
|
|
46
|
-
|
|
67
|
+
A general-purpose toolkit for image classification and decoder-only language
|
|
68
|
+
models. Supports self-supervised pre-training, supervised fine-tuning, LoRA,
|
|
69
|
+
causal-LM training / generation, preference alignment (DPO, reward models),
|
|
70
|
+
and XGBoost ensemble inference — all from the command line.
|
|
47
71
|
|
|
48
72
|
`genml_kit` is domain-agnostic: point it at any HuggingFace dataset, local
|
|
49
73
|
`ImageFolder` tree, or timm/HuggingFace backbone. It grew out of a
|
|
@@ -67,6 +91,14 @@ in [`vo/README.md`](vo/README.md).
|
|
|
67
91
|
- [Gradient Monitor](#gradient-monitor)
|
|
68
92
|
- [Custom Losses](#custom-losses)
|
|
69
93
|
- [Custom Models](#custom-models)
|
|
94
|
+
- [Semantic Segmentation](#semantic-segmentation)
|
|
95
|
+
- [Object Detection](#object-detection)
|
|
96
|
+
- [Visual Odometry](#visual-odometry)
|
|
97
|
+
- [Knowledge Distillation](#knowledge-distillation)
|
|
98
|
+
- [Large Language Models (causal LM)](#large-language-models-causal-lm)
|
|
99
|
+
- [Reinforcement Learning](#reinforcement-learning)
|
|
100
|
+
- [Multi-GPU Training](#multi-gpu-training)
|
|
101
|
+
- [Hyperparameter Search](#hyperparameter-search)
|
|
70
102
|
- [References And Further Reading](#references-and-further-reading)
|
|
71
103
|
- [Development](#development)
|
|
72
104
|
- [License](#license)
|
|
@@ -188,15 +220,17 @@ reuse the same hook.
|
|
|
188
220
|
|
|
189
221
|
| Module | Contents |
|
|
190
222
|
|---|---|
|
|
191
|
-
| `genml_kit.training` | `train.py` and `infer.py`
|
|
192
|
-
| `genml_kit.methods` | objective-side registry (classification, vo_pair, SimMIM, I-JEPA, DINO, BYOL, SupCon) |
|
|
193
|
-
| `genml_kit.losses` | shared loss library (focal, NT-Xent contrastive, DINO self-distillation, BYOL) |
|
|
223
|
+
| `genml_kit.training` | `train.py` (dataset loading: `load_and_split_dataset`, plus legacy compat re-exports) and `infer.py` (inference helpers), `optim_factory.py` (optimizers, LLRD, schedulers), `model_utils.py` (loading, freezing, feature extraction), `param_align.py`, `eval.py` + `metrics.py`, `grad_monitor.py`, `train_reporting.py`, `tta.py`, `xgb_utils.py` + `xgb_pipeline.py`, `text_collator.py` + `preference_collator.py` (text / preference batches), `llm_model.py` + `generate.py` + `llm_eval.py` (LLM inference, generation, eval), `classifiers/` (pluggable heads) |
|
|
224
|
+
| `genml_kit.methods` | objective-side registry (classification, segmentation, detection, vo_pair, SimMIM, I-JEPA, DINO, BYOL, SupCon, distillation, `causal_lm`, `dpo`, `reward_model`) |
|
|
225
|
+
| `genml_kit.losses` | shared loss library (focal, Dice+CE segmentation, NT-Xent contrastive, DINO self-distillation, BYOL, KD distillation, alignment: `sequence_logps` / `dpo_loss` / `bradley_terry_loss`) |
|
|
194
226
|
| `genml_kit.augmentations` | objective-level transforms (dual-view, multi-crop) |
|
|
195
|
-
| `genml_kit.models` | model/processor registry; `timm/`, `convvit/`, `uvito/`, `cls_model_wrapper/` backends; `processors/base.py` |
|
|
196
|
-
| `genml_kit.datasets` | `hf_proxy.py` (HuggingFace → PyTorch bridge), `image_folder.py`, `ensemble.py`, `field_dataset.py`, `balanced_sampler.py`, `weighted_sampler.py`, `retry.py`, `vo_pairs.py` (synthetic VO pairs) |
|
|
227
|
+
| `genml_kit.models` | model/processor/tokenizer registry; `timm/`, `convvit/`, `uvito/`, `cls_model_wrapper/`, `seg_model_wrapper/` backends; `decoders/` (pluggable segmentation decoders); `generative.py` (`GenerativeLM` contract), `tokenizer.py` (`TOKENIZERS`), `processors/base.py` |
|
|
228
|
+
| `genml_kit.datasets` | `hf_proxy.py` (HuggingFace → PyTorch bridge), `image_folder.py`, `seg_folder.py` (paired image/mask), `det_folder.py` (COCO detection), `text_proxy.py` (HF text → items), `preference_proxy.py` (chosen/rejected pairs), `ensemble.py`, `field_dataset.py`, `balanced_sampler.py`, `weighted_sampler.py`, `retry.py`, `vo_pairs.py` (synthetic VO pairs) |
|
|
197
229
|
| `genml_kit.geometry` | `similarity.py` (2-D similarity algebra, batched differentiable Umeyama, backward-map warp) — the math behind the visual-odometry front-end lives in [`vo/README.md`](vo/README.md) |
|
|
198
230
|
| `genml_kit.io` | `checkpointing.py` (atomic saves, LoRA state, remote fetch), `storage_utils.py` (S3 / GCS / R2) |
|
|
199
|
-
| `genml_kit.utils` | glog-style logging, CLI arg groups, seeding, signal handling, GPU info, tables, external `.py` script loading, `image_dump`, transformer init helpers |
|
|
231
|
+
| `genml_kit.utils` | glog-style logging, CLI arg groups + `shared_args.py` (globally shared `ARG_*` CLI flags), seeding, signal handling, GPU info, tables, external `.py` script loading, `image_dump`, transformer init helpers, `generation.py` (samplers + batched `generate`) |
|
|
232
|
+
| `genml_kit.cli` | console-script entry points: `train.py` (`genml-kit-train`), `infer.py` (`genml-kit-infer`), `llm.py` (`genml-kit-llm`, `generate` / `eval` subcommands), `tune.py` (`genml-kit-tune`, `run` / `report` subcommands) |
|
|
233
|
+
| `genml_kit.search` | hyperparameter search: `space.py` (typed dimensions + argv encoding), `study.py` (declarative study spec), `samplers.py` (`SEARCHERS`: grid/random/tpe/gpei), `pruners.py` (`PRUNERS`: none/median/asha/hyperband), `runner.py` (subprocess-isolated trials), `store.py` (resumable ledger), `trial.py` |
|
|
200
234
|
|
|
201
235
|
## Installation
|
|
202
236
|
|
|
@@ -215,10 +249,30 @@ pip install "genml_kit[s3]"
|
|
|
215
249
|
# With LoRA fine-tuning:
|
|
216
250
|
pip install "genml_kit[lora]"
|
|
217
251
|
|
|
252
|
+
# With causal-LM fine-tuning (LoRA + transformers):
|
|
253
|
+
pip install "genml_kit[llm]"
|
|
254
|
+
|
|
255
|
+
# With a bare SentencePiece tokenizer.model (the 'spm' tokenizer entry):
|
|
256
|
+
pip install "genml_kit[llm-spm]"
|
|
257
|
+
|
|
218
258
|
# With UVito model support:
|
|
219
259
|
pip install "genml_kit[uvito]"
|
|
220
260
|
|
|
221
|
-
#
|
|
261
|
+
# With the scipy-backed hyperparameter sampler (genml-kit-tune gpei):
|
|
262
|
+
pip install "genml_kit[tune]"
|
|
263
|
+
|
|
264
|
+
# With NVML GPU utilization in the training logs (the 'util=...%' field):
|
|
265
|
+
pip install "genml_kit[gpu]"
|
|
266
|
+
|
|
267
|
+
# With Weights & Biases metric streaming (the 'wandb' metrics sink):
|
|
268
|
+
pip install "genml_kit[wandb]"
|
|
269
|
+
|
|
270
|
+
# With flash-attn packing support (requires the CUDA toolkit with CUDA_HOME
|
|
271
|
+
# set to an nvcc, plus torch already installed):
|
|
272
|
+
pip install "genml_kit[flash]"
|
|
273
|
+
|
|
274
|
+
# Everything above in one shot (gcs, s3, lora, llm, llm-spm, timm, uvito, rl, tune, gpu, wandb):
|
|
275
|
+
# (flash is intentionally excluded: it needs a CUDA toolkit to build)
|
|
222
276
|
pip install "genml_kit[all]"
|
|
223
277
|
```
|
|
224
278
|
|
|
@@ -229,10 +283,19 @@ NumPy, scikit-learn ≥ 1.3, XGBoost ≥ 2.0, Pillow, tensorboard.
|
|
|
229
283
|
|
|
230
284
|
The fastest way to get started:
|
|
231
285
|
|
|
286
|
+
<!-- genml-cli-test
|
|
287
|
+
name: quick-start-cifar10
|
|
288
|
+
network: true
|
|
289
|
+
override:
|
|
290
|
+
- --dataset {tmp}/img
|
|
291
|
+
- -.--image_column
|
|
292
|
+
extra: --epochs 1 --max_items 16
|
|
293
|
+
-->
|
|
232
294
|
```bash
|
|
233
295
|
# Fine-tune a ViT on an image-classification dataset (5 epochs, ~2 minutes on GPU)
|
|
234
296
|
genml-kit-train --model google/vit-base-patch16-224 \
|
|
235
|
-
--dataset cifar10 \
|
|
297
|
+
--dataset uoft-cs/cifar10 \
|
|
298
|
+
--image_column img \
|
|
236
299
|
--label_column label \
|
|
237
300
|
--epochs 5 \
|
|
238
301
|
--batch_size 32 \
|
|
@@ -293,12 +356,13 @@ SimMIM minimizes the mean squared reconstruction error over masked patches:
|
|
|
293
356
|
|
|
294
357
|
$$
|
|
295
358
|
\large
|
|
296
|
-
\mathcal{L}_{\text{MIM}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left
|
|
359
|
+
\mathcal{L}_{\text{MIM}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\\| \hat{x}_i - x_i \right\\|_2^2
|
|
297
360
|
$$
|
|
298
361
|
|
|
299
362
|
where:
|
|
300
363
|
|
|
301
|
-
- $`x_i`$ — the pixels of the $`i`$-th original patch
|
|
364
|
+
- $`x_i`$ — the pixels of the $`i`$-th original patch ($`i = 1, \dots, N`$;
|
|
365
|
+
$`N`$ = total number of patches);
|
|
302
366
|
- $`\hat{x}_i`$ — the decoder's reconstruction of that patch;
|
|
303
367
|
- $`\mathcal{M}`$ — the set of masked patch indices, and $`|\mathcal{M}|`$ its
|
|
304
368
|
size (how many patches are masked);
|
|
@@ -325,7 +389,7 @@ masked region. The objective is regression in representation space:
|
|
|
325
389
|
|
|
326
390
|
$$
|
|
327
391
|
\large
|
|
328
|
-
\mathcal{L}_{\text{IJEPA}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left
|
|
392
|
+
\mathcal{L}_{\text{IJEPA}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\\| q_\theta(f_\theta(\text{context}))_i - \mathop{\mathrm{stopgrad}}\left(f_\xi(x_i)\right) \right\\|_2^2
|
|
329
393
|
$$
|
|
330
394
|
|
|
331
395
|
where:
|
|
@@ -344,7 +408,7 @@ an exponential moving average:
|
|
|
344
408
|
|
|
345
409
|
$$
|
|
346
410
|
\large
|
|
347
|
-
\xi \leftarrow m
|
|
411
|
+
\xi \leftarrow m \\, \xi + (1 - m) \\, \theta
|
|
348
412
|
$$
|
|
349
413
|
|
|
350
414
|
where $`m`$ is the momentum (`--teacher_momentum`) and the arrow denotes
|
|
@@ -371,6 +435,28 @@ Three mechanism details complete the picture:
|
|
|
371
435
|
matures, the teacher freezes progressively, stabilizing the targets late in
|
|
372
436
|
training.
|
|
373
437
|
|
|
438
|
+
Run it like any other image pre-training configuration — here on a small
|
|
439
|
+
unlabeled collection:
|
|
440
|
+
|
|
441
|
+
<!-- genml-cli-test
|
|
442
|
+
name: pretrain-ijepa
|
|
443
|
+
network: true
|
|
444
|
+
gated: true
|
|
445
|
+
override:
|
|
446
|
+
- --datasets {tmp}/img
|
|
447
|
+
extra: --epochs 1 --batch_size 2 --max_items 16 --predictor_heads 8 --predictor_depth 2 --ijepa_weight 0.5
|
|
448
|
+
-->
|
|
449
|
+
```bash
|
|
450
|
+
genml-kit-train --pipeline images --method ijepa \
|
|
451
|
+
--model convvit \
|
|
452
|
+
--datasets uoft-cs/cifar10 \
|
|
453
|
+
--image_size 32 \
|
|
454
|
+
--batch_size 32 \
|
|
455
|
+
--epochs 100 \
|
|
456
|
+
--predictor_heads 8 \
|
|
457
|
+
--lr 1e-4
|
|
458
|
+
```
|
|
459
|
+
|
|
374
460
|
**SupCon** (Supervised Contrastive Learning): Uses labels to define "positive"
|
|
375
461
|
pairs (same class) and "negative" pairs (different classes). The loss pulls
|
|
376
462
|
features of same-class images together and pushes different-class features
|
|
@@ -406,7 +492,7 @@ Fix one example $`i`$, called the *anchor*. Its *positive set* is
|
|
|
406
492
|
|
|
407
493
|
$$
|
|
408
494
|
\large
|
|
409
|
-
\mathcal{P}(i) =
|
|
495
|
+
\mathcal{P}(i) = \\{\\, j : j \neq i \text{ and } y_j = y_i \\,\\}
|
|
410
496
|
$$
|
|
411
497
|
|
|
412
498
|
i.e. all other examples in the batch that share the anchor's class label
|
|
@@ -469,25 +555,58 @@ into 2 **global** crops plus `--dino_local_num` (8) small **local** crops:
|
|
|
469
555
|
the teacher sees only global views, the student sees all of them, so the
|
|
470
556
|
student must produce globally consistent features from partial views.
|
|
471
557
|
|
|
472
|
-
Both towers end in an MLP projection head
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
$`c
|
|
558
|
+
Both towers end in an MLP projection head that maps each backbone feature to a
|
|
559
|
+
$`D`$-dimensional vector (`--dino_proj_dim`, 256); these vectors are turned
|
|
560
|
+
into probability distributions over the $`D`$ projection dimensions. The
|
|
561
|
+
teacher's logits are *centered* (a running-mean vector $`c`$ is subtracted) and
|
|
562
|
+
*sharpened* (divided by a small temperature $`\tau_\xi`$) before the softmax,
|
|
563
|
+
while the student's logits are divided only by its larger temperature
|
|
564
|
+
$`\tau_\theta`$. Writing $`f_\theta(x)`$ and $`f_\xi(x)`$ for the student and
|
|
565
|
+
teacher projections of an image $`x`$, the center is updated once per step,
|
|
476
566
|
|
|
477
567
|
$$
|
|
478
568
|
\large
|
|
479
|
-
c \leftarrow m_c
|
|
569
|
+
c \leftarrow m_c \\, c + (1 - m_c) \cdot \frac{1}{B_g}\sum_{i=1}^{B_g} f_\xi(x_i)
|
|
480
570
|
$$
|
|
481
571
|
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
572
|
+
and the teacher target and student prediction are
|
|
573
|
+
|
|
574
|
+
$$
|
|
575
|
+
\large
|
|
576
|
+
p_\xi(x) = \mathrm{softmax}\left(\frac{f_\xi(x) - c}{\tau_\xi}\right),
|
|
577
|
+
\qquad
|
|
578
|
+
p_\theta(x) = \mathrm{softmax}\left(\frac{f_\theta(x)}{\tau_\theta}\right)
|
|
579
|
+
$$
|
|
485
580
|
|
|
486
581
|
$$
|
|
487
582
|
\large
|
|
488
|
-
\mathcal{L} = - \sum_{
|
|
583
|
+
\mathcal{L} = - \frac{1}{N}\sum_{n=1}^{N} \sum_{d=1}^{D} p_\xi(x^{(n)})_d \\, \log p_\theta(x^{(n)})_d
|
|
489
584
|
$$
|
|
490
585
|
|
|
586
|
+
where:
|
|
587
|
+
|
|
588
|
+
- $`f_\theta(x), f_\xi(x) \in \mathbb{R}^{D}`$ — the student and teacher
|
|
589
|
+
projection-head outputs for image $`x`$ ($`D`$ = projection dimension,
|
|
590
|
+
`--dino_proj_dim`);
|
|
591
|
+
- $`c \in \mathbb{R}^{D}`$ — the running center: a per-dimension mean of the
|
|
592
|
+
teacher's outputs, updated once per step with momentum $`m_c`$
|
|
593
|
+
(`--dino_center_momentum`, 0.9) over the $`B_g`$ teacher *global-crop*
|
|
594
|
+
outputs in the batch (two global views per image, so $`B_g = 2B`$). It is
|
|
595
|
+
subtracted from the teacher logits so that no single dimension can dominate
|
|
596
|
+
the target;
|
|
597
|
+
- $`\tau_\xi`$ — the teacher temperature (`--dino_teacher_temp`, 0.04); small,
|
|
598
|
+
so the teacher distribution is sharp (confident);
|
|
599
|
+
- $`\tau_\theta`$ — the student temperature (`--dino_student_temp`, 0.1);
|
|
600
|
+
larger, so the student distribution is softer and its gradients stay alive;
|
|
601
|
+
- $`p_\xi(x)`$ — the teacher's target distribution over the $`D`$ dimensions
|
|
602
|
+
(centered, sharpened, softmaxed, and detached from the gradient graph);
|
|
603
|
+
- $`p_\theta(x)`$ — the student's predicted distribution over the same
|
|
604
|
+
$`D`$ dimensions;
|
|
605
|
+
- $`x^{(n)}`$ — the $`n`$-th (student crop, teacher view) pairing: each global
|
|
606
|
+
view is paired with the *other* global view of the same image, and each local
|
|
607
|
+
crop with the teacher's second global view. The loss averages the per-pairing
|
|
608
|
+
cross-entropy over all $`N`$ pairings.
|
|
609
|
+
|
|
491
610
|
Centering and sharpening are the two anti-collapse mechanisms: centering
|
|
492
611
|
cancels any component the teacher produces for *every* input (a constant
|
|
493
612
|
output carries no information and would otherwise still lower the loss), and
|
|
@@ -496,27 +615,72 @@ its argmax rather than drifting toward a flat, uninformative distribution.
|
|
|
496
615
|
Because the targets come from the teacher and the teacher follows the student,
|
|
497
616
|
this is self-distillation — there are no external labels anywhere.
|
|
498
617
|
|
|
618
|
+
A DINO run adds the multi-crop knobs on top of the standard configuration:
|
|
619
|
+
|
|
620
|
+
<!-- genml-cli-test
|
|
621
|
+
name: pretrain-dino
|
|
622
|
+
network: true
|
|
623
|
+
gated: true
|
|
624
|
+
override:
|
|
625
|
+
- --datasets {tmp}/img
|
|
626
|
+
extra: --epochs 1 --batch_size 2 --max_items 16 --dino_local_num 2 --dino_local_size 16 --dino_global_size 32
|
|
627
|
+
-->
|
|
628
|
+
```bash
|
|
629
|
+
genml-kit-train --pipeline images --method dino \
|
|
630
|
+
--model convvit \
|
|
631
|
+
--datasets uoft-cs/cifar10 \
|
|
632
|
+
--image_size 32 \
|
|
633
|
+
--batch_size 32 \
|
|
634
|
+
--epochs 100 \
|
|
635
|
+
--lr 1e-4
|
|
636
|
+
```
|
|
637
|
+
|
|
499
638
|
**BYOL** (**B**ootstrap **Y**our **O**wn **L**atent): Grill et al.,
|
|
500
639
|
[Bootstrap Your Own Latent](https://arxiv.org/abs/2006.07733), NeurIPS 2020.
|
|
501
640
|
The same two-tower idea as DINO but with two augmented views and no
|
|
502
641
|
multi-crop. The **online** network (encoder + projector + predictor MLP) and
|
|
503
642
|
the **target** network (encoder + projector, EMA copy) process the two views;
|
|
504
|
-
only the online tower receives gradients.
|
|
505
|
-
|
|
506
|
-
those projections, the loss is the symmetric cosine regression
|
|
643
|
+
only the online tower receives gradients. The loss is the symmetric cosine
|
|
644
|
+
regression
|
|
507
645
|
|
|
508
646
|
$$
|
|
509
647
|
\large
|
|
510
648
|
\mathcal{L} = 2 - \cos(p_1, z_2) - \cos(p_2, z_1)
|
|
511
649
|
$$
|
|
512
650
|
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
651
|
+
where:
|
|
652
|
+
|
|
653
|
+
- $`z_1, z_2`$ — the target network's projections of the two views
|
|
654
|
+
(stop-gradient, so no gradients flow into the target tower);
|
|
655
|
+
- $`p_1, p_2`$ — the online network's *predictions* of those projections;
|
|
656
|
+
- $`\cos(u, v) = u^\top v / (\lVert u \rVert_2 \lVert v \rVert_2)`$ — the
|
|
657
|
+
cosine of the angle between the two vectors.
|
|
658
|
+
|
|
659
|
+
The loss is 0 when predictions perfectly match the target projections and
|
|
660
|
+
reaches 4 when they point in opposite directions. Two asymmetries keep this
|
|
661
|
+
from collapsing to a constant: the predictor exists only on the online side
|
|
662
|
+
(predicting a target is easier than being one), and the targets move slowly
|
|
663
|
+
under the EMA — exactly the teacher-follows-student trick noted under I-JEPA.
|
|
664
|
+
|
|
665
|
+
The BYOL configuration differs only in the method name:
|
|
666
|
+
|
|
667
|
+
<!-- genml-cli-test
|
|
668
|
+
name: pretrain-byol
|
|
669
|
+
network: true
|
|
670
|
+
gated: true
|
|
671
|
+
override:
|
|
672
|
+
- --datasets {tmp}/img
|
|
673
|
+
extra: --epochs 1 --batch_size 2 --max_items 16 --byol_proj_hidden 256 --byol_predictor_hidden 256 --byol_final_momentum 1.0
|
|
674
|
+
-->
|
|
675
|
+
```bash
|
|
676
|
+
genml-kit-train --pipeline images --method byol \
|
|
677
|
+
--model convvit \
|
|
678
|
+
--datasets uoft-cs/cifar10 \
|
|
679
|
+
--image_size 32 \
|
|
680
|
+
--batch_size 32 \
|
|
681
|
+
--epochs 100 \
|
|
682
|
+
--lr 1e-4
|
|
683
|
+
```
|
|
520
684
|
|
|
521
685
|
### Typical Hyperparameters
|
|
522
686
|
|
|
@@ -546,6 +710,9 @@ size and GPU memory:
|
|
|
546
710
|
- DINO and BYOL train on smaller crops (224) with a larger LR because their
|
|
547
711
|
loss acts on a small projection space rather than every pixel; the defaults
|
|
548
712
|
above match the method-specific `--dino_*` / `--byol_*` flags.
|
|
713
|
+
- The DINO/BYOL loss lives in that projection space (`--dino_proj_dim` /
|
|
714
|
+
`--byol_proj_dim`, both 256), not in the backbone feature space: the
|
|
715
|
+
objective compares 256-dimensional distributions or vectors, not pixels.
|
|
549
716
|
- `--samples_per_class 16` with `--batch_size 64` gives 4 classes per batch
|
|
550
717
|
on a 7-class dataset. batch_size need not be divisible by
|
|
551
718
|
samples_per_class; the remainder is spread evenly across the per-class
|
|
@@ -555,6 +722,16 @@ size and GPU memory:
|
|
|
555
722
|
|
|
556
723
|
### Example: Full Pre-Training Pipeline
|
|
557
724
|
|
|
725
|
+
<!-- genml-cli-test
|
|
726
|
+
name: pretrain-full-pipeline
|
|
727
|
+
network: true
|
|
728
|
+
override:
|
|
729
|
+
- --datasets {tmp}/img
|
|
730
|
+
- --dataset {tmp}/img
|
|
731
|
+
- --label_column label
|
|
732
|
+
- -.--source_checkpoint
|
|
733
|
+
extra: --epochs 1
|
|
734
|
+
-->
|
|
558
735
|
```bash
|
|
559
736
|
# Step 1: Pre-train with SimMIM on two large image collections.
|
|
560
737
|
# v4.2: pre-training is a *configuration* of the unified genml-kit-train
|
|
@@ -584,6 +761,14 @@ genml-kit-train --model convvit \
|
|
|
584
761
|
|
|
585
762
|
### Example: Supervised Contrastive Pre-Training
|
|
586
763
|
|
|
764
|
+
<!-- genml-cli-test
|
|
765
|
+
name: pretrain-supcon
|
|
766
|
+
network: true
|
|
767
|
+
override:
|
|
768
|
+
- --datasets {tmp}/img
|
|
769
|
+
- --label_column label
|
|
770
|
+
extra: --epochs 1
|
|
771
|
+
-->
|
|
587
772
|
```bash
|
|
588
773
|
genml-kit-train --pipeline images --method supcon \
|
|
589
774
|
--model convvit \
|
|
@@ -611,7 +796,6 @@ genml-kit-train --pipeline images --method supcon \
|
|
|
611
796
|
| `--datasets` | (required) | Space-separated dataset names or local paths. |
|
|
612
797
|
| `--cache_dir` | `None` | HuggingFace cache directory for downloads. |
|
|
613
798
|
| `--remote_checkpoint` | `None` | Remote URI for checkpoint sync (`gs://BUCKET/PREFIX`, `r2://BUCKET/PREFIX`, or `s3://BUCKET/PREFIX`). |
|
|
614
|
-
| `--hf_token` | `None` | HuggingFace token for gated datasets (or set `HF_TOKEN` env var). |
|
|
615
799
|
| `--image_column` | auto-detected | Explicit HF image column name. |
|
|
616
800
|
| `--label_column` | `None` | Explicit HF label column name. Required by `--method supcon` if non-standard. |
|
|
617
801
|
| `STRICT_DATASETS` (env) | `False` | Abort on first dataset-loading failure instead of skipping. |
|
|
@@ -619,6 +803,8 @@ genml-kit-train --pipeline images --method supcon \
|
|
|
619
803
|
| `--batch_size` | `32` | Per-GPU batch size. |
|
|
620
804
|
| `--seed` | `None` | Explicit RNG seed: pins the shuffling and enables full determinism (cuDNN deterministic kernels, benchmark off). Omit (default) to keep runs fast — RNG streams are still seeded from `$GENML_KIT_SEED` (default 42). See [Reproducibility](#reproducibility). |
|
|
621
805
|
| `--epochs` | `5` | Number of epochs. The recipes below override this for pre-training. |
|
|
806
|
+
| `--max_batches` | unset | Cap the micro-batches **per epoch**; the next epoch still runs (up to `--epochs`). Unset drains the loader each epoch. |
|
|
807
|
+
| `--max_train_time` | unset | Wall-clock budget **in seconds for the whole run**. Stops mid-epoch once exceeded (DDP-synchronized); unset means no time limit. |
|
|
622
808
|
| `--lr` | `3e-5` | Peak learning rate (base). The recipes below override this. |
|
|
623
809
|
| `--amp_dtype` | `None` | Mixed precision: `float16` or `bfloat16`. Omit to disable. |
|
|
624
810
|
| `--num_workers` | `4` | DataLoader worker processes. |
|
|
@@ -633,6 +819,10 @@ genml-kit-train --pipeline images --method supcon \
|
|
|
633
819
|
| `--norm_history` | `0` | Keep last N norm snapshots per parameter for trend analysis. |
|
|
634
820
|
| `--trend_top_n` | `10` | Show top N params in trend table by abs change %. `0` = show all. |
|
|
635
821
|
| `--grad_clip` | `1.0` | Maximum gradient norm for clipping. `0` disables. |
|
|
822
|
+
| `--ema_decay` | `0.0` | Weights-EMA decay for validation and the best checkpoint. `0` disables the shadow (supported methods only -- see [Weights EMA](#weights-ema)). |
|
|
823
|
+
| `--ema_final_decay` | `None` | Optional ramp target: the effective decay moves linearly to this value over the run. |
|
|
824
|
+
| `--ema_warmup_steps` | `0` | Linear warmup of the effective decay from `0` up to `--ema_decay` over this many optimizer steps. Ignored when `--ema_final_decay` is set. |
|
|
825
|
+
| `--ema_eval` | `False` | Evaluate and best-save the EMA weights instead of the raw trajectory (`best.pt` carries EMA weights; `latest.pt` always carries the raw trajectory plus the shadow). |
|
|
636
826
|
| `--lr_group` | `None` | Per-parameter-group learning rates (repeatable). Format: `"REGEX=LR"`. |
|
|
637
827
|
| `--llrd_decay` | `None` | Layer-wise learning rate decay factor. |
|
|
638
828
|
| `--vis_every` | `0` | Log reconstruction visualisation every N steps (SimMIM only). |
|
|
@@ -641,8 +831,9 @@ genml-kit-train --pipeline images --method supcon \
|
|
|
641
831
|
| `--proc_arg` | `{}` | Override processor configuration (repeatable). |
|
|
642
832
|
| `--optimizer` | `AdamW` | `torch.optim` optimizer class name or `.py` script path. |
|
|
643
833
|
| `--opt_arg` | `{}` | Extra optimizer kwargs (repeatable). |
|
|
644
|
-
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path
|
|
834
|
+
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path, or a built-in warmup shape: `warmup_cosine`, `warmup_linear`, `warmup_rsqrt`. |
|
|
645
835
|
| `--sched_arg` | `{}` | Extra scheduler kwargs (repeatable). |
|
|
836
|
+
| `--warmup_epochs` | `0` | Linear LR ramp length in epochs (integer or fraction) for the `warmup_*` schedulers; 0 disables the ramp. See [Warmup](#warmup). |
|
|
646
837
|
| `--source_checkpoint` | `None` | Path to source checkpoint to absorb parameters from. |
|
|
647
838
|
| `--param_rename` | `None` | Regex-based key rename patterns (`SEARCH;REPLACE`). |
|
|
648
839
|
| `--grad_checkpoint` | `False` | Enable gradient checkpointing. Reduces activation memory by ~40-50% at the cost of ~25-35% more compute per step. Enables larger batch sizes. See [Gradient Checkpointing](#gradient-checkpointing). |
|
|
@@ -669,8 +860,8 @@ single dataset is large enough for effective pre-training on its own.
|
|
|
669
860
|
|
|
670
861
|
Supported dataset types:
|
|
671
862
|
- **HuggingFace datasets** — any HF dataset ID that returns decoded image
|
|
672
|
-
data (e.g. `cifar10`, `food101`). Gated datasets require
|
|
673
|
-
|
|
863
|
+
data (e.g. `cifar10`, `food101`). Gated datasets require the `HF_TOKEN`
|
|
864
|
+
environment variable.
|
|
674
865
|
- **Local image directories** — pass a path to a folder of images
|
|
675
866
|
(ImageFolder format).
|
|
676
867
|
|
|
@@ -692,6 +883,167 @@ Labels are automatically remapped to a shared global label space across all
|
|
|
692
883
|
datasets, so mixing datasets with overlapping but differently-named classes
|
|
693
884
|
works transparently.
|
|
694
885
|
|
|
886
|
+
#### Streaming (Iterable) Datasets
|
|
887
|
+
|
|
888
|
+
A dataset does not have to fit on disk or in RAM. When a source can only be
|
|
889
|
+
*iterated* (no index, no length), the pipeline streams it: items are read once,
|
|
890
|
+
converted on the fly, and never materialised into an index space.
|
|
891
|
+
|
|
892
|
+
Streaming sources can be used in two places:
|
|
893
|
+
|
|
894
|
+
- **`--dataset` with `--pipeline text`** — pass `streaming=True` through
|
|
895
|
+
`--dataset_arg` to have the HuggingFace loader return a streaming source
|
|
896
|
+
instead of a materialised split:
|
|
897
|
+
|
|
898
|
+
<!-- genml-cli-test-exclude: illustrative streaming run against a user corpus; covered by tests/test_streaming_text_pipeline.py -->
|
|
899
|
+
```bash
|
|
900
|
+
genml-kit-train --pipeline text --model gpt2 \
|
|
901
|
+
--dataset my-org/huge-corpus \
|
|
902
|
+
--text_column text \
|
|
903
|
+
--dataset_arg streaming=True \
|
|
904
|
+
--epochs 1
|
|
905
|
+
```
|
|
906
|
+
|
|
907
|
+
- **`--datasets` (image pre-training ensemble)** — use a `webdataset:` spec
|
|
908
|
+
(see below). If every dataset in an ensemble streams, the ensemble streams
|
|
909
|
+
too, in declaration order.
|
|
910
|
+
|
|
911
|
+
- **`--dataset` with `--pipeline images` (classification)** — pass
|
|
912
|
+
`streaming=True` through `--dataset_arg` to stream an image *classification*
|
|
913
|
+
source instead of materialising it. The train/val split becomes a positional
|
|
914
|
+
one (`IterableDatasetFilter`), and the label column must be a typed
|
|
915
|
+
`ClassLabel` so the class vocabulary is read from the dataset's `features`
|
|
916
|
+
without scanning the corpus. Class weighting (`--sampler weighted` /
|
|
917
|
+
`balanced`, inverse-frequency weights) needs the whole label column, so it is
|
|
918
|
+
refused for streams with a clear error.
|
|
919
|
+
|
|
920
|
+
**What works, and what does not.** A stream has no index space, so anything
|
|
921
|
+
that needs one is refused with a clear error rather than silently misbehaving:
|
|
922
|
+
|
|
923
|
+
| Feature | Random-access | Streaming |
|
|
924
|
+
| --- | --- | --- |
|
|
925
|
+
| Plain batching | yes | yes |
|
|
926
|
+
| `--length_bucket` | yes | yes (over a bounded buffer) |
|
|
927
|
+
| `--packing wrapped` / `bfd` | yes | yes (over a bounded buffer) |
|
|
928
|
+
| `--val_split` | index slice | positional split (`IterableDatasetFilter`) |
|
|
929
|
+
| `--sampler weighted` / `balanced` | yes | not available (needs labels across the whole corpus) |
|
|
930
|
+
| `--max_items` | yes | yes (caps the stream) |
|
|
931
|
+
| Classification (`--dataset`) | yes | yes, with a typed `ClassLabel` label column |
|
|
932
|
+
|
|
933
|
+
Because a stream cannot be shuffled by index, `shuffle` is disabled for it and
|
|
934
|
+
its order is the source order. Under DDP, `--shuffle_buffer_k` supplies
|
|
935
|
+
batch-level shuffling instead — see below.
|
|
936
|
+
|
|
937
|
+
### `--shuffle_buffer_k` (Streaming Shuffle)
|
|
938
|
+
|
|
939
|
+
A streaming source cannot be shuffled by sampler: there is no index space to
|
|
940
|
+
permute. Instead the *batch* stream is shuffled, with a bounded round-robin
|
|
941
|
+
buffer.
|
|
942
|
+
|
|
943
|
+
`--shuffle_buffer_k K` sets the size of that buffer in **batches per rank**, so
|
|
944
|
+
the buffer holds `world_size * K` batches. It only has an effect on a streaming
|
|
945
|
+
loader; random-access datasets shuffle by sampler and ignore it.
|
|
946
|
+
|
|
947
|
+
- **`K = 10`** (default) — a reasonable trade-off between shuffle quality and
|
|
948
|
+
the number of batches buffered in memory.
|
|
949
|
+
- **`K = 1`** — the minimal round-robin: each rank takes one batch per round,
|
|
950
|
+
with no permuting.
|
|
951
|
+
- **`K = 0`** — round-robin without permuting.
|
|
952
|
+
|
|
953
|
+
Memory cost scales with `K`: each buffered batch is a full batch of decoded
|
|
954
|
+
samples, so raise `K` only as far as your RAM allows.
|
|
955
|
+
|
|
956
|
+
<!-- genml-cli-test-exclude: illustrative DDP invocation over local shards; covered by tests/test_iterable_ddp.py -->
|
|
957
|
+
```bash
|
|
958
|
+
torchrun --nproc_per_node=4 -m genml_kit.cli.train --model convvit \
|
|
959
|
+
--datasets "webdataset:/data/train-{0000..0999}.tar" \
|
|
960
|
+
--shuffle_buffer_k 20 \
|
|
961
|
+
--batch_size 64
|
|
962
|
+
```
|
|
963
|
+
|
|
964
|
+
### WebDataset Tar Shards (`webdataset:`)
|
|
965
|
+
|
|
966
|
+
For corpora already stored as webdataset tar shards, pass a spec of the form
|
|
967
|
+
`webdataset:<shard-list>` inside `--datasets`. The shard list is passed to
|
|
968
|
+
webdataset verbatim, so its brace expansion applies:
|
|
969
|
+
|
|
970
|
+
<!-- genml-cli-test-exclude: illustrative webdataset pre-training run; covered by tests/test_webdataset_pipeline.py -->
|
|
971
|
+
```bash
|
|
972
|
+
genml-kit-train --pipeline images --model convvit --method simmim \
|
|
973
|
+
--datasets "webdataset:/data/train-{0000..0999}.tar" \
|
|
974
|
+
--batch_size 64 \
|
|
975
|
+
--epochs 100
|
|
976
|
+
```
|
|
977
|
+
|
|
978
|
+
A webdataset "sample" is a group of members named `<key>.<extension>` inside the
|
|
979
|
+
tar (`s0001.png`, `s0001.cls`, `s0001.txt`). Members are decoded by extension:
|
|
980
|
+
`png`/`jpg` become images, `txt` a string, `cls` an integer, `json` a dict, `npy`
|
|
981
|
+
an ndarray, `pth` a tensor; anything else stays raw bytes.
|
|
982
|
+
|
|
983
|
+
The image member is detected from the first sample (its name is the extension,
|
|
984
|
+
e.g. `png`). If a shard carries more than one image member, name it explicitly:
|
|
985
|
+
|
|
986
|
+
```bash
|
|
987
|
+
--datasets "webdataset:/data/train-{0000..0999}.tar" --image_column png
|
|
988
|
+
```
|
|
989
|
+
|
|
990
|
+
The `webdataset` package ships with the core install (it is a hard dependency,
|
|
991
|
+
like `datasets` and `transformers`), so no extra is needed.
|
|
992
|
+
|
|
993
|
+
A webdataset member is keyed by extension, so these sources are unlabeled by
|
|
994
|
+
default (there is no dataset-level class vocabulary). Use a method that does not
|
|
995
|
+
need labels (`simmim`, `dino`, `byol`, `ijepa`), or select the label member
|
|
996
|
+
explicitly in code with `IterableFieldSelectorDataset`.
|
|
997
|
+
|
|
998
|
+
### HuggingFace Datasets
|
|
999
|
+
|
|
1000
|
+
Any HF dataset ID whose rows carry decoded image data works directly as
|
|
1001
|
+
`--dataset` (classification) or inside `--datasets` (ensemble). Public
|
|
1002
|
+
datasets need no credentials:
|
|
1003
|
+
|
|
1004
|
+
<!-- genml-cli-test
|
|
1005
|
+
name: hf-dataset-public
|
|
1006
|
+
network: true
|
|
1007
|
+
service: hf
|
|
1008
|
+
extra: --epochs 1 --max_items 64 --batch_size 8
|
|
1009
|
+
-->
|
|
1010
|
+
```bash
|
|
1011
|
+
genml-kit-train --model convvit \
|
|
1012
|
+
--dataset uoft-cs/cifar10 \
|
|
1013
|
+
--image_column img \
|
|
1014
|
+
--label_column label \
|
|
1015
|
+
--image_size 224 \
|
|
1016
|
+
--epochs 5
|
|
1017
|
+
```
|
|
1018
|
+
|
|
1019
|
+
`--max_items` caps each split (applied *after* the train/val split), which
|
|
1020
|
+
keeps these examples fast (the full `cifar10` train split is 50k images).
|
|
1021
|
+
|
|
1022
|
+
### Gated / Private Datasets
|
|
1023
|
+
|
|
1024
|
+
Gated datasets require a HuggingFace token whose owner has been granted access.
|
|
1025
|
+
Set the `HF_TOKEN` environment variable:
|
|
1026
|
+
|
|
1027
|
+
<!-- genml-cli-test
|
|
1028
|
+
name: hf-dataset-gated
|
|
1029
|
+
network: true
|
|
1030
|
+
service: hf
|
|
1031
|
+
gated: true
|
|
1032
|
+
extra: --epochs 1 --batch_size 2
|
|
1033
|
+
-->
|
|
1034
|
+
```bash
|
|
1035
|
+
genml-kit-train --model convvit \
|
|
1036
|
+
--dataset sirtolkien/autotrain-data-test \
|
|
1037
|
+
--image_column image \
|
|
1038
|
+
--label_column target \
|
|
1039
|
+
--image_size 224 \
|
|
1040
|
+
--epochs 5
|
|
1041
|
+
```
|
|
1042
|
+
|
|
1043
|
+
Because the gated dataset cannot be fetched without a grant, the README test
|
|
1044
|
+
harness remaps `--dataset` to a synthesized local fixture and runs the
|
|
1045
|
+
example offline.
|
|
1046
|
+
|
|
695
1047
|
### Preparing Datasets
|
|
696
1048
|
|
|
697
1049
|
Some datasets store images inside zip archives or need custom preprocessing
|
|
@@ -704,10 +1056,18 @@ python my_prepare_script.py --output_dir ./prepared_images
|
|
|
704
1056
|
|
|
705
1057
|
Then use the extracted directory as a local dataset:
|
|
706
1058
|
|
|
1059
|
+
<!-- genml-cli-test
|
|
1060
|
+
name: pretrain-prepared-images
|
|
1061
|
+
network: true
|
|
1062
|
+
override:
|
|
1063
|
+
- --datasets {tmp}/img
|
|
1064
|
+
extra: --epochs 1
|
|
1065
|
+
-->
|
|
707
1066
|
```bash
|
|
708
1067
|
genml-kit-train --pipeline images --method simmim \
|
|
1068
|
+
--model convvit \
|
|
709
1069
|
--datasets ./prepared_images ./other-images \
|
|
710
|
-
--image_size 448 --batch_size 32
|
|
1070
|
+
--image_size 448 --batch_size 32 --epochs 20
|
|
711
1071
|
```
|
|
712
1072
|
|
|
713
1073
|
For a concrete worked example of such a preparation script, see
|
|
@@ -823,6 +1183,37 @@ The confusion matrix uses rows for true classes and columns for predicted
|
|
|
823
1183
|
classes. Diagonal entries are correct predictions. The compact confusion
|
|
824
1184
|
summary reports each class's recall and its largest confusion destinations.
|
|
825
1185
|
|
|
1186
|
+
Two ranking/calibration metrics are also reported when the validation
|
|
1187
|
+
predictions carry logits (the default for model-based methods):
|
|
1188
|
+
|
|
1189
|
+
- **AUROC (macro, one-vs-rest):** how well the model's scores rank a random
|
|
1190
|
+
positive of a class above a random negative, averaged over classes. It is
|
|
1191
|
+
threshold-free and therefore the right summary for imbalanced clinical
|
|
1192
|
+
workloads where the operating point is chosen later.
|
|
1193
|
+
- **ECE (expected calibration error):** the average absolute gap between the
|
|
1194
|
+
model's confidence and its observed accuracy over 15 confidence bins. A
|
|
1195
|
+
model can be accurate and still overconfident; ECE quantifies that gap.
|
|
1196
|
+
|
|
1197
|
+
Overconfidence is fixed post hoc with `--fit_temperature`: after training, a
|
|
1198
|
+
single scalar $`T`$ is fitted on the validation set (minimising negative
|
|
1199
|
+
log-likelihood of `logits / T`), written to `temperature.json` next to the
|
|
1200
|
+
checkpoint, and applied at inference via `genml-kit-infer --temperature T`:
|
|
1201
|
+
|
|
1202
|
+
<!-- genml-cli-test
|
|
1203
|
+
name: fit-temperature
|
|
1204
|
+
override:
|
|
1205
|
+
- --dataset {tmp}/img
|
|
1206
|
+
- --label_column label
|
|
1207
|
+
extra: --epochs 1 --max_items 16 --image_size 32
|
|
1208
|
+
-->
|
|
1209
|
+
```bash
|
|
1210
|
+
genml-kit-train --model convvit --method classification --fit_temperature \
|
|
1211
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1212
|
+
--epochs 5 --batch_size 32
|
|
1213
|
+
genml-kit-infer --model convvit --checkpoint genml_kit_best.pt \
|
|
1214
|
+
--temperature 1.31 --top_k 3 path/to/lesion.jpg
|
|
1215
|
+
```
|
|
1216
|
+
|
|
826
1217
|
|
|
827
1218
|
When TTA is enabled, validation loss is computed from the original image only,
|
|
828
1219
|
while Top-1 accuracy, balanced accuracy, macro F1, weighted F1, and per-class
|
|
@@ -836,6 +1227,14 @@ the same sampling and preprocessing scheme.
|
|
|
836
1227
|
|
|
837
1228
|
### Basic Fine-Tuning
|
|
838
1229
|
|
|
1230
|
+
<!-- genml-cli-test
|
|
1231
|
+
name: fine-tune-basic
|
|
1232
|
+
network: true
|
|
1233
|
+
override:
|
|
1234
|
+
- --dataset {tmp}/img
|
|
1235
|
+
- --label_column label
|
|
1236
|
+
extra: --epochs 1
|
|
1237
|
+
-->
|
|
839
1238
|
```bash
|
|
840
1239
|
genml-kit-train --model google/vit-base-patch16-224 \
|
|
841
1240
|
--dataset my-org/my-labeled-images \
|
|
@@ -843,7 +1242,7 @@ genml-kit-train --model google/vit-base-patch16-224 \
|
|
|
843
1242
|
--epochs 5 \
|
|
844
1243
|
--batch_size 32 \
|
|
845
1244
|
--lr 3e-5 \
|
|
846
|
-
--image_size
|
|
1245
|
+
--image_size 224
|
|
847
1246
|
```
|
|
848
1247
|
|
|
849
1248
|
### With A Custom Classifier Head
|
|
@@ -851,11 +1250,20 @@ genml-kit-train --model google/vit-base-patch16-224 \
|
|
|
851
1250
|
Replace the default linear head with a custom MLP or attention-based
|
|
852
1251
|
classifier:
|
|
853
1252
|
|
|
1253
|
+
<!-- genml-cli-test
|
|
1254
|
+
name: fine-tune-custom-head
|
|
1255
|
+
network: true
|
|
1256
|
+
override:
|
|
1257
|
+
- --dataset {tmp}/img
|
|
1258
|
+
- --label_column label
|
|
1259
|
+
extra: --epochs 1
|
|
1260
|
+
-->
|
|
854
1261
|
```bash
|
|
855
1262
|
# Freeze backbone, train only the custom head
|
|
856
1263
|
genml-kit-train --model cls_model_wrapper:google/vit-base-patch16-224 \
|
|
857
1264
|
--dataset my-org/my-labeled-images \
|
|
858
1265
|
--label_column category \
|
|
1266
|
+
--image_size 224 \
|
|
859
1267
|
--model_arg classifier=mlp:hidden=512,dropout=0.3 \
|
|
860
1268
|
--freeze ".*\.(head|pool)"
|
|
861
1269
|
```
|
|
@@ -865,11 +1273,20 @@ genml-kit-train --model cls_model_wrapper:google/vit-base-patch16-224 \
|
|
|
865
1273
|
Freeze the entire backbone and train only small low-rank adapter matrices.
|
|
866
1274
|
Reduces trainable parameters by ~97% while often matching full fine-tuning:
|
|
867
1275
|
|
|
1276
|
+
<!-- genml-cli-test
|
|
1277
|
+
name: fine-tune-lora
|
|
1278
|
+
network: true
|
|
1279
|
+
extras: lora
|
|
1280
|
+
override:
|
|
1281
|
+
- --dataset {tmp}/img
|
|
1282
|
+
- --label_column label
|
|
1283
|
+
extra: --epochs 1
|
|
1284
|
+
-->
|
|
868
1285
|
```bash
|
|
869
1286
|
genml-kit-train \
|
|
870
1287
|
--model cls_model_wrapper:facebook/dinov2-with-registers-large \
|
|
1288
|
+
--model_arg classifier=mlp:hidden=512 \
|
|
871
1289
|
--lora --lora_r 16 --lora_alpha 32 \
|
|
872
|
-
--lora_target_modules "query,key,value" \
|
|
873
1290
|
--freeze "classifier\.(head|pool|encoder)" \
|
|
874
1291
|
--lr 3e-5 \
|
|
875
1292
|
--dataset my-org/my-labeled-images \
|
|
@@ -881,6 +1298,14 @@ genml-kit-train \
|
|
|
881
1298
|
|
|
882
1299
|
Load encoder weights from a pre-training run (SimMIM, I-JEPA, or SupCon):
|
|
883
1300
|
|
|
1301
|
+
<!-- genml-cli-test
|
|
1302
|
+
name: fine-tune-from-checkpoint
|
|
1303
|
+
override:
|
|
1304
|
+
- --dataset {tmp}/img
|
|
1305
|
+
- --label_column label
|
|
1306
|
+
- -.--source_checkpoint
|
|
1307
|
+
extra: --epochs 1
|
|
1308
|
+
-->
|
|
884
1309
|
```bash
|
|
885
1310
|
genml-kit-train --model convvit \
|
|
886
1311
|
--dataset my-org/my-labeled-images \
|
|
@@ -901,7 +1326,7 @@ schedule is (introduced by Howard & Ruder, [Universal Language Model Fine-tuning
|
|
|
901
1326
|
|
|
902
1327
|
$$
|
|
903
1328
|
\large
|
|
904
|
-
\text{lr}(\text{layer}) = \text{lr}_{\text{base}} \cdot d^{
|
|
1329
|
+
\text{lr}(\text{layer}) = \text{lr}_{\text{base}} \cdot d^{\\,L - \text{layer}}
|
|
905
1330
|
$$
|
|
906
1331
|
|
|
907
1332
|
where:
|
|
@@ -936,6 +1361,88 @@ the model settle into a minimum instead of bouncing around it. Late-training
|
|
|
936
1361
|
low learning rates are also what make the EMA teacher of DINO/BYOL
|
|
937
1362
|
effective: the teacher tracks a student that changes gently.
|
|
938
1363
|
|
|
1364
|
+
### Warmup
|
|
1365
|
+
|
|
1366
|
+
Transformer fine-tuning is unstable at the step-off: the randomly initialised
|
|
1367
|
+
head meets a pre-trained backbone with full force. The three warmup
|
|
1368
|
+
schedulers ramp the learning rate up linearly from zero over `--warmup_epochs`
|
|
1369
|
+
epochs (an integer or fraction) before handing over to the chosen decay
|
|
1370
|
+
shape:
|
|
1371
|
+
|
|
1372
|
+
<!-- genml-cli-test
|
|
1373
|
+
name: warmup-scheduler
|
|
1374
|
+
override:
|
|
1375
|
+
- --dataset {tmp}/img
|
|
1376
|
+
- --label_column label
|
|
1377
|
+
extra: --epochs 2 --max_items 16
|
|
1378
|
+
-->
|
|
1379
|
+
```bash
|
|
1380
|
+
genml-kit-train --model convvit --method classification \
|
|
1381
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1382
|
+
--scheduler warmup_cosine --warmup_epochs 2.5 \
|
|
1383
|
+
--epochs 20 --batch_size 32
|
|
1384
|
+
```
|
|
1385
|
+
|
|
1386
|
+
`--warmup_epochs 0` (the default) disables the ramp and every `warmup_*`
|
|
1387
|
+
name then behaves exactly like its plain shape. Available shapes:
|
|
1388
|
+
|
|
1389
|
+
| Name | Shape after warmup |
|
|
1390
|
+
|---|---|
|
|
1391
|
+
| `warmup_cosine` | half-cosine decay to `eta_min` (0), as above |
|
|
1392
|
+
| `warmup_linear` | linear decay to 0 over `--epochs` |
|
|
1393
|
+
| `warmup_rsqrt` | inverse square-root decay |
|
|
1394
|
+
|
|
1395
|
+
A `--sched_arg warmup_epochs=...` override wins over the generic flag, and
|
|
1396
|
+
`--sched_arg T_max=...` overrides the cosine total (the epoch of the first
|
|
1397
|
+
minimum). Plain torch names (`CosineAnnealingLR`, ...) ignore the warmup
|
|
1398
|
+
flag entirely, so existing invocations are unchanged.
|
|
1399
|
+
|
|
1400
|
+
### Weights EMA
|
|
1401
|
+
|
|
1402
|
+
Alongside the *loss-coupled* EMA teachers (DINO/BYOL/I-JEPA, and
|
|
1403
|
+
`--teacher_kind ema` in distillation), the trainer can maintain an
|
|
1404
|
+
*eval-time* shadow of the trained weights: after every optimizer step the
|
|
1405
|
+
shadow blends toward the live weights (`shadow <- decay * shadow +
|
|
1406
|
+
(1 - decay) * model`), damping the step-to-step noise of the optimisation
|
|
1407
|
+
trajectory. Validation and the best checkpoint then run on the shadow
|
|
1408
|
+
instead of the raw weights -- the classic timm / MMDetection weights-EMA,
|
|
1409
|
+
which typically buys a fraction of a point on noisy runs.
|
|
1410
|
+
|
|
1411
|
+
The feature is **opt-in per method**: only the method knows whether a
|
|
1412
|
+
shadow of "the model" is meaningful for its topology (DPO owns a frozen
|
|
1413
|
+
reference tower, distillation owns teacher towers -- neither is EMA
|
|
1414
|
+
material). Methods opt in by inheriting `WeightsEMAMixin`
|
|
1415
|
+
(`ClassificationMethod` does); methods without the mixin never see the
|
|
1416
|
+
feature, and setting `--ema_decay` on one logs a warning instead of
|
|
1417
|
+
silently doing nothing. The RL trainer shares the loop and simply never
|
|
1418
|
+
wires a shadow. The `--ema_*` flags are declared by the mixin's
|
|
1419
|
+
`add_args`, which adopting methods chain into their own `--help` group,
|
|
1420
|
+
so the flags render inside "classification method".
|
|
1421
|
+
|
|
1422
|
+
<!-- genml-cli-test
|
|
1423
|
+
name: weights-ema
|
|
1424
|
+
override:
|
|
1425
|
+
- --model convvit
|
|
1426
|
+
- --dataset {tmp}/img
|
|
1427
|
+
- --label_column label
|
|
1428
|
+
extra: --epochs 1 --max_items 16
|
|
1429
|
+
-->
|
|
1430
|
+
```bash
|
|
1431
|
+
genml-kit-train --model google/vit-base-patch16-224 --method classification \
|
|
1432
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1433
|
+
--ema_decay 0.9995 --ema_eval \
|
|
1434
|
+
--epochs 5 --batch_size 32
|
|
1435
|
+
```
|
|
1436
|
+
|
|
1437
|
+
`--ema_decay 0` (the default) keeps the run exactly as before. With
|
|
1438
|
+
`--ema_eval`, validation and `best.pt` carry the EMA weights while
|
|
1439
|
+
`latest.pt` keeps the raw trajectory plus the shadow in `ema_state`, so an
|
|
1440
|
+
interrupted run resumes exactly. `--ema_final_decay` ramps the effective
|
|
1441
|
+
decay linearly over the run (the DINO-style momentum schedule) and
|
|
1442
|
+
`--ema_warmup_steps` ramps it up from zero; when both are absent the decay
|
|
1443
|
+
is constant. `run.json` records the decay, so provenance of "which weights
|
|
1444
|
+
the best checkpoint holds" is unambiguous.
|
|
1445
|
+
|
|
939
1446
|
### Mixup, Label Smoothing, And Imbalance
|
|
940
1447
|
|
|
941
1448
|
Mixup forms a virtual example from two training examples.
|
|
@@ -989,13 +1496,13 @@ The loss for one example is
|
|
|
989
1496
|
|
|
990
1497
|
$$
|
|
991
1498
|
\large
|
|
992
|
-
\mathcal{L} = \left(1 - p_t\right)^{\gamma} \sum_{c=1}^{C} w_c
|
|
993
|
-
\left(-\log p_c\right), \qquad p_t = \sum_{c=1}^{C} q_c
|
|
1499
|
+
\mathcal{L} = \left(1 - p_t\right)^{\gamma} \sum_{c=1}^{C} w_c\\, q_c
|
|
1500
|
+
\left(-\log p_c\right), \qquad p_t = \sum_{c=1}^{C} q_c\\, p_c
|
|
994
1501
|
$$
|
|
995
1502
|
|
|
996
1503
|
Two details matter for the reader combining these flags:
|
|
997
1504
|
|
|
998
|
-
- **Unified $`p_t
|
|
1505
|
+
- **Unified $`p_t`$.** The modulating factor uses the *expected* probability
|
|
999
1506
|
under the target distribution, $`p_t = \sum_c q_c p_c`$, rather than the
|
|
1000
1507
|
probability of a single class. For a one-hot target $`q`$ this reduces
|
|
1001
1508
|
exactly to the basic form above ($`p_t`$ becomes the correct-class
|
|
@@ -1011,6 +1518,91 @@ Two details matter for the reader combining these flags:
|
|
|
1011
1518
|
Use these tools deliberately; combining every regularizer is not
|
|
1012
1519
|
automatically better.
|
|
1013
1520
|
|
|
1521
|
+
### Data Augmentation
|
|
1522
|
+
|
|
1523
|
+
By default the `classification` method augments the training set with the
|
|
1524
|
+
chain built by `build_transforms` in
|
|
1525
|
+
`genml_kit/training/train_compat.py`:
|
|
1526
|
+
|
|
1527
|
+
```python
|
|
1528
|
+
import torch
|
|
1529
|
+
from torchvision.transforms import v2
|
|
1530
|
+
|
|
1531
|
+
train_augmentations = v2.Compose([
|
|
1532
|
+
# Randomly crop 20-100% of the image, then resize to image_size.
|
|
1533
|
+
v2.RandomResizedCrop(size=(image_size, image_size),
|
|
1534
|
+
scale=(0.2, 1.0),
|
|
1535
|
+
antialias=True),
|
|
1536
|
+
# Mirror left/right, applied half the time.
|
|
1537
|
+
v2.RandomHorizontalFlip(p=0.5),
|
|
1538
|
+
# Mirror top/bottom, applied half the time.
|
|
1539
|
+
v2.RandomVerticalFlip(p=0.5),
|
|
1540
|
+
# Rotate by an arbitrary angle.
|
|
1541
|
+
v2.RandomRotation(degrees=360),
|
|
1542
|
+
# Photometric jitter on brightness, contrast, saturation and hue.
|
|
1543
|
+
v2.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.15, hue=0.05),
|
|
1544
|
+
# Local elastic warping.
|
|
1545
|
+
v2.ElasticTransform(alpha=50.0, sigma=5.0),
|
|
1546
|
+
# Fixed tail, always appended automatically:
|
|
1547
|
+
# PIL image -> tensor.
|
|
1548
|
+
v2.ToImage(),
|
|
1549
|
+
# uint8 -> float in [0, 1].
|
|
1550
|
+
v2.ToDtype(torch.float32, scale=True),
|
|
1551
|
+
# Model-specific normalization.
|
|
1552
|
+
v2.Normalize(mean=processor.image_mean, std=processor.image_std),
|
|
1553
|
+
])
|
|
1554
|
+
```
|
|
1555
|
+
|
|
1556
|
+
The validation chain is never augmented: it is always `Resize` →
|
|
1557
|
+
`CenterCrop` → `Normalize`.
|
|
1558
|
+
|
|
1559
|
+
#### Custom Training Augmentation
|
|
1560
|
+
|
|
1561
|
+
To override the training chain, pass `--train_augmentation_script <path>.py`
|
|
1562
|
+
(a local path or an HTTP/HTTPS URL). The script is imported lazily, so a
|
|
1563
|
+
plain run never loads it. It must define
|
|
1564
|
+
|
|
1565
|
+
```python
|
|
1566
|
+
def create_train_transform(image_size: int):
|
|
1567
|
+
...
|
|
1568
|
+
```
|
|
1569
|
+
|
|
1570
|
+
for example:
|
|
1571
|
+
|
|
1572
|
+
```python
|
|
1573
|
+
# my_aug.py
|
|
1574
|
+
from torchvision.transforms import v2
|
|
1575
|
+
|
|
1576
|
+
def create_train_transform(image_size):
|
|
1577
|
+
return [
|
|
1578
|
+
v2.RandomResizedCrop(image_size, scale=(0.4, 1.0), antialias=True),
|
|
1579
|
+
v2.RandomHorizontalFlip(p=0.5),
|
|
1580
|
+
v2.ColorJitter(brightness=0.3, contrast=0.3),
|
|
1581
|
+
]
|
|
1582
|
+
```
|
|
1583
|
+
|
|
1584
|
+
The contract is:
|
|
1585
|
+
|
|
1586
|
+
- The function is called once as `create_train_transform(image_size)` and
|
|
1587
|
+
**must return a `list`** of transforms. Any other return type raises a
|
|
1588
|
+
`TypeError` (`create_train_transform() must return a list of transforms,
|
|
1589
|
+
got <type>`).
|
|
1590
|
+
- The returned list is composed as `v2.Compose(user_transforms + tail)`,
|
|
1591
|
+
where the fixed tail above (`ToImage` → `ToDtype` → `Normalize`) is
|
|
1592
|
+
**always appended automatically**. Your transforms therefore run on the
|
|
1593
|
+
**raw PIL image**, and you must **not** add `ToImage`, `ToDtype`, or
|
|
1594
|
+
`Normalize` yourself.
|
|
1595
|
+
- This differs from the segmentation method's `create_seg_transform`, which
|
|
1596
|
+
owns the whole image + mask chain *including* normalization. For
|
|
1597
|
+
classification, normalization stays under the processor's control.
|
|
1598
|
+
- Only the **train** chain is replaced; validation is unaffected. Use
|
|
1599
|
+
`--tta` for test-time augmentation (see
|
|
1600
|
+
[XGBoost And Test-Time Augmentation](#xgboost-and-test-time-augmentation)).
|
|
1601
|
+
|
|
1602
|
+
Modules placed next to the script (or importable via `import` from the same
|
|
1603
|
+
directory) resolve by plain name, so a custom transform can live in a
|
|
1604
|
+
sibling file.
|
|
1605
|
+
|
|
1014
1606
|
### Hyperparameter Guidance
|
|
1015
1607
|
|
|
1016
1608
|
| Scenario | `--lr` | `--epochs` | `--batch_size` | Notes |
|
|
@@ -1047,8 +1639,17 @@ Two levels of determinism:
|
|
|
1047
1639
|
Passing an explicit `--seed` therefore buys full determinism at a
|
|
1048
1640
|
throughput cost (typically 10–20%); omit it for maximum speed:
|
|
1049
1641
|
|
|
1642
|
+
<!-- genml-cli-test
|
|
1643
|
+
name: reproducibility-seed
|
|
1644
|
+
override:
|
|
1645
|
+
- --dataset {tmp}/img
|
|
1646
|
+
- --label_column label
|
|
1647
|
+
extra: --epochs 1 --batch_size 4 --max_items 16
|
|
1648
|
+
-->
|
|
1050
1649
|
```bash
|
|
1051
|
-
genml-kit-train --model convvit --seed 42
|
|
1650
|
+
genml-kit-train --model convvit --seed 42 \
|
|
1651
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1652
|
+
--epochs 5 --batch_size 32
|
|
1052
1653
|
```
|
|
1053
1654
|
|
|
1054
1655
|
The train/val split and the XGBoost stage follow the user's seed: with
|
|
@@ -1068,6 +1669,67 @@ Checkpointing is atomic: `_latest.pt` is written to a temporary file
|
|
|
1068
1669
|
and renamed into place, so an interrupted run never leaves a truncated
|
|
1069
1670
|
resume point.
|
|
1070
1671
|
|
|
1672
|
+
### Run Manifest
|
|
1673
|
+
|
|
1674
|
+
Every training run writes `<log_dir>/run.json` — a reproducibility
|
|
1675
|
+
snapshot of *how* the run was launched: the fully resolved argparse
|
|
1676
|
+
namespace (defaults filled in), the git SHA of the checkout, the
|
|
1677
|
+
`torch` / `transformers` versions, hostname, device, resolved seeds, and
|
|
1678
|
+
timestamp. Together with `metrics.jsonl` this makes any result traceable
|
|
1679
|
+
back to the exact code and invocation that produced it:
|
|
1680
|
+
|
|
1681
|
+
<!-- genml-cli-test-exclude: illustrative ellipsis, no dataset; not a runnable command -->
|
|
1682
|
+
```bash
|
|
1683
|
+
genml-kit-train --model convvit --method classification ...
|
|
1684
|
+
cat <checkpoint_dir>/logs/run.json
|
|
1685
|
+
```
|
|
1686
|
+
|
|
1687
|
+
The same facts are emitted to the metric sinks as `manifest/*` and
|
|
1688
|
+
`args/<name>` run-facts, so they land in TensorBoard and any configured
|
|
1689
|
+
sink alongside the metrics. `--dry_run` logs the manifest as JSON and
|
|
1690
|
+
exits before the dataset is touched — a fast way to inspect what an
|
|
1691
|
+
invocation would actually do:
|
|
1692
|
+
|
|
1693
|
+
<!-- genml-cli-test
|
|
1694
|
+
name: run-manifest-dry-run
|
|
1695
|
+
-->
|
|
1696
|
+
```bash
|
|
1697
|
+
genml-kit-train --model convvit --method classification --dry_run
|
|
1698
|
+
```
|
|
1699
|
+
|
|
1700
|
+
### Metric Sinks
|
|
1701
|
+
|
|
1702
|
+
`--metrics_sinks` selects where training metrics go, as a comma-separated
|
|
1703
|
+
fan-out: `tb` (TensorBoard event files), `jsonl` (one JSON record per line in
|
|
1704
|
+
`<log_dir>/metrics.jsonl` — the file the search subsystem reads), and `wandb`
|
|
1705
|
+
(a live Weights & Biases run). The default `tb,jsonl` is unchanged; sinks
|
|
1706
|
+
are independent, so any combination works:
|
|
1707
|
+
|
|
1708
|
+
<!-- genml-cli-test
|
|
1709
|
+
name: metric-sinks
|
|
1710
|
+
extras: wandb
|
|
1711
|
+
override:
|
|
1712
|
+
- --dataset {tmp}/img
|
|
1713
|
+
- --label_column label
|
|
1714
|
+
extra: --epochs 1 --max_items 16 --wandb_mode disabled
|
|
1715
|
+
-->
|
|
1716
|
+
```bash
|
|
1717
|
+
genml-kit-train --model convvit --method classification \
|
|
1718
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1719
|
+
--metrics_sinks tb,jsonl,wandb --wandb_project my-team \
|
|
1720
|
+
--epochs 5 --batch_size 32
|
|
1721
|
+
```
|
|
1722
|
+
|
|
1723
|
+
The `wandb` sink requires the optional extra (`pip install
|
|
1724
|
+
"genml_kit[wandb]"`); the run name defaults to `--run_name` when given, the
|
|
1725
|
+
run finishes on exit, and the fully resolved args are uploaded as the run
|
|
1726
|
+
config. `--wandb_mode` selects how the run is recorded: `online` (default,
|
|
1727
|
+
syncs to the W&B service), `offline` (a local `.wandb` transaction log,
|
|
1728
|
+
synced later with `wandb sync` — no network or credentials needed), or
|
|
1729
|
+
`disabled` (wandb's no-op stub). New sinks implement the same tiny
|
|
1730
|
+
duck-typed protocol (`scalar` / `image` / `video` / `run_fact` / `close`) in
|
|
1731
|
+
`genml_kit/training/metrics_sinks.py`.
|
|
1732
|
+
|
|
1071
1733
|
### Fine-Tuning CLI Reference
|
|
1072
1734
|
|
|
1073
1735
|
| Argument | Default | Description |
|
|
@@ -1103,7 +1765,12 @@ resume point.
|
|
|
1103
1765
|
| `--num_workers` | `2` | DataLoader worker processes. |
|
|
1104
1766
|
| `--log_level` | `INFO` | Logging level. |
|
|
1105
1767
|
| `--log_targets` | `STDERR` | Comma-separated log destinations. `STDERR` logs to standard error; any other entry is a log file path (appended). Example: `STDERR,/tmp/train.log` logs to both. |
|
|
1106
|
-
| `--log_dir` | `None` | TensorBoard log directory (default: `<checkpoint_dir>/logs`). Requires the `tensorboard` package
|
|
1768
|
+
| `--log_dir` | `None` | TensorBoard log directory (default: `<checkpoint_dir>/logs`). Requires the `tensorboard` package. |
|
|
1769
|
+
| `--metrics_sinks` | `tb,jsonl` | Comma-separated metric sinks: `tb` (TensorBoard), `jsonl` (`<log_dir>/metrics.jsonl`), `wandb` (Weights & Biases, requires `pip install "genml_kit[wandb]"`), `none`. |
|
|
1770
|
+
| `--wandb_project` | `None` | W&B project name when `wandb` is in `--metrics_sinks` (default: `genml-kit`). |
|
|
1771
|
+
| `--wandb_mode` | `online` | W&B run mode: `online` (sync to the service), `offline` (local `.wandb` log, sync later with `wandb sync`), `disabled` (stub, no network/files). |
|
|
1772
|
+
| `--run_name` | `None` | W&B run name when `wandb` is in `--metrics_sinks`. |
|
|
1773
|
+
| `--fit_temperature` | `False` | After training, fit a temperature-scaling T on the validation set of the best checkpoint and write `temperature.json` next to it. Classification methods only. |
|
|
1107
1774
|
| `--cache_dir` | `None` | HuggingFace cache directory. |
|
|
1108
1775
|
| `--remote_checkpoint` | `None` | Remote URI for checkpoint sync (`gs://BUCKET/PREFIX`, `r2://BUCKET/PREFIX`, or `s3://BUCKET/PREFIX`). |
|
|
1109
1776
|
| `--source_checkpoint` | `None` | Path to source checkpoint to absorb parameters from. |
|
|
@@ -1113,11 +1780,12 @@ resume point.
|
|
|
1113
1780
|
| `--lora_r` | `8` | LoRA rank. |
|
|
1114
1781
|
| `--lora_alpha` | `16` | LoRA alpha (scaling = `alpha / r`). |
|
|
1115
1782
|
| `--lora_dropout` | `0.0` | Dropout on LoRA layers. |
|
|
1116
|
-
| `--lora_target_modules` | `None` | Comma-separated module names for LoRA (e.g. `"
|
|
1783
|
+
| `--lora_target_modules` | `None` | Comma-separated module names for LoRA (e.g. `"q_proj,k_proj,v_proj"`). When omitted, targets are resolved from the model (see LoRA section). |
|
|
1117
1784
|
| `--optimizer` | `AdamW` | `torch.optim` optimizer class name or `.py` script path. |
|
|
1118
1785
|
| `--opt_arg` | `{}` | Extra optimizer kwargs (repeatable). |
|
|
1119
|
-
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path
|
|
1786
|
+
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path, or a built-in warmup shape: `warmup_cosine`, `warmup_linear`, `warmup_rsqrt`. |
|
|
1120
1787
|
| `--sched_arg` | `{}` | Extra scheduler kwargs (repeatable). |
|
|
1788
|
+
| `--warmup_epochs` | `0` | Linear LR ramp length in epochs (integer or fraction) for the `warmup_*` schedulers; 0 disables the ramp. See [Warmup](#warmup). |
|
|
1121
1789
|
| `--state_save` | `opt,sched,amp` | States to save: `opt`, `sched`, `amp`, `none`. |
|
|
1122
1790
|
| `--state_load` | `opt,sched,amp` | States to restore on resume. |
|
|
1123
1791
|
| `--xgboost_model` | `None` | Output path for XGBoost model (trains after PyTorch). |
|
|
@@ -1131,6 +1799,15 @@ resume point.
|
|
|
1131
1799
|
Training automatically resumes from an existing `_latest.pt` or `_best.pt`
|
|
1132
1800
|
checkpoint if one exists at the `--checkpoint` path.
|
|
1133
1801
|
|
|
1802
|
+
Alongside the epoch-end and exit saves, `--save_every` writes `_latest.pt`
|
|
1803
|
+
every N optimizer steps (grad-accum flush boundary) so a hard kill between
|
|
1804
|
+
epochs does not lose the whole epoch. When such a checkpoint carries the epoch
|
|
1805
|
+
in progress, resume **continues** it rather than replaying it: supervised runs
|
|
1806
|
+
fast-forward to the recorded step, and PPO skips re-collecting the rollout and
|
|
1807
|
+
re-enters the optimize sweep (a resume taken mid-collection re-runs the epoch,
|
|
1808
|
+
since the environment is not persisted). Off-policy RL (DQN/SAC) restarts the
|
|
1809
|
+
epoch: the replay buffer is not checkpointed.
|
|
1810
|
+
|
|
1134
1811
|
### Remote Checkpoint Sync (GCS / R2 / S3)
|
|
1135
1812
|
|
|
1136
1813
|
`--remote_checkpoint` uploads each saved checkpoint to cloud storage.
|
|
@@ -1206,6 +1883,15 @@ Only $`A`$, $`B`$ (and the classifier head) receive gradients.
|
|
|
1206
1883
|
|
|
1207
1884
|
LoRA can be combined with a custom classifier:
|
|
1208
1885
|
|
|
1886
|
+
<!-- genml-cli-test
|
|
1887
|
+
name: lora-custom-classifier
|
|
1888
|
+
network: true
|
|
1889
|
+
extras: lora
|
|
1890
|
+
override:
|
|
1891
|
+
- --dataset {tmp}/img
|
|
1892
|
+
- --label_column label
|
|
1893
|
+
extra: --epochs 1
|
|
1894
|
+
-->
|
|
1209
1895
|
```bash
|
|
1210
1896
|
genml-kit-train \
|
|
1211
1897
|
--model cls_model_wrapper:facebook/dinov2-with-registers-large \
|
|
@@ -1213,13 +1899,23 @@ genml-kit-train \
|
|
|
1213
1899
|
--lora --lora_r 16 --lora_alpha 32 \
|
|
1214
1900
|
--freeze 'classifier\.(head|pool|encoder)' \
|
|
1215
1901
|
--lr_group 'backbone.*=1e-5' 'classifier.*=3e-4' \
|
|
1216
|
-
|
|
1902
|
+
--dataset my-org/my-labeled-images \
|
|
1903
|
+
--label_column category \
|
|
1904
|
+
--epochs 20
|
|
1217
1905
|
```
|
|
1218
1906
|
|
|
1219
1907
|
### Cross-Dataset Resume
|
|
1220
1908
|
|
|
1221
1909
|
Switch from one dataset to another while keeping backbone weights:
|
|
1222
1910
|
|
|
1911
|
+
<!-- genml-cli-test
|
|
1912
|
+
name: cross-dataset-resume
|
|
1913
|
+
network: true
|
|
1914
|
+
override:
|
|
1915
|
+
- --dataset {tmp}/img
|
|
1916
|
+
- --label_column label
|
|
1917
|
+
extra: --epochs 1
|
|
1918
|
+
-->
|
|
1223
1919
|
```bash
|
|
1224
1920
|
genml-kit-train --model facebook/convnextv2-base-22k-224 \
|
|
1225
1921
|
--dataset my-org/other-labeled-images \
|
|
@@ -1245,14 +1941,24 @@ limits the maximum batch size even on a 24 GB GPU. Gradient checkpointing
|
|
|
1245
1941
|
trades compute for memory by discarding intermediate activations during the
|
|
1246
1942
|
forward pass and recomputing them during the backward pass.
|
|
1247
1943
|
|
|
1944
|
+
<!-- genml-cli-test
|
|
1945
|
+
name: gradient-checkpointing
|
|
1946
|
+
network: true
|
|
1947
|
+
override:
|
|
1948
|
+
- --dataset {tmp}/img
|
|
1949
|
+
- --label_column label
|
|
1950
|
+
extra: --epochs 1
|
|
1951
|
+
-->
|
|
1248
1952
|
```bash
|
|
1249
1953
|
genml-kit-train \
|
|
1250
1954
|
--model timm:eva02_base_patch14_448.mim_in22k_ft_in22k_in1k \
|
|
1955
|
+
--dataset my-org/my-labeled-images \
|
|
1956
|
+
--label_column category \
|
|
1251
1957
|
--batch_size 32 \
|
|
1252
1958
|
--grad_accum_steps 2 \
|
|
1253
1959
|
--grad_checkpoint \
|
|
1254
1960
|
--amp_dtype bfloat16 \
|
|
1255
|
-
|
|
1961
|
+
--epochs 20
|
|
1256
1962
|
```
|
|
1257
1963
|
|
|
1258
1964
|
**What it does:** Each transformer block stores only its input and output.
|
|
@@ -1279,10 +1985,27 @@ HuggingFace, ConvViT, and UVito — via native APIs or per-block
|
|
|
1279
1985
|
|
|
1280
1986
|
Run inference on individual images:
|
|
1281
1987
|
|
|
1988
|
+
<!-- genml-cli-test
|
|
1989
|
+
name: inference-guide
|
|
1990
|
+
network: true
|
|
1991
|
+
override:
|
|
1992
|
+
- --checkpoint {tmp}/chk_infer/model_best.pt
|
|
1993
|
+
- --model convvit --image_size 32
|
|
1994
|
+
extra: --top_k 3 --image_size 32
|
|
1995
|
+
-->
|
|
1282
1996
|
```bash
|
|
1283
1997
|
genml-kit-infer --model facebook/convnextv2-base-22k-224 \
|
|
1284
1998
|
--checkpoint genml_kit_best.pt \
|
|
1285
1999
|
path/to/image.jpg path/to/other_image.png
|
|
2000
|
+
|
|
2001
|
+
# Structured output and an XGBoost head on the backbone features (the .json
|
|
2002
|
+
# model is the one trained by `train_xgboost_on_backbone`, see
|
|
2003
|
+
# tests/test_xgb_pipeline.py).
|
|
2004
|
+
genml-kit-infer --model facebook/convnextv2-base-22k-224 \
|
|
2005
|
+
--checkpoint genml_kit_best.pt \
|
|
2006
|
+
--xgboost_model path/to/xgboost_model.json \
|
|
2007
|
+
--output results.json \
|
|
2008
|
+
path/to/image.jpg
|
|
1286
2009
|
```
|
|
1287
2010
|
|
|
1288
2011
|
Output is JSON with per-class probabilities:
|
|
@@ -1303,7 +2026,7 @@ The neural classifier makes decisions through its head. The XGBoost option
|
|
|
1303
2026
|
takes the encoder representation $`h = f_\theta(x)`$ instead and fits an
|
|
1304
2027
|
ensemble of decision trees to those vectors.
|
|
1305
2028
|
Reference: Chen & Guestrin, [XGBoost: A Scalable Tree Boosting System](https://arxiv.org/abs/1603.02754), KDD 2016.
|
|
1306
|
-
A tree partitions feature space with rules such as $`h_{37}
|
|
2029
|
+
A tree partitions feature space with rules such as $`h_{37} \lt t`$ (comparing
|
|
1307
2030
|
the 37th feature of $`h`$ against a learned threshold $`t`$); boosting adds
|
|
1308
2031
|
trees sequentially so each new tree focuses on errors left by previous trees.
|
|
1309
2032
|
This can work well when the dataset is small
|
|
@@ -1334,6 +2057,7 @@ does not erase the cue that distinguishes the classes.
|
|
|
1334
2057
|
| `--model` | (required) | HuggingFace model name or custom model. |
|
|
1335
2058
|
| `--checkpoint` | (required) | Path to state dict or wrapped checkpoint. |
|
|
1336
2059
|
| `--top_k` | `None` | Show top-K predictions; omit for all classes. |
|
|
2060
|
+
| `--temperature` | `1.0` | Divide logits by this before the softmax (1.0 = unchanged). Use the value written to `temperature.json` by `--fit_temperature`. |
|
|
1337
2061
|
| `--output` | `None` | Write JSON results to file. |
|
|
1338
2062
|
| `--device` | `None` | Force PyTorch device (`cuda`, `cpu`). Auto-detected if omitted. |
|
|
1339
2063
|
| `--cache_dir` | `None` | HuggingFace cache directory. |
|
|
@@ -1369,12 +2093,26 @@ training run. `--source_checkpoint` imports weights from another run before
|
|
|
1369
2093
|
the new optimizer is created. When moving a SupCon encoder into classification,
|
|
1370
2094
|
use:
|
|
1371
2095
|
|
|
2096
|
+
<!-- genml-cli-test
|
|
2097
|
+
name: param-rename
|
|
2098
|
+
network: true
|
|
2099
|
+
override:
|
|
2100
|
+
- --dataset {tmp}/img
|
|
2101
|
+
- --label_column label
|
|
2102
|
+
- --checkpoint {tmp}/ckpt_param_rename
|
|
2103
|
+
- -.--source_checkpoint
|
|
2104
|
+
extra: --epochs 1
|
|
2105
|
+
-->
|
|
1372
2106
|
```bash
|
|
1373
2107
|
genml-kit-train \
|
|
2108
|
+
--model google/vit-base-patch16-224 \
|
|
2109
|
+
--image_size 224 \
|
|
1374
2110
|
--checkpoint /content/eva02_finetune \
|
|
1375
2111
|
--source_checkpoint /content/eva02_supcon_latest.pt \
|
|
1376
2112
|
--param_rename 'encoder\\.model\\.(.*);model.$1' \
|
|
1377
|
-
|
|
2113
|
+
--dataset my-org/my-labeled-images \
|
|
2114
|
+
--label_column category \
|
|
2115
|
+
--epochs 20
|
|
1378
2116
|
```
|
|
1379
2117
|
|
|
1380
2118
|
The rename is needed because the pre-training wrapper stores backbone keys
|
|
@@ -1414,7 +2152,10 @@ The periodic `GPU:` line reports memory in MiB and SM utilization as a
|
|
|
1414
2152
|
percentage sampled at log time (`mem=allocated/totalMiB (pct) res=…MiB
|
|
1415
2153
|
util=…%`). `mem` counts live tensors, `res` is the caching allocator's
|
|
1416
2154
|
pool, and `util` is one instant — not an average — so an isolated 0 is
|
|
1417
|
-
normal.
|
|
2155
|
+
normal. The `util=` field needs the optional NVML bindings
|
|
2156
|
+
(`pip install "genml_kit[gpu]"`); when they are absent, `util=` is simply
|
|
2157
|
+
omitted from the `GPU:` line — the run is never affected.
|
|
2158
|
+
A *persistent* pattern of `util=0` at every log line, with
|
|
1418
2159
|
`mem`/`res` byte-identical between samples, means the GPU finished the
|
|
1419
2160
|
last batch and is idle waiting for the next one: the run is **input
|
|
1420
2161
|
bound**, not compute bound. The throughput line (`img/s=`) is the ground
|
|
@@ -1533,9 +2274,20 @@ def build_loss(alpha=1.0, **kwargs):
|
|
|
1533
2274
|
return MyLoss(alpha=alpha)
|
|
1534
2275
|
```
|
|
1535
2276
|
|
|
2277
|
+
<!-- genml-cli-test
|
|
2278
|
+
name: custom-loss
|
|
2279
|
+
network: true
|
|
2280
|
+
override:
|
|
2281
|
+
- --dataset {tmp}/img
|
|
2282
|
+
- --label_column label
|
|
2283
|
+
extra: --epochs 1
|
|
2284
|
+
-->
|
|
1536
2285
|
```bash
|
|
1537
2286
|
genml-kit-train --model google/vit-base-patch16-224 \
|
|
1538
|
-
--
|
|
2287
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
2288
|
+
--image_size 224 \
|
|
2289
|
+
--loss ./my_loss.py --loss_args '{"alpha": 2.0}' \
|
|
2290
|
+
--epochs 20
|
|
1539
2291
|
```
|
|
1540
2292
|
|
|
1541
2293
|
**RL losses** work the same way but return a *bundle* (dict of
|
|
@@ -1549,8 +2301,14 @@ def build_loss(**kwargs):
|
|
|
1549
2301
|
return {"entropy": lambda ent: ent.mean() * 2.0} # override only entropy
|
|
1550
2302
|
```
|
|
1551
2303
|
|
|
2304
|
+
<!-- genml-cli-test
|
|
2305
|
+
name: rl-custom-loss
|
|
2306
|
+
extras: rl
|
|
2307
|
+
extra: --epochs 1
|
|
2308
|
+
-->
|
|
1552
2309
|
```bash
|
|
1553
|
-
genml-kit-train --method ppo --
|
|
2310
|
+
genml-kit-train --pipeline rl --method ppo --env_id CartPole-v1 \
|
|
2311
|
+
--loss ./my_ppo_entropy.py
|
|
1554
2312
|
```
|
|
1555
2313
|
|
|
1556
2314
|
The built-in bundles are merged over (`ppo`: `policy`/`value`/`entropy`;
|
|
@@ -1607,8 +2365,18 @@ def create_model(num_labels, image_size=224, id2label=None, label2id=None,
|
|
|
1607
2365
|
return Tiny(num_labels=num_labels, hidden=kwargs.get("hidden", 128))
|
|
1608
2366
|
```
|
|
1609
2367
|
|
|
2368
|
+
<!-- genml-cli-test
|
|
2369
|
+
name: custom-model
|
|
2370
|
+
network: true
|
|
2371
|
+
override:
|
|
2372
|
+
- --dataset {tmp}/img
|
|
2373
|
+
- --label_column label
|
|
2374
|
+
extra: --epochs 1
|
|
2375
|
+
-->
|
|
1610
2376
|
```bash
|
|
1611
|
-
genml-kit-train --model ./my_model.py --loss focal
|
|
2377
|
+
genml-kit-train --model ./my_model.py --loss focal \
|
|
2378
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
2379
|
+
--epochs 20
|
|
1612
2380
|
```
|
|
1613
2381
|
|
|
1614
2382
|
The script is **fully responsible for weight loading**: when
|
|
@@ -1632,11 +2400,13 @@ class Classifier(nn.Module):
|
|
|
1632
2400
|
super().__init__()
|
|
1633
2401
|
self.head = nn.Linear(hidden_size, num_labels)
|
|
1634
2402
|
|
|
1635
|
-
def forward(self, hidden_states):
|
|
2403
|
+
def forward(self, hidden_states):
|
|
2404
|
+
# (B, N, D) tensor.
|
|
1636
2405
|
features = hidden_states[:, 0] # CLS token
|
|
1637
2406
|
return self.head(features)
|
|
1638
2407
|
|
|
1639
|
-
def extract_features(self, hidden_states):
|
|
2408
|
+
def extract_features(self, hidden_states):
|
|
2409
|
+
# (B, N, D) → (B, D).
|
|
1640
2410
|
return hidden_states[:, 0]
|
|
1641
2411
|
```
|
|
1642
2412
|
|
|
@@ -1650,8 +2420,8 @@ The head is selected through `--model_arg`, not a dedicated flag:
|
|
|
1650
2420
|
```
|
|
1651
2421
|
|
|
1652
2422
|
The `classifier` value is an inline spec: the head name (a registered
|
|
1653
|
-
built-in
|
|
1654
|
-
`
|
|
2423
|
+
built-in \u2014 `mlp`, `cls_attention` \u2014 or a path to a `.py` file defining a
|
|
2424
|
+
`create_classifier(num_labels, hidden_size, **kwargs)` function), optionally followed by `:key=value,...` kwargs. Values
|
|
1655
2425
|
are parsed as int/float/bool, and bracketed lists such as
|
|
1656
2426
|
`cls_slice=(0, 1)` work. A head is required: `cls_model_wrapper` fails fast
|
|
1657
2427
|
with a clear message when `classifier` is missing. Models that embed their
|
|
@@ -1662,6 +2432,855 @@ head before loading the checkpoint weights.
|
|
|
1662
2432
|
|
|
1663
2433
|
---
|
|
1664
2434
|
|
|
2435
|
+
## Semantic Segmentation
|
|
2436
|
+
|
|
2437
|
+
Segmentation predicts a class for **every pixel**: an image `(B, 3, H, W)`
|
|
2438
|
+
goes in and per-pixel logits `(B, num_classes, H, W)` come out. `genml_kit`
|
|
2439
|
+
models this as `encoder + decoder`: the encoder is any hierarchical timm
|
|
2440
|
+
backbone (ResNet / ConvNeXt / Swin / MiT) emitting a feature pyramid, and
|
|
2441
|
+
the decoder fuses that pyramid back up to full resolution (with skip
|
|
2442
|
+
connections for `unet`/`fpn`).
|
|
2443
|
+
|
|
2444
|
+
### Quick start
|
|
2445
|
+
|
|
2446
|
+
<!-- genml-cli-test
|
|
2447
|
+
name: segmentation-quick-start
|
|
2448
|
+
network: true
|
|
2449
|
+
override:
|
|
2450
|
+
- --dataset segfolder/{tmp}/seg
|
|
2451
|
+
extra: --epochs 1
|
|
2452
|
+
-->
|
|
2453
|
+
```bash
|
|
2454
|
+
genml-kit-train \
|
|
2455
|
+
--pipeline segmentation --method segmentation \
|
|
2456
|
+
--dataset segfolder//data/voc2012 \
|
|
2457
|
+
--model seg_wrapper:timm/resnet34 \
|
|
2458
|
+
--model_arg decoder=unet:depth=4 \
|
|
2459
|
+
--loss dice_ce \
|
|
2460
|
+
--image_size 512 --batch_size 8 --epochs 40
|
|
2461
|
+
```
|
|
2462
|
+
|
|
2463
|
+
The layout expected by `segfolder/` is a paired tree:
|
|
2464
|
+
|
|
2465
|
+
```
|
|
2466
|
+
/data/voc2012/images/2007_000032.jpg
|
|
2467
|
+
/data/voc2012/masks/2007_000032.png
|
|
2468
|
+
```
|
|
2469
|
+
|
|
2470
|
+
or, for a corpus with named splits, `train/`, `val/` and `test/`
|
|
2471
|
+
sub-directories each containing `images/` + `masks/`. Image and mask are
|
|
2472
|
+
matched on the file stem (case-insensitive) and a mismatch fails fast.
|
|
2473
|
+
|
|
2474
|
+
HuggingFace datasets that carry a mask feature are used with `--mask_column`:
|
|
2475
|
+
|
|
2476
|
+
<!-- genml-cli-test
|
|
2477
|
+
name: segmentation-mask-column
|
|
2478
|
+
network: true
|
|
2479
|
+
override:
|
|
2480
|
+
- --dataset {tmp}/hf_seg
|
|
2481
|
+
extra: --epochs 1
|
|
2482
|
+
-->
|
|
2483
|
+
```bash
|
|
2484
|
+
genml-kit-train --pipeline segmentation --method segmentation \
|
|
2485
|
+
--dataset scene_parse_150 --mask_column annotation \
|
|
2486
|
+
--model seg_wrapper:timm/convnext_tiny --model_arg decoder=fpn
|
|
2487
|
+
```
|
|
2488
|
+
|
|
2489
|
+
### Class-id compaction
|
|
2490
|
+
|
|
2491
|
+
Masks are expected to carry integer class ids, but the ids do **not** have
|
|
2492
|
+
to be dense. The pipeline scans every mask (train **and** val, so a class
|
|
2493
|
+
that only appears in validation is still mapped) and compacts whatever ids
|
|
2494
|
+
it finds into a dense `[0, C)` range:
|
|
2495
|
+
|
|
2496
|
+
| Source id | Compact id |
|
|
2497
|
+
|---|---|
|
|
2498
|
+
| 12 | 0 |
|
|
2499
|
+
| 17 | 1 |
|
|
2500
|
+
| 255 (= ignore) | dropped |
|
|
2501
|
+
|
|
2502
|
+
- **segfolder path:** the mapping is rebuilt on every run (a full scan)
|
|
2503
|
+
and remapped through `apply_mask_map` on every access.
|
|
2504
|
+
- **HuggingFace path:** the `{raw → compact}` map is cached to disk under
|
|
2505
|
+
`<cache_dir>/seg_class_maps/segclassmap__<hash>.json`, or
|
|
2506
|
+
`~/.cache/genml_kit/seg_class_maps/` when no `--cache_dir` is given
|
|
2507
|
+
(falling back to a temp dir if that is unwritable). The cache is keyed
|
|
2508
|
+
on the dataset spec **and** a content hash of the masks themselves, so
|
|
2509
|
+
editing a mask invalidates the entry even if the spec string is
|
|
2510
|
+
unchanged.
|
|
2511
|
+
|
|
2512
|
+
The class count reported by training (and the default for
|
|
2513
|
+
`--seg_num_classes`) is the **compact** count, i.e. the number of
|
|
2514
|
+
distinct ids found after dropping `--seg_ignore_index` (`len(mask_map)`).
|
|
2515
|
+
(The count falls back to `max_label + 1` over a sample of loader masks
|
|
2516
|
+
only when the pipeline did not build a mask map, e.g. a method wired
|
|
2517
|
+
without the segmentation pipeline.)
|
|
2518
|
+
|
|
2519
|
+
### Data augmentation
|
|
2520
|
+
|
|
2521
|
+
The default transforms for the `segmentation` method (in
|
|
2522
|
+
`genml_kit/datasets/transforms.py`) are split into a geometric stage and
|
|
2523
|
+
a photometric stage:
|
|
2524
|
+
|
|
2525
|
+
- **Geometric stage** — applied to the image **and** the mask together,
|
|
2526
|
+
so they always stay pixel-aligned:
|
|
2527
|
+
- `v2.RandomResizedCrop(image_size, scale=(0.5, 2.0),
|
|
2528
|
+
ratio=(0.75, 4/3))`
|
|
2529
|
+
- `v2.RandomHorizontalFlip(p=0.5)`
|
|
2530
|
+
- **Photometric stage** — applied to the **image only** (masks contain
|
|
2531
|
+
class ids, never color):
|
|
2532
|
+
- `v2.RandomApply([v2.ColorJitter(brightness=0.4, contrast=0.6,
|
|
2533
|
+
saturation=0.8, hue=0.1)], p=0.8)`
|
|
2534
|
+
|
|
2535
|
+
Validation is deterministic: `v2.Resize(image_size, BICUBIC)` then
|
|
2536
|
+
`CenterCrop(image_size)`.
|
|
2537
|
+
|
|
2538
|
+
The geometric/photometric split is a correctness contract: geometric ops
|
|
2539
|
+
go through `tv_tensors.Mask`, so torchvision issues one random draw per
|
|
2540
|
+
sample and the mask gets the same crop/translation as the image; color
|
|
2541
|
+
ops never see the mask.
|
|
2542
|
+
|
|
2543
|
+
To take full control, pass `--seg_augmentation_script <path>.py` pointing
|
|
2544
|
+
at a module that defines
|
|
2545
|
+
|
|
2546
|
+
```python
|
|
2547
|
+
def create_seg_transform(image_size: int, *, is_train: bool):
|
|
2548
|
+
...
|
|
2549
|
+
```
|
|
2550
|
+
|
|
2551
|
+
When set, the script **owns the whole paired transform** (image + mask),
|
|
2552
|
+
including normalization (the model processor's `image_mean`/`image_std`
|
|
2553
|
+
are *not* applied automatically in this mode), and is loaded lazily so a
|
|
2554
|
+
plain run never imports it.
|
|
2555
|
+
|
|
2556
|
+
### Encoder + decoder
|
|
2557
|
+
|
|
2558
|
+
`--model seg_wrapper:<backbone>` selects the encoder; the decoder is chosen
|
|
2559
|
+
with `--model_arg decoder=<spec>`. Built-in decoders:
|
|
2560
|
+
|
|
2561
|
+
| Decoder | Skip connections | Notes |
|
|
2562
|
+
|---|---|---|
|
|
2563
|
+
| `unet` | yes (all levels) | Classic U-Net top-down fusion; `depth=N` selects the deepest N encoder levels |
|
|
2564
|
+
| `fpn` | yes (additive) | Feature-pyramid fusion into one high-resolution map |
|
|
2565
|
+
| `linear` | no | 1×1 conv + bilinear upsample; good baseline, ViT-friendly |
|
|
2566
|
+
|
|
2567
|
+
The decoder spec accepts `:key=value` kwargs (`decoder=unet:depth=4,decoder_channels=256`)
|
|
2568
|
+
and, like classifier heads, a path to a `.py` file defining
|
|
2569
|
+
`create_decoder(in_channels, num_classes, **kwargs)` where `in_channels` is
|
|
2570
|
+
the probed per-level encoder channel list.
|
|
2571
|
+
|
|
2572
|
+
### Training from scratch vs. fine-tuning
|
|
2573
|
+
|
|
2574
|
+
The decoder is always randomly initialized and trained from scratch; the
|
|
2575
|
+
encoder is what you initialise. Options, in increasing order of GPU cost:
|
|
2576
|
+
|
|
2577
|
+
- **Pretrained encoder (recommended):** `seg_wrapper` loads timm ImageNet
|
|
2578
|
+
weights by default (`pretrained=False` disables this).
|
|
2579
|
+
- **Reuse a self-supervised checkpoint:** pre-train with SimMIM / I-JEPA /
|
|
2580
|
+
DINO, then load the encoder with `--source_checkpoint <ckpt>` (see
|
|
2581
|
+
[From A Pre-Trained Checkpoint](#from-a-pre-trained-checkpoint)).
|
|
2582
|
+
- **CNN encoder with a frozen backbone:** `--freeze 'decoder.*'` keeps the
|
|
2583
|
+
decoder trainable and freezes everything else.
|
|
2584
|
+
- **Transformer encoder with LoRA:** `--lora` (PEFT discovers the
|
|
2585
|
+
`q_proj`/`k_proj`/`v_proj` projections automatically). CNN encoders are
|
|
2586
|
+
almost all `Conv2d`, so LoRA does not apply to them — use `--freeze`.
|
|
2587
|
+
|
|
2588
|
+
### Loss and metric
|
|
2589
|
+
|
|
2590
|
+
The default criterion is `dice_ce` (Dice + cross-entropy), selectable via
|
|
2591
|
+
`--loss dice_ce`; the validation metric is **mean IoU** (`miou`, maximize,
|
|
2592
|
+
also reporting `mdice` and `pixel_acc`). The class count is inferred from
|
|
2593
|
+
the training masks (ignoring `--seg_ignore_index`, default 255) unless
|
|
2594
|
+
`--seg_num_classes` overrides it. Pixels equal to `--seg_ignore_index` are
|
|
2595
|
+
excluded from both the loss and the metric.
|
|
2596
|
+
|
|
2597
|
+
### Validation overlays
|
|
2598
|
+
|
|
2599
|
+
Set `--vis_every N` to have the trainer call the method's
|
|
2600
|
+
`log_validation` every N epochs, which logs a `seg/overlay` TensorBoard
|
|
2601
|
+
grid of **image / ground-truth mask / predicted mask** rows (8 samples by
|
|
2602
|
+
default) into the run's log dir (default `<checkpoint_dir>/logs`, see
|
|
2603
|
+
`--log_dir`). The masks are rendered with a fixed 20-hue palette; the
|
|
2604
|
+
ignore class (`--seg_ignore_index`, or 255) renders black. Overlays use
|
|
2605
|
+
the *current* model state, so they double as a quick training-progress
|
|
2606
|
+
check (e.g. whether the decoder is learning the class layout before the
|
|
2607
|
+
mIoU catches up).
|
|
2608
|
+
|
|
2609
|
+
Note that some backbones (e.g. plain ViT, Swin) require their native input
|
|
2610
|
+
resolution (224) and reject other sizes.
|
|
2611
|
+
|
|
2612
|
+
### Inference
|
|
2613
|
+
|
|
2614
|
+
`genml-kit-infer` is classification-only today: it predicts a top-K label
|
|
2615
|
+
for each image and does **not** emit per-pixel masks. For segmentation,
|
|
2616
|
+
use the validation loop (mIoU + overlays above) or the checkpoint's
|
|
2617
|
+
weights directly.
|
|
2618
|
+
|
|
2619
|
+
---
|
|
2620
|
+
|
|
2621
|
+
## Object Detection
|
|
2622
|
+
|
|
2623
|
+
Detection predicts a set of **bounding boxes**, each with a class label, for
|
|
2624
|
+
an image. `genml_kit` ships an anchor-free FCOS head over the same encoder
|
|
2625
|
+
feature pyramid that segmentation uses, wired as the `detection` pipeline and
|
|
2626
|
+
the `detection` method.
|
|
2627
|
+
|
|
2628
|
+
### How It Works
|
|
2629
|
+
|
|
2630
|
+
The model is `det_wrapper:<backbone>` = **encoder + FCOS head**:
|
|
2631
|
+
|
|
2632
|
+
* The encoder is any timm backbone loaded with `features_only=True`, exactly
|
|
2633
|
+
as for segmentation, so it yields a feature pyramid (finest level first).
|
|
2634
|
+
* The head runs a small convolutional tower on **every** pyramid level and
|
|
2635
|
+
predicts, per spatial location, a `num_classes`-way class score, four
|
|
2636
|
+
positive box distances `(l, t, r, b)` in input pixels, and a centerness
|
|
2637
|
+
score. The distances are `exp()`-activated and scaled by the level's
|
|
2638
|
+
stride, so a single location grid spans all scales.
|
|
2639
|
+
|
|
2640
|
+
Training uses the FCOS objective (`--loss fcos` by default), a sum of three
|
|
2641
|
+
terms:
|
|
2642
|
+
|
|
2643
|
+
* **Focal loss** over all locations (the class imbalance between the few
|
|
2644
|
+
positives and the many background locations is large).
|
|
2645
|
+
* **GIoU loss** over the positive locations only.
|
|
2646
|
+
* **Centerness BCE**, which teaches each positive location how central it is
|
|
2647
|
+
inside its target box, so NMS can rank central boxes higher.
|
|
2648
|
+
|
|
2649
|
+
The **assignment** is purely geometric and needs no Hungarian matcher: a
|
|
2650
|
+
location is a *candidate* for a box when it lies inside the box, and a
|
|
2651
|
+
*positive* when it additionally lies inside the box's centre region
|
|
2652
|
+
(`--loss_args center_sampling_radius=1.5`). When several boxes cover a
|
|
2653
|
+
location, the **smallest-area** box wins, so small objects are not starved of
|
|
2654
|
+
positives by large ones.
|
|
2655
|
+
|
|
2656
|
+
At validation time the dense predictions are decoded to `xyxy` boxes,
|
|
2657
|
+
per-class NMS is applied (`--det_nms_thresh`), and the result feeds a
|
|
2658
|
+
COCO-style **mAP** meter (`--det_max_detections`, default 100). The reported
|
|
2659
|
+
metric is `map` (COCO `mAP@[.5:.95]`, maximize), so the best checkpoint is
|
|
2660
|
+
stored under `best_map`; `map_50` and `map_75` are logged alongside it.
|
|
2661
|
+
|
|
2662
|
+
### Dataset Format
|
|
2663
|
+
|
|
2664
|
+
The `detection` pipeline reads **COCO-JSON** corpora via the `detfolder/`
|
|
2665
|
+
spec:
|
|
2666
|
+
|
|
2667
|
+
```
|
|
2668
|
+
<root>/images/<file_name>
|
|
2669
|
+
<root>/annotations.json
|
|
2670
|
+
```
|
|
2671
|
+
|
|
2672
|
+
or, with named splits:
|
|
2673
|
+
|
|
2674
|
+
```
|
|
2675
|
+
<root>/train/images/<file_name>
|
|
2676
|
+
<root>/train/annotations.json
|
|
2677
|
+
<root>/val/images/<file_name>
|
|
2678
|
+
<root>/val/annotations.json
|
|
2679
|
+
```
|
|
2680
|
+
|
|
2681
|
+
`annotations.json` is the standard COCO object-detection schema (`images`,
|
|
2682
|
+
`annotations` with `[x, y, width, height]` `bbox`, and `categories`). The
|
|
2683
|
+
loader converts `bbox` to `xyxy` and **compacts** the (1-based, sparse) COCO
|
|
2684
|
+
category ids into a dense `[0, C)` label space, so the head width equals the
|
|
2685
|
+
number of categories actually present. Images with no annotations are legal
|
|
2686
|
+
(pure-background samples) and still contribute negatives to the loss. The
|
|
2687
|
+
annotation file name and image sub-directory are configurable via
|
|
2688
|
+
`--det_annotation_file` and `--det_image_subdir`.
|
|
2689
|
+
|
|
2690
|
+
To generate a throwaway corpus for a CPU smoke run without downloading COCO,
|
|
2691
|
+
use the bundled generator (`scripts/make_tiny_coco.py`), then point the
|
|
2692
|
+
pipeline at it:
|
|
2693
|
+
|
|
2694
|
+
<!-- genml-cli-test
|
|
2695
|
+
name: detection-dataset-format
|
|
2696
|
+
network: true
|
|
2697
|
+
override:
|
|
2698
|
+
- --dataset detfolder/{tmp}/det
|
|
2699
|
+
extra: --epochs 1
|
|
2700
|
+
-->
|
|
2701
|
+
```bash
|
|
2702
|
+
genml-kit-train --pipeline detection --method detection \
|
|
2703
|
+
--dataset detfolder/tiny_coco --model det_wrapper:timm/resnet18 \
|
|
2704
|
+
--image_size 640 --batch_size 4 --epochs 20
|
|
2705
|
+
```
|
|
2706
|
+
|
|
2707
|
+
### Example
|
|
2708
|
+
|
|
2709
|
+
<!-- genml-cli-test
|
|
2710
|
+
name: detection-example
|
|
2711
|
+
network: true
|
|
2712
|
+
override:
|
|
2713
|
+
- --dataset detfolder/{tmp}/det
|
|
2714
|
+
extra: --epochs 1
|
|
2715
|
+
-->
|
|
2716
|
+
```bash
|
|
2717
|
+
genml-kit-train \
|
|
2718
|
+
--pipeline detection --method detection \
|
|
2719
|
+
--dataset detfolder//data/coco_tiny \
|
|
2720
|
+
--model det_wrapper:timm/resnet50 \
|
|
2721
|
+
--model_arg head=fcos_head:num_convs=2 \
|
|
2722
|
+
--image_size 640 --batch_size 4 \
|
|
2723
|
+
--epochs 20 --lr 1e-4
|
|
2724
|
+
```
|
|
2725
|
+
|
|
2726
|
+
Override the inferred class count with `--det_num_classes`; tune the decode
|
|
2727
|
+
with `--det_score_thresh` and `--det_nms_thresh`; override the criterion with
|
|
2728
|
+
`--loss` / `--loss_args` (e.g. `--loss_args center_sampling_radius=1.0`).
|
|
2729
|
+
|
|
2730
|
+
### Transforms
|
|
2731
|
+
|
|
2732
|
+
Detection does **not** reuse the segmentation transform stack: `torchvision`
|
|
2733
|
+
remaps masks automatically through geometric ops but leaves boxes untouched,
|
|
2734
|
+
so every resize / flip must move the coordinates itself. The default
|
|
2735
|
+
pipeline is therefore a hand-written **letterbox** (aspect-preserving resize
|
|
2736
|
+
then pad, which keeps boxes rectangular) followed by a box-aware horizontal
|
|
2737
|
+
flip (train only) and a `(C, H, W)` float conversion. Boxes stay in absolute
|
|
2738
|
+
`xyxy` pixels throughout.
|
|
2739
|
+
|
|
2740
|
+
---
|
|
2741
|
+
|
|
2742
|
+
## Visual Odometry
|
|
2743
|
+
|
|
2744
|
+
Visual odometry (VO) estimates the 2-D **similarity transform** — rotation,
|
|
2745
|
+
uniform scale, and translation — that maps one image onto another taken a
|
|
2746
|
+
fraction of a second later. `genml_kit` ships a learned VO front-end plus a
|
|
2747
|
+
synthetic pair generator, wired as the `vo_pair` pipeline and the `vo_pair`
|
|
2748
|
+
method.
|
|
2749
|
+
|
|
2750
|
+
The full mathematics (coordinate frames, the rotation matrix, Umeyama's
|
|
2751
|
+
closed-form solution, the photometric objective, the `mce` metric, and a
|
|
2752
|
+
failure-mode appendix) lives in [`vo/README.md`](vo/README.md) — a
|
|
2753
|
+
from-the-ground-up tutorial. This section is the practical *usage* guide.
|
|
2754
|
+
|
|
2755
|
+
### Quick start
|
|
2756
|
+
|
|
2757
|
+
<!-- genml-cli-test
|
|
2758
|
+
name: vo-quick-start
|
|
2759
|
+
extra: --epochs 1 --vo_length 16 --vo_val_length 8
|
|
2760
|
+
-->
|
|
2761
|
+
```bash
|
|
2762
|
+
# Supervised VO on synthetic pairs (default npu-small profile)
|
|
2763
|
+
genml-kit-train \
|
|
2764
|
+
--pipeline vo_pair --method vo_pair \
|
|
2765
|
+
--model vo/npu-small \
|
|
2766
|
+
--image_size 64 --batch_size 32 --epochs 10
|
|
2767
|
+
|
|
2768
|
+
# Photometric self-supervised stage (adds the photometric residual term)
|
|
2769
|
+
genml-kit-train \
|
|
2770
|
+
--pipeline vo_pair --method vo_pair \
|
|
2771
|
+
--vo_stage photometric --epochs 10
|
|
2772
|
+
```
|
|
2773
|
+
|
|
2774
|
+
### Model profiles
|
|
2775
|
+
|
|
2776
|
+
The VO net is selected through the model registry as `vo/<profile>`:
|
|
2777
|
+
|
|
2778
|
+
| `--model` | Profile | Encoder widths |
|
|
2779
|
+
|---|---|---|
|
|
2780
|
+
| `vo/npu-small` | default | `(16, 32, 64, 128)` |
|
|
2781
|
+
| `vo/edge-mid` | mid-size | see `genml_kit/models/vo/vo_similar.py` |
|
|
2782
|
+
| `vo/station` | large | see `genml_kit/models/vo/vo_similar.py` |
|
|
2783
|
+
|
|
2784
|
+
### Pipeline flags (`--pipeline vo_pair`)
|
|
2785
|
+
|
|
2786
|
+
| Flag | Default | Meaning |
|
|
2787
|
+
|---|---|---|
|
|
2788
|
+
| `--vo_length` | `20000` | number of synthetic training pairs |
|
|
2789
|
+
| `--vo_val_length` | `2000` | number of synthetic validation pairs |
|
|
2790
|
+
| `--pitch_deg` | `45.0` | camera pitch for the ground-plane render |
|
|
2791
|
+
| `--terrain` | `field` | terrain profile name |
|
|
2792
|
+
| `--seeded_pairs` | `0` | seed the pair generator for reproducible pairs |
|
|
2793
|
+
|
|
2794
|
+
### Method flags (`--method vo_pair`)
|
|
2795
|
+
|
|
2796
|
+
| Flag | Default | Meaning |
|
|
2797
|
+
|---|---|---|
|
|
2798
|
+
| `--vo_stage` | `supervised` | `supervised` (mce + similarity deltas) or `photometric` (adds the photometric residual) |
|
|
2799
|
+
| `--vo_profile` | `npu-small` | VO model profile (also selectable via `--model vo/<profile>`) |
|
|
2800
|
+
| `--vo_loss_cfg` | — | comma-separated loss weights `w_log_s,w_theta,w_conf,w_photo` |
|
|
2801
|
+
|
|
2802
|
+
The reported metric is `mce` (mean corner error, in pixels), minimized — the
|
|
2803
|
+
best checkpoint is stored under `best_mce`. Set `--image_size 64` for the
|
|
2804
|
+
synthetic pairs, whose default resolution is 64×64.
|
|
2805
|
+
|
|
2806
|
+
---
|
|
2807
|
+
|
|
2808
|
+
## Knowledge Distillation
|
|
2809
|
+
|
|
2810
|
+
Knowledge distillation trains a small **student** to mimic a large **teacher**.
|
|
2811
|
+
`genml_kit` implements the three families of distillation (response / logit
|
|
2812
|
+
matching, feature / FitNets, attention, and relation / RKD) and both teacher
|
|
2813
|
+
sources: a separately-loaded frozen model (`--teacher_kind external`) and an
|
|
2814
|
+
exponential-moving-average copy of the student itself
|
|
2815
|
+
(`--teacher_kind ema`, i.e. self-distillation). It is wired as the
|
|
2816
|
+
`distillation` method over the standard `images` pipeline.
|
|
2817
|
+
|
|
2818
|
+
The full mathematics (temperature and the soft-target insight, the KL↔CE
|
|
2819
|
+
collapse and the `T²` scaling, attention-transfer normalisation, RKD, and the
|
|
2820
|
+
EMA-teacher variance argument) lives in
|
|
2821
|
+
[`knowledge_distillation/README.md`](knowledge_distillation/README.md) — a
|
|
2822
|
+
from-the-ground-up tutorial. This section is the practical *usage* guide.
|
|
2823
|
+
|
|
2824
|
+
### Quick start
|
|
2825
|
+
|
|
2826
|
+
<!-- genml-cli-test
|
|
2827
|
+
name: distillation-quick-start
|
|
2828
|
+
network: true
|
|
2829
|
+
override:
|
|
2830
|
+
- --dataset {tmp}/img
|
|
2831
|
+
- -.--image_column
|
|
2832
|
+
- --teacher_model google/vit-base-patch16-224
|
|
2833
|
+
- --teacher_checkpoint {tmp}/teacher/model_best.pt
|
|
2834
|
+
extra: --epochs 1
|
|
2835
|
+
-->
|
|
2836
|
+
```bash
|
|
2837
|
+
# Self-distillation: teacher is an EMA of the student (no external model)
|
|
2838
|
+
genml-kit-train \
|
|
2839
|
+
--pipeline images --method distillation \
|
|
2840
|
+
--model google/vit-base-patch16-224 --image_size 224 \
|
|
2841
|
+
--dataset uoft-cs/cifar10 --image_column img --label_column label \
|
|
2842
|
+
--teacher_kind ema --kd_temperature 4.0 --kd_alpha 1.0 \
|
|
2843
|
+
--epochs 10
|
|
2844
|
+
|
|
2845
|
+
# External teacher: frozen checkpoint hands over its soft targets
|
|
2846
|
+
genml-kit-train \
|
|
2847
|
+
--pipeline images --method distillation \
|
|
2848
|
+
--model google/vit-base-patch16-224 --image_size 224 \
|
|
2849
|
+
--dataset uoft-cs/cifar10 --image_column img --label_column label \
|
|
2850
|
+
--teacher_kind external \
|
|
2851
|
+
--teacher_model google/vit-large-patch16-224 \
|
|
2852
|
+
--teacher_checkpoint /path/to/teacher.pt \
|
|
2853
|
+
--kd_temperature 4.0 --kd_alpha 1.0 --epochs 10
|
|
2854
|
+
```
|
|
2855
|
+
|
|
2856
|
+
### Method flags (`--method distillation`)
|
|
2857
|
+
|
|
2858
|
+
| Flag | Default | Meaning |
|
|
2859
|
+
|---|---|---|
|
|
2860
|
+
| `--teacher_kind` | `external` | `external` loads a frozen teacher; `ema` self-distils from an EMA of the student |
|
|
2861
|
+
| `--teacher_model` | — | teacher model spec (required for `--teacher_kind external`) |
|
|
2862
|
+
| `--teacher_checkpoint` | — | checkpoint holding the external teacher's weights |
|
|
2863
|
+
| `--kd_temperature` | `4.0` | distillation temperature $`T`$ |
|
|
2864
|
+
| `--kd_alpha` | `1.0` | weight of the response (logit) KD term (`0` disables it) |
|
|
2865
|
+
| `--kd_feature_alpha` | `0.0` | weight of the FitNets feature KD term (`0` disables it) |
|
|
2866
|
+
| `--kd_attention` / `--no-kd_attention` | off | use attention transfer on the matched layers instead of raw feature MSE |
|
|
2867
|
+
| `--kd_relation_alpha` | `0.0` | weight of the relational (RKD) KD term (`0` disables it) |
|
|
2868
|
+
| `--kd_layer_strategy` | `uniform` | how to pair teacher/student layers: `by_index`, `uniform`, `by_name` |
|
|
2869
|
+
| `--kd_layer_pattern` | — | regex selecting the feature layers to pair (feature KD) |
|
|
2870
|
+
| `--kd_proj_hidden` | `512` | hidden width of the FitNets projection |
|
|
2871
|
+
| `--kd_ema_momentum` | `0.996` | initial EMA momentum for `--teacher_kind ema` |
|
|
2872
|
+
| `--kd_ema_final_momentum` | `1.0` | final EMA momentum for `--teacher_kind ema` |
|
|
2873
|
+
|
|
2874
|
+
Enable one family at a time and watch the validation metric after each addition
|
|
2875
|
+
(see Appendix B of the distillation tutorial for the pitfalls). A labeled
|
|
2876
|
+
`--dataset` is required: the ground-truth cross-entropy branch anchors the
|
|
2877
|
+
student on examples where the teacher is confidently wrong.
|
|
2878
|
+
|
|
2879
|
+
---
|
|
2880
|
+
|
|
2881
|
+
## Large Language Models (causal LM)
|
|
2882
|
+
|
|
2883
|
+
genml_kit can fine-tune decoder-only language models through the same
|
|
2884
|
+
branchless driver: a new `text` pipeline (token collation) pairs with a new
|
|
2885
|
+
`causal_lm` method (shifted next-token cross-entropy, perplexity metric).
|
|
2886
|
+
|
|
2887
|
+
### Quick start
|
|
2888
|
+
|
|
2889
|
+
<!-- genml-cli-test
|
|
2890
|
+
name: causal-lm-quick-start
|
|
2891
|
+
network: true
|
|
2892
|
+
extras: lora
|
|
2893
|
+
override:
|
|
2894
|
+
- --dataset {tmp}/text
|
|
2895
|
+
extra: --epochs 1
|
|
2896
|
+
-->
|
|
2897
|
+
```bash
|
|
2898
|
+
# Fine-tune a tiny GPT-2 on a local text corpus
|
|
2899
|
+
genml-kit-train --pipeline text \
|
|
2900
|
+
--method causal_lm \
|
|
2901
|
+
--model sshleifer/tiny-gpt2 \
|
|
2902
|
+
--dataset ./my_text_corpus \
|
|
2903
|
+
--text_column text \
|
|
2904
|
+
--max_seq_len 256 \
|
|
2905
|
+
--lora
|
|
2906
|
+
```
|
|
2907
|
+
|
|
2908
|
+
The `--dataset` value may be a HuggingFace dataset name, or a directory saved
|
|
2909
|
+
with `datasets.Dataset.save_to_disk` (containing `train` / `validation`
|
|
2910
|
+
splits). With `--val_split` a held-out fraction of the training split is used
|
|
2911
|
+
for validation instead.
|
|
2912
|
+
|
|
2913
|
+
### Pipeline flags (`--pipeline text`)
|
|
2914
|
+
|
|
2915
|
+
| Flag | Default | Meaning |
|
|
2916
|
+
|---|---|---|
|
|
2917
|
+
| `--text_column` | — | column holding a raw LM sequence (plain-LM mode) |
|
|
2918
|
+
| `--instruction_column` | — | prompt column (instruction mode, with `--response_column`) |
|
|
2919
|
+
| `--response_column` | — | response column (instruction mode) |
|
|
2920
|
+
| `--max_seq_len` | `512` | token sequence length (truncation / packing window) |
|
|
2921
|
+
| `--sequence_mode` | `document` | sequence recipe: `document` (no packing, document-exact boundary loss, prompt framed as a document prefix) or `open` (packed concatenation, boundary loss kept, prompt framed as a whole document) |
|
|
2922
|
+
| `--mask_prompt` | off | mask prompt tokens out of the loss (instruction tuning) |
|
|
2923
|
+
| `--packing [wrapped\|bfd]` | off | pack sequences into fixed `max_seq_len` windows (`wrapped` = concatenate-and-slice, `bfd` = best-fit-decreasing) |
|
|
2924
|
+
| `--length_bucket` | off | batch items of similar length so padded batches waste little compute (mutually exclusive with `--packing`) |
|
|
2925
|
+
| `--pad_to_multiple_of` | — | round padded length up to a multiple (e.g. `8`) |
|
|
2926
|
+
| `--max_items` | — | cap the number of loaded items (smoke tests) |
|
|
2927
|
+
|
|
2928
|
+
#### Packing and document boundaries
|
|
2929
|
+
|
|
2930
|
+
Padding every item up to the batch's longest wastes compute on short documents;
|
|
2931
|
+
packing removes that waste but, naively, lets tokens of one document attend to a
|
|
2932
|
+
*different* document sharing the same window. Three modes trade padding for
|
|
2933
|
+
compute, and packing stays document-aware without binding the data side to any
|
|
2934
|
+
model:
|
|
2935
|
+
|
|
2936
|
+
| Mode | How it batches | Document isolation |
|
|
2937
|
+
|---|---|---|
|
|
2938
|
+
| *(default)* | pad every row to the batch's longest item | plain, document-exact 2-D mask |
|
|
2939
|
+
| `--packing wrapped` | concatenate the tokenized stream (EOS-separated) and slice into `max_seq_len` windows | via `segment_ids` (model-dependent, below) |
|
|
2940
|
+
| `--packing bfd` | best-fit-decreasing: longest document first into the open bin with the least remaining space | via `segment_ids` (model-dependent, below) |
|
|
2941
|
+
| `--length_bucket` | no packing: sort by a cheap character-length proxy, group into buckets, shuffle within and across them, emit similar-length batches | plain, document-exact 2-D mask |
|
|
2942
|
+
|
|
2943
|
+
`wrapped` is the historical behaviour and may split a document across two
|
|
2944
|
+
windows; `bfd` yields fewer, fuller windows but reorders documents, so batch
|
|
2945
|
+
order is **not** reproducible with respect to `--seed` (it warns once).
|
|
2946
|
+
`--length_bucket` is mutually exclusive with `--packing`.
|
|
2947
|
+
|
|
2948
|
+
**Batching and memory.** On every path except `wrapped`, `--batch_size` counts
|
|
2949
|
+
*documents* (each becomes one model row, or one packed row for `bfd`). Under
|
|
2950
|
+
`wrapped` the collator concatenates the batch and slices it into `max_seq_len`
|
|
2951
|
+
windows, so the row count is `ceil(total_tokens / max_seq_len)` -- which the
|
|
2952
|
+
document count does **not** bound. A batch of long documents can therefore fan
|
|
2953
|
+
out into far more rows than `--batch_size` and exhaust VRAM (one long document
|
|
2954
|
+
alone becomes `len(doc) / max_seq_len` rows), and lowering `--batch_size` or
|
|
2955
|
+
`--grad_accum_steps` barely helps because the cost tracks *tokens*, not
|
|
2956
|
+
documents.
|
|
2957
|
+
|
|
2958
|
+
To make `wrapped` safe, `--batch_size` is honored as a **token budget**:
|
|
2959
|
+
`PackingBatchSampler` groups documents until a batch fills `--batch_size *
|
|
2960
|
+
max_seq_len` tokens (the constant-length packing recipe Megatron-LM and
|
|
2961
|
+
HuggingFace packed datasets use), and the collator hard-caps every emitted batch
|
|
2962
|
+
at `--batch_size` windows. A document longer than one window is still streamed
|
|
2963
|
+
in -- its tail lands in a later window of the same batch rather than being
|
|
2964
|
+
truncated away -- and only the overflow past the whole batch budget (a single
|
|
2965
|
+
document longer than `--batch_size` windows) is dropped, with a one-time
|
|
2966
|
+
warning. If you see that warning, raise `--batch_size` or lower `--max_items`.
|
|
2967
|
+
|
|
2968
|
+
`--sequence_mode` compiles the windowing flags and the two coupled policies
|
|
2969
|
+
below into one validated object (`genml_kit/training/sequence_mode.py`), so an
|
|
2970
|
+
incoherent pairing cannot be expressed. The two recipes are:
|
|
2971
|
+
|
|
2972
|
+
- `document` (default) -- unpacked, **boundary loss masked**, prompts framed as
|
|
2973
|
+
a document *prefix* (`[BOS] ...`, no trailing `EOS`). The validation
|
|
2974
|
+
perplexity is document-exact and the first sampled token is a trained
|
|
2975
|
+
"continue this document" state.
|
|
2976
|
+
- `open` -- packed (`wrapped`), **boundary loss kept**, prompts framed as a whole
|
|
2977
|
+
*document* (`[BOS] ... [EOS]`). This is the GPT-2/GPT-3 pre-training recipe:
|
|
2978
|
+
the token following a document's `EOS` keeps a trained next-token
|
|
2979
|
+
distribution, so "what follows a document" is a learned state rather than an
|
|
2980
|
+
untrained one. `open` requires packing, and `--length_bucket` is rejected
|
|
2981
|
+
with it.
|
|
2982
|
+
|
|
2983
|
+
Explicit `--packing` / `--length_bucket` override the recipe's default
|
|
2984
|
+
windowing; the boundary-loss and framing policies still come from the recipe.
|
|
2985
|
+
A kept boundary loss is meaningful only under packing: unpacked, the tokens
|
|
2986
|
+
after a document's `EOS` are padding, so keeping the boundary label would train
|
|
2987
|
+
`EOS -> PAD` -- which is why `open` (and the collator) reject it.
|
|
2988
|
+
|
|
2989
|
+
Document isolation (independent of the boundary loss) works as follows:
|
|
2990
|
+
|
|
2991
|
+
- Under `--packing`, the collator emits two extra tensors per batch, requiring
|
|
2992
|
+
no model knowledge: `segment_ids` (the contiguous run -- i.e. source document
|
|
2993
|
+
-- of each token, `-1` for padding) and `position_ids` (restarts at `0` at
|
|
2994
|
+
every run start, so a window is self-describing).
|
|
2995
|
+
- The `causal_lm` method, the only place that holds both the batch and the
|
|
2996
|
+
built model, translates those tensors to whatever the model declares via its
|
|
2997
|
+
`PACKED_ATTENTION` class attribute:
|
|
2998
|
+
|
|
2999
|
+
| `PACKED_ATTENTION` | What the model receives | Document isolation |
|
|
3000
|
+
|---|---|---|
|
|
3001
|
+
| `"none"` (default) | nothing extra | `--packing` is refused with a clear error |
|
|
3002
|
+
| `"block_diagonal"` | a 4-D `[B, 1, L, L]` additive mask | exact (no flash kernel) |
|
|
3003
|
+
| `"varlen"` | `FlashAttentionKwargs` (`cu_seq_lens_q/k`, `max_length_q/k`) on a flattened `[1, total]` batch | exact (flash when `flash-attn` is installed) |
|
|
3004
|
+
| `"segment_ids"` | the raw `[B, L]` `segment_ids` runs plus the 2-D padding mask and `position_ids` | model-defined (reset state per run) |
|
|
3005
|
+
| `"causal"` | only the 2-D padding mask and `position_ids` (no boundary tensors) | none -- documents are concatenated and may attend across boundaries (the standard LLM pre-training recipe) |
|
|
3006
|
+
|
|
3007
|
+
HuggingFace decoders are inferred automatically: `--attn_implementation
|
|
3008
|
+
flash_attention_2` implies `varlen`, otherwise `block_diagonal`. Custom
|
|
3009
|
+
models (subclasses of `GenerativeLM`) opt in explicitly. The `segment_ids`
|
|
3010
|
+
mode is for models that compute attention *without* materializing the `L x L`
|
|
3011
|
+
score matrix (linear / kernelized, streaming or recurrent attention): a dense
|
|
3012
|
+
block-diagonal mask would defeat the cost such a mechanism is built to avoid,
|
|
3013
|
+
so the model instead receives the raw per-token document runs and resets its
|
|
3014
|
+
state at each boundary. The `causal` mode opts *out* of document isolation
|
|
3015
|
+
altogether: the model receives only the ordinary 2-D key-padding mask and
|
|
3016
|
+
`position_ids`, and a plain causal mask attends across document boundaries.
|
|
3017
|
+
This is the recipe used by GPT-3, PaLM, LLaMA, Falcon and Mistral
|
|
3018
|
+
(concatenate with an EOD marker, no cross-document blocking), so it is the
|
|
3019
|
+
right choice when reproducing that setup. The collator still emits
|
|
3020
|
+
`segment_ids`; a `causal` model simply ignores them. Use `causal` for models
|
|
3021
|
+
that compute attention without an `L x L` score matrix *and* want to match
|
|
3022
|
+
mainstream pre-training; keep `segment_ids` when document isolation is itself
|
|
3023
|
+
the property under test.
|
|
3024
|
+
|
|
3025
|
+
Because document boundaries are derived from `segment_ids` (not the raw
|
|
3026
|
+
`attention_mask`), unless the model declares `causal`, packing a model that only
|
|
3027
|
+
understands a 2-D mask is a hard error rather than a silent cross-document leak.
|
|
3028
|
+
The collator also masks the label opening each document, so the causal shift
|
|
3029
|
+
never predicts across a boundary -- packed training stays equivalent to the
|
|
3030
|
+
per-document padded case and `--mask_prompt` is honoured. `flash-attn` is an
|
|
3031
|
+
optional extra (`pip install genml_kit[flash]`); without it everything runs on
|
|
3032
|
+
the block-diagonal (pure-torch) path, so CPU and CI never require a GPU.
|
|
3033
|
+
|
|
3034
|
+
`--dataset`, `--split`, `--val_split`, `--cache_dir`, `--image_column`,
|
|
3035
|
+
`--max_seq_len`, `--sequence_mode`, `--mask_prompt`, `--pad_to_multiple_of` and
|
|
3036
|
+
`--max_items` are
|
|
3037
|
+
**globally shared** CLI arguments: they are declared once by the harness
|
|
3038
|
+
(`genml_kit/utils/shared_args.py`), so every pipeline and method can read
|
|
3039
|
+
`args.<flag>` directly instead of guarding with `getattr(...)`.
|
|
3040
|
+
|
|
3041
|
+
### Method flags (`--method causal_lm`)
|
|
3042
|
+
|
|
3043
|
+
| Flag | Default | Meaning |
|
|
3044
|
+
|---|---|---|
|
|
3045
|
+
| `--tokenizer` | `--model` | tokenizer name/path (defaults to the model) |
|
|
3046
|
+
| `--trust_remote_code` | off | allow custom code from the model hub |
|
|
3047
|
+
| `--attn_implementation` | — | attention backend (`sdpa`, `eager`, `flash_attention_2`) |
|
|
3048
|
+
|
|
3049
|
+
The metric is `ppl` (perplexity, lower is better), so the best checkpoint is
|
|
3050
|
+
stored under `best_ppl`. `--lora`, `--grad_checkpoint`, `--freeze`,
|
|
3051
|
+
`--source_checkpoint` and `--resume` all work for causal LMs through the same
|
|
3052
|
+
shared model-extras path every other objective uses.
|
|
3053
|
+
|
|
3054
|
+
### Efficient fine-tuning (LoRA, gradient checkpointing, bf16)
|
|
3055
|
+
|
|
3056
|
+
LoRA is the recommended way to fine-tune a causal LM. When
|
|
3057
|
+
`--lora_target_modules` is omitted, genml_kit **resolves the target modules
|
|
3058
|
+
from the model's own structure** rather than from a hardcoded name list, so it
|
|
3059
|
+
works across architectures without configuration:
|
|
3060
|
+
|
|
3061
|
+
| Model family | Auto-resolved targets |
|
|
3062
|
+
|---|---|
|
|
3063
|
+
| GPT-2 (HF `Conv1D`) | `c_attn`, `attn.c_proj` |
|
|
3064
|
+
| Llama / Mistral / Qwen2 / HF ViT | `q_proj`, `k_proj`, `v_proj`, `o_proj` |
|
|
3065
|
+
| BERT | `query`, `key`, `value`, `attention.output.dense` |
|
|
3066
|
+
| timm ViT | `qkv`, `attn.proj` |
|
|
3067
|
+
| No attention (plain MLP backbone) | every `nn.Linear` (with a warning) |
|
|
3068
|
+
|
|
3069
|
+
The resolver detects attention blocks by class/structure, keeps only their
|
|
3070
|
+
projection layers, and emits the shortest dotted suffix that does **not** also
|
|
3071
|
+
match a non-attention layer (so GPT-2's `attn.c_proj` is not confused with
|
|
3072
|
+
`mlp.c_proj`). Pass `--lora_target_modules` explicitly to override.
|
|
3073
|
+
|
|
3074
|
+
<!-- genml-cli-test
|
|
3075
|
+
name: causal-lm-lora
|
|
3076
|
+
network: true
|
|
3077
|
+
extras: lora
|
|
3078
|
+
override:
|
|
3079
|
+
- --dataset {tmp}/text
|
|
3080
|
+
- --model sshleifer/tiny-gpt2
|
|
3081
|
+
-->
|
|
3082
|
+
```bash
|
|
3083
|
+
# LoRA + gradient checkpointing + bf16 on a small LM
|
|
3084
|
+
genml-kit-train --pipeline text \
|
|
3085
|
+
--method causal_lm \
|
|
3086
|
+
--model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
|
|
3087
|
+
--dataset ./my_text_corpus \
|
|
3088
|
+
--text_column text \
|
|
3089
|
+
--max_seq_len 1024 \
|
|
3090
|
+
--lora --lora_r 16 --lora_alpha 32 \
|
|
3091
|
+
--grad_checkpoint \
|
|
3092
|
+
--amp_dtype bfloat16
|
|
3093
|
+
```
|
|
3094
|
+
|
|
3095
|
+
`--lora`, `--lora_r`, `--lora_alpha`, `--lora_dropout` and
|
|
3096
|
+
`--lora_target_modules` are the shared LoRA flags; `--grad_checkpoint` enables
|
|
3097
|
+
per-block activation checkpointing (HF `gradient_checkpointing_enable` for
|
|
3098
|
+
transformers models, native APIs for timm/custom backbones); `--amp_dtype
|
|
3099
|
+
bfloat16` selects the autocast dtype.
|
|
3100
|
+
|
|
3101
|
+
When the model is a PEFT `PeftModel`, checkpoints also embed a **portable
|
|
3102
|
+
adapter blob** (`lora_state_blob`: a tar of `adapter_model.safetensors` +
|
|
3103
|
+
`adapter_config.json`), so the adapters can be loaded with stock PEFT or
|
|
3104
|
+
published to the Hub, while `--resume` continues to restore the raw tensor
|
|
3105
|
+
state exactly.
|
|
3106
|
+
|
|
3107
|
+
### Tokenizers
|
|
3108
|
+
|
|
3109
|
+
`load_tokenizer(name)` mirrors the model/processor split. Normal model IDs
|
|
3110
|
+
resolve through `transformers.AutoTokenizer`, which already handles both
|
|
3111
|
+
SentencePiece and tiktoken Hub vocabularies. Two registry entries cover the
|
|
3112
|
+
gaps: `byte` (a dependency-free tokenizer for offline tests) and
|
|
3113
|
+
`spm` (a bare SentencePiece `tokenizer.model`, behind the optional
|
|
3114
|
+
`sentencepiece` dependency).
|
|
3115
|
+
|
|
3116
|
+
A byte-level BPE is built separately, from a corpus subset, by
|
|
3117
|
+
`genml-kit-tokenizer`. Building is a one-shot offline job rather than a side
|
|
3118
|
+
effect of a training run, so the text behind the vocabulary is a reproducible
|
|
3119
|
+
artifact: pass the resulting directory straight to `--tokenizer`. The
|
|
3120
|
+
`tokenizer_meta.json` sidecar records the corpus, split, fraction, seed and
|
|
3121
|
+
row counts it was trained on.
|
|
3122
|
+
|
|
3123
|
+
<!-- genml-cli-test
|
|
3124
|
+
name: tokenizer-build
|
|
3125
|
+
override:
|
|
3126
|
+
- --dataset {tmp}/text
|
|
3127
|
+
- --out {tmp}/tok
|
|
3128
|
+
extra: --vocab_size 128 --fraction 0.5 --seed 0
|
|
3129
|
+
-->
|
|
3130
|
+
```bash
|
|
3131
|
+
genml-kit-tokenizer --dataset roneneldan/TinyStories --text_column text \
|
|
3132
|
+
--fraction 0.1 --seed 42 --vocab_size 4096 \
|
|
3133
|
+
--cache_dir hf --out hf/tokenizers/ts_bpe_4096_s0.1
|
|
3134
|
+
```
|
|
3135
|
+
|
|
3136
|
+
`--fraction (0.0, 1.0]` trains on that share of the rows: an indexable dataset
|
|
3137
|
+
yields exactly `round(n*fraction)` rows, drawn without replacement and
|
|
3138
|
+
seed-stable, while `--streaming` corpora get a seeded per-row sample (use
|
|
3139
|
+
`--max_rows` to bound how much of a stream is read).
|
|
3140
|
+
|
|
3141
|
+
### Generation
|
|
3142
|
+
|
|
3143
|
+
`genml-kit-llm generate` turns prompts into completions. Generation is a
|
|
3144
|
+
**model contract**, not a HuggingFace-only path: a model opts in by
|
|
3145
|
+
subclassing `GenerativeLM` (`genml_kit/models/generative.py`), whose default
|
|
3146
|
+
`generate` runs a portable token loop over the architecture-agnostic
|
|
3147
|
+
`forward(...) -> .logits` contract that `causal_lm` already trains against.
|
|
3148
|
+
HuggingFace models are *not* wrapped (their `state_dict` keys must stay intact
|
|
3149
|
+
for `--resume`); the dispatcher calls their own `generate` directly. So custom
|
|
3150
|
+
models registered in `MODELS` generate exactly like HF ones.
|
|
3151
|
+
|
|
3152
|
+
<!-- genml-cli-test
|
|
3153
|
+
name: llm-generate
|
|
3154
|
+
network: true
|
|
3155
|
+
override:
|
|
3156
|
+
- --prompts {tmp}/prompts/prompts.txt
|
|
3157
|
+
- --dataset {tmp}/prompt_ds --split test
|
|
3158
|
+
- --model sshleifer/tiny-gpt2
|
|
3159
|
+
- --checkpoint {tmp}/chk_infer/model_best.pt
|
|
3160
|
+
- -.--lora_adapter
|
|
3161
|
+
-->
|
|
3162
|
+
```bash
|
|
3163
|
+
# Prompts from a file (one per line, or JSONL with --prompt_column)
|
|
3164
|
+
genml-kit-llm generate --model sshleifer/tiny-gpt2 \
|
|
3165
|
+
--prompts prompts.txt \
|
|
3166
|
+
--max_new_tokens 64 \
|
|
3167
|
+
--do_sample --temperature 0.8 --top_p 0.95 \
|
|
3168
|
+
--output completions.jsonl
|
|
3169
|
+
|
|
3170
|
+
# Prompts from a HuggingFace dataset column, with a fine-tuned checkpoint
|
|
3171
|
+
genml-kit-llm generate --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
|
|
3172
|
+
--dataset my-org/prompts --split test \
|
|
3173
|
+
--prompt_column question \
|
|
3174
|
+
--checkpoint ./checkpoints/llm_best.pt \
|
|
3175
|
+
--lora_adapter ./adapters/run1
|
|
3176
|
+
```
|
|
3177
|
+
|
|
3178
|
+
| Argument | Default | Description |
|
|
3179
|
+
|---|---|---|
|
|
3180
|
+
| `--model` | (required) | HuggingFace id or registered custom model. |
|
|
3181
|
+
| `--tokenizer` | `--model` | Tokenizer name/path override. |
|
|
3182
|
+
| `--checkpoint` | `None` | State-dict `.pt`, or a `<root>` resolving to `<root>_best.pt` / `<root>_latest.pt`. |
|
|
3183
|
+
| `--lora_adapter` | `None` | PEFT adapter directory (`save_pretrained` layout). |
|
|
3184
|
+
| `--prompts` / `--dataset` | — | Prompt source (file/JSONL, or HF dataset + `--prompt_column`). |
|
|
3185
|
+
| `--max_new_tokens` | `64` | Tokens to generate. |
|
|
3186
|
+
| `--do_sample` | off | Sample instead of greedy decoding. |
|
|
3187
|
+
| `--temperature` / `--top_k` / `--top_p` | `1.0` / `0` / `1.0` | Sampling knobs. |
|
|
3188
|
+
| `--repetition_penalty` | `1.0` | Down-weight already-seen tokens (`1.0` disables). |
|
|
3189
|
+
| `--num_beams` | `1` | Beam count (portable backend; `1` = greedy). |
|
|
3190
|
+
| `--num_return_sequences` | `1` | Completions per prompt. |
|
|
3191
|
+
| `--no_chat_template` | off | Skip the tokenizer's chat template. |
|
|
3192
|
+
|
|
3193
|
+
When the tokenizer exposes a chat template (most instruct models do),
|
|
3194
|
+
prompts are automatically wrapped as a single user turn; `--no_chat_template`
|
|
3195
|
+
keeps the raw prompt for base-model completion.
|
|
3196
|
+
|
|
3197
|
+
### Evaluation
|
|
3198
|
+
|
|
3199
|
+
`genml-kit-llm eval` reports quality metrics for a labeled dataset:
|
|
3200
|
+
|
|
3201
|
+
- **perplexity** — teacher-forced (`exp` of the mean shifted cross-entropy),
|
|
3202
|
+
over the `--text_column`; no generation involved.
|
|
3203
|
+
- **exact-match** — fraction of generations matching the reference exactly.
|
|
3204
|
+
- **token-F1** — SQuAD-style token-overlap F1 against the reference.
|
|
3205
|
+
- **pass@k** — optional and pluggable: `--pass_at_k K --scorer score.py`
|
|
3206
|
+
invokes `score(prompt, completion, reference) -> float` `K` times per item
|
|
3207
|
+
and reports the pass rate. No code execution is built in.
|
|
3208
|
+
|
|
3209
|
+
<!-- genml-cli-test
|
|
3210
|
+
name: llm-eval
|
|
3211
|
+
network: true
|
|
3212
|
+
override:
|
|
3213
|
+
- --dataset {tmp}/eval
|
|
3214
|
+
- --model sshleifer/tiny-gpt2
|
|
3215
|
+
-->
|
|
3216
|
+
```bash
|
|
3217
|
+
genml-kit-llm eval --model sshleifer/tiny-gpt2 \
|
|
3218
|
+
--dataset ./my_eval_corpus --split test \
|
|
3219
|
+
--text_column text --prompt_column prompt \
|
|
3220
|
+
--reference_column reference \
|
|
3221
|
+
--metrics ppl exact_match token_f1 \
|
|
3222
|
+
--output report.json
|
|
3223
|
+
```
|
|
3224
|
+
|
|
3225
|
+
Both subcommands load their model through the same registry the trainer uses,
|
|
3226
|
+
so a registered custom model generates and evaluates with no code changes.
|
|
3227
|
+
|
|
3228
|
+
### Alignment (DPO and reward models)
|
|
3229
|
+
|
|
3230
|
+
Preference-based post-training runs through the same branchless driver: a new
|
|
3231
|
+
`preference` pipeline (chosen/rejected pairs) pairs with the `dpo` and
|
|
3232
|
+
`reward_model` methods. The typical recipe is SFT → DPO → (optional) reward
|
|
3233
|
+
model:
|
|
3234
|
+
|
|
3235
|
+
<!-- genml-cli-test
|
|
3236
|
+
name: preference-chain
|
|
3237
|
+
network: true
|
|
3238
|
+
extras: lora
|
|
3239
|
+
extra: --epochs 1
|
|
3240
|
+
-->
|
|
3241
|
+
```bash
|
|
3242
|
+
# 1. Supervised fine-tune (masking the prompt out of the loss)
|
|
3243
|
+
genml-kit-train --pipeline text --method causal_lm \
|
|
3244
|
+
--model sshleifer/tiny-gpt2 --dataset ./sft_corpus \
|
|
3245
|
+
--instruction_column prompt --response_column response \
|
|
3246
|
+
--mask_prompt --checkpoint ./sft/model
|
|
3247
|
+
|
|
3248
|
+
# 2. DPO against a frozen reference (defaults to --model)
|
|
3249
|
+
genml-kit-train --pipeline preference --method dpo \
|
|
3250
|
+
--model sshleifer/tiny-gpt2 \
|
|
3251
|
+
--reference_checkpoint ./sft/model_best.pt \
|
|
3252
|
+
--dataset ./preferences --chosen_column chosen \
|
|
3253
|
+
--rejected_column rejected --prompt_column prompt \
|
|
3254
|
+
--mask_prompt --beta 0.1 --lora
|
|
3255
|
+
|
|
3256
|
+
# 3. Train a reward model on the same pairs
|
|
3257
|
+
genml-kit-train --pipeline preference --method reward_model \
|
|
3258
|
+
--model sshleifer/tiny-gpt2 --dataset ./preferences \
|
|
3259
|
+
--chosen_column chosen --rejected_column rejected
|
|
3260
|
+
```
|
|
3261
|
+
|
|
3262
|
+
**DPO** compares the policy against a **frozen reference** and pushes the
|
|
3263
|
+
chosen response's implicit reward above the rejected one. The reference is
|
|
3264
|
+
loaded explicitly (`--reference_model`, defaulting to `--model`, plus
|
|
3265
|
+
`--reference_checkpoint`) and held as a plain attribute — it is never written
|
|
3266
|
+
to `state_dict` and is rebuilt from args on resume. The sequence score is the
|
|
3267
|
+
mean per-token log-probability of the response tokens, and `--beta` (default
|
|
3268
|
+
`0.1`) sets the KL strength. Validation reports pairwise preference accuracy.
|
|
3269
|
+
|
|
3270
|
+
**Reward model** attaches a scalar `nn.Linear` head directly to the backbone
|
|
3271
|
+
as `model.score` — no wrapper class, so the backbone's `state_dict` keys stay
|
|
3272
|
+
unprefixed and `--source_checkpoint` matches them exactly (only `score.*` is
|
|
3273
|
+
new). It scores the last non-pad hidden state and trains with the
|
|
3274
|
+
Bradley–Terry ranking loss; validation reports pairwise ranking accuracy. The
|
|
3275
|
+
hidden size is read from the config (`hidden_size` / `n_embd` / `d_model`),
|
|
3276
|
+
falling back to a one-token probe.
|
|
3277
|
+
|
|
3278
|
+
The backbone must expose last-layer hidden states (accept
|
|
3279
|
+
`output_hidden_states=True`); HuggingFace models do, custom models opt in —
|
|
3280
|
+
the same contract used by `GenerativeLM`.
|
|
3281
|
+
|
|
3282
|
+
---
|
|
3283
|
+
|
|
1665
3284
|
## Reinforcement Learning
|
|
1666
3285
|
|
|
1667
3286
|
genml_kit ships a self-contained reinforcement-learning (RL) toolkit built on
|
|
@@ -1688,7 +3307,12 @@ pip install 'genml_kit[rl]'
|
|
|
1688
3307
|
|
|
1689
3308
|
### Quick start
|
|
1690
3309
|
|
|
1691
|
-
|
|
3310
|
+
<!-- genml-cli-test
|
|
3311
|
+
name: rl-quick-start
|
|
3312
|
+
extras: rl
|
|
3313
|
+
extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
|
|
3314
|
+
-->
|
|
3315
|
+
```bash
|
|
1692
3316
|
# DQN on CartPole (discrete)
|
|
1693
3317
|
genml-kit-train --pipeline rl --method dqn --env_id CartPole-v1 \
|
|
1694
3318
|
--epochs 20 --batch_size 64
|
|
@@ -1713,7 +3337,12 @@ Every RL method resolves its network through the same model registry as the
|
|
|
1713
3337
|
rest of the framework: `--model` selects a registered factory and
|
|
1714
3338
|
`--model_arg key=value` forwards constructor kwargs.
|
|
1715
3339
|
|
|
1716
|
-
|
|
3340
|
+
<!-- genml-cli-test
|
|
3341
|
+
name: rl-custom-models
|
|
3342
|
+
extras: rl
|
|
3343
|
+
extra: --epochs 1
|
|
3344
|
+
-->
|
|
3345
|
+
```bash
|
|
1717
3346
|
# DQN: dueling Q-head (also the default for --dqn_dueling)
|
|
1718
3347
|
genml-kit-train --pipeline rl --method dqn --model rl/qnet_dueling
|
|
1719
3348
|
|
|
@@ -1907,12 +3536,34 @@ a small positive default as light regularization.
|
|
|
1907
3536
|
|
|
1908
3537
|
A typical "serious" continuous-PPO setup (Pendulum-style) looks like:
|
|
1909
3538
|
|
|
1910
|
-
|
|
3539
|
+
<!-- genml-cli-test
|
|
3540
|
+
name: rl-ppo-serious
|
|
3541
|
+
extras: rl
|
|
3542
|
+
extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
|
|
3543
|
+
-->
|
|
3544
|
+
```bash
|
|
1911
3545
|
genml-kit-train --pipeline rl --method ppo --env_id Pendulum-v1 \
|
|
1912
3546
|
--ppo_continuous --obs_normalize --reward_normalize --param_init ortho \
|
|
1913
3547
|
--ppo_target_kl 0.03 --epochs 500
|
|
1914
3548
|
```
|
|
1915
3549
|
|
|
3550
|
+
SAC's temperature auto-tuning and critic/policy learning rates follow the
|
|
3551
|
+
same pattern (the critic head is `rl/sac_critic`, the policy is
|
|
3552
|
+
`rl/actor_critic`):
|
|
3553
|
+
|
|
3554
|
+
<!-- genml-cli-test
|
|
3555
|
+
name: rl-sac-tuning
|
|
3556
|
+
extras: rl
|
|
3557
|
+
extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
|
|
3558
|
+
-->
|
|
3559
|
+
```bash
|
|
3560
|
+
genml-kit-train --pipeline rl --method sac --env_id Pendulum-v1 \
|
|
3561
|
+
--sac_auto_alpha --sac_alpha_lr 3e-4 --sac_target_entropy -1.0 \
|
|
3562
|
+
--sac_critic_lr 3e-4 --sac_actor_lr 3e-4 --sac_tau 0.005 \
|
|
3563
|
+
--replay_capacity 5000 --batch_size 64 \
|
|
3564
|
+
--epochs 200
|
|
3565
|
+
```
|
|
3566
|
+
|
|
1916
3567
|
### Custom environments
|
|
1917
3568
|
|
|
1918
3569
|
Pass `--env_script /path/to/env.py` to load a custom environment. The
|
|
@@ -1928,6 +3579,31 @@ Relevant flags:
|
|
|
1928
3579
|
- `--env_id`, `--obs_dim`, `--env_script` -- environment selection.
|
|
1929
3580
|
- `--replay_capacity`, `--batch_size`, `--n_step`, `--prioritized`,
|
|
1930
3581
|
`--per_alpha`, `--per_beta_start`, `--per_beta_frames` -- replay tuning.
|
|
3582
|
+
|
|
3583
|
+
A DQN run with the full replay/epsilon machinery (dueling head, double-Q,
|
|
3584
|
+
n-step returns, prioritized replay):
|
|
3585
|
+
|
|
3586
|
+
<!-- genml-cli-test
|
|
3587
|
+
name: rl-dqn-replay
|
|
3588
|
+
extras: rl
|
|
3589
|
+
extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
|
|
3590
|
+
-->
|
|
3591
|
+
```bash
|
|
3592
|
+
genml-kit-train --pipeline rl --method dqn --env_id CartPole-v1 \
|
|
3593
|
+
--dqn_dueling --dqn_ddqn --n_step 3 --prioritized \
|
|
3594
|
+
--replay_capacity 5000 --batch_size 32 \
|
|
3595
|
+
--dqn_gamma 0.99 --dqn_tau 0.005 --dqn_target_update_freq 100 \
|
|
3596
|
+
--dqn_epsilon_start 1.0 --dqn_epsilon_end 0.05 \
|
|
3597
|
+
--dqn_epsilon_decay_steps 500 \
|
|
3598
|
+
--epochs 50
|
|
3599
|
+
```
|
|
3600
|
+
- **Discount factor** -- there is intentionally **no** global `--gamma`: each
|
|
3601
|
+
method owns its own because RL objectives are not active simultaneously and
|
|
3602
|
+
tune it differently. DQN uses `--dqn_gamma`, PPO `--ppo_gamma`, SAC
|
|
3603
|
+
`--sac_gamma` (all default `0.99`). The n-step returns computed by the
|
|
3604
|
+
shared replay buffer use the discount of the **selected method**; the
|
|
3605
|
+
method installs it into the buffer during `wire_data`, so a single run never
|
|
3606
|
+
mixes discounts.
|
|
1931
3607
|
- `--obs_normalize`, `--obs_norm_clip` -- running observation
|
|
1932
3608
|
normalization (Welford). Both train and eval observations are
|
|
1933
3609
|
normalized with the same RMS statistics.
|
|
@@ -1960,6 +3636,209 @@ bootstraps `γ V(s′)`, while a terminated one does not. Environments using
|
|
|
1960
3636
|
the legacy 4-tuple gym API (no `terminated`/`truncated` split) fall back to
|
|
1961
3637
|
treating `done` as `terminated`.
|
|
1962
3638
|
|
|
3639
|
+
## Multi-GPU Training
|
|
3640
|
+
|
|
3641
|
+
The harness runs on a single device by default and needs no launcher for that.
|
|
3642
|
+
To use more than one GPU, launch the same `genml-kit-train` command under
|
|
3643
|
+
`torchrun` (which ships with PyTorch -- it is the `torch.distributed.run`
|
|
3644
|
+
module, so there is nothing extra to install):
|
|
3645
|
+
|
|
3646
|
+
<!-- genml-cli-test-exclude: illustrative GPU/nccl launch; the runnable distributed example follows -->
|
|
3647
|
+
```bash
|
|
3648
|
+
# 4 GPUs, one process per GPU
|
|
3649
|
+
torchrun --standalone --nproc_per_node=4 -m genml_kit.cli.train \
|
|
3650
|
+
--model convvit \
|
|
3651
|
+
--dataset /data/imagenet \
|
|
3652
|
+
--epochs 90 \
|
|
3653
|
+
--batch_size 64 \
|
|
3654
|
+
--dist_arg backend=nccl
|
|
3655
|
+
```
|
|
3656
|
+
|
|
3657
|
+
The same command runs on CPU with the `gloo` backend, which is what the
|
|
3658
|
+
runnable example below does (two ranks, a tiny synthetic dataset):
|
|
3659
|
+
|
|
3660
|
+
<!-- genml-cli-test
|
|
3661
|
+
name: multi-gpu-torchrun
|
|
3662
|
+
launcher: torchrun
|
|
3663
|
+
nproc: 2
|
|
3664
|
+
extra: --device cpu --batch_size 4 --image_size 32 --num_workers 0
|
|
3665
|
+
override:
|
|
3666
|
+
- --dataset {tmp}/img_ddp
|
|
3667
|
+
-->
|
|
3668
|
+
```bash
|
|
3669
|
+
# 2 processes on CPU (gloo); swap in 4 GPUs + `backend=nccl` for a real run
|
|
3670
|
+
torchrun --standalone --nproc_per_node=2 -m genml_kit.cli.train \
|
|
3671
|
+
--model convvit \
|
|
3672
|
+
--dataset /data/imagenet \
|
|
3673
|
+
--epochs 1 \
|
|
3674
|
+
--dist_arg backend=gloo
|
|
3675
|
+
```
|
|
3676
|
+
|
|
3677
|
+
Every rank reads `RANK` / `WORLD_SIZE` / `LOCAL_RANK` / `MASTER_ADDR` /
|
|
3678
|
+
`MASTER_PORT` from the environment; when they are absent (a plain
|
|
3679
|
+
`genml-kit-train ...` invocation) the distributed layer is completely inert --
|
|
3680
|
+
no process group is created, the model is not wrapped, and the data loader is
|
|
3681
|
+
untouched. Single-GPU behaviour is therefore unchanged, and `torchrun` is
|
|
3682
|
+
never required.
|
|
3683
|
+
|
|
3684
|
+
What the launcher turns on automatically:
|
|
3685
|
+
|
|
3686
|
+
| Behaviour | Effect |
|
|
3687
|
+
|---|---|
|
|
3688
|
+
| Process group | `nccl` on CUDA, `gloo` otherwise; override with `--dist_arg backend=` |
|
|
3689
|
+
| Device | each rank pins to `cuda:$LOCAL_RANK` |
|
|
3690
|
+
| Seeding | `base_seed + rank`, so ranks shuffle differently but reproducibly |
|
|
3691
|
+
| Data sharding | each rank gets a disjoint slice of the training set (see below) |
|
|
3692
|
+
| Checkpoints | written by rank 0 only; keys carry no `module.` prefix, so a multi-GPU checkpoint loads back into a single-GPU run unchanged |
|
|
3693
|
+
| Metrics / log files | TensorBoard, JSONL and log files are written by rank 0 only |
|
|
3694
|
+
| Validation | sharded across ranks like training; each rank evaluates its disjoint slice, the per-rank results are merged (`Method.merge_validation`), and rank 0 reports the whole-set metric |
|
|
3695
|
+
|
|
3696
|
+
Distributed knobs travel through a single repeatable `--dist_arg KEY=VALUE`
|
|
3697
|
+
flag, matching `--model_arg` / `--opt_arg`. A bare key means `true`:
|
|
3698
|
+
|
|
3699
|
+
| Key | Default | Description |
|
|
3700
|
+
|---|---|---|
|
|
3701
|
+
| `backend` | auto (`nccl` on CUDA, else `gloo`) | Process-group backend. |
|
|
3702
|
+
| `shard` | `true` when more than one rank | Shard the training set across ranks. Set `shard=false` to run every rank over the full dataset (rarely what you want: it multiplies the compute with no error). |
|
|
3703
|
+
| `find_unused_parameters` | `false` (declared `true` by BYOL / DINO / SupCon) | Pass through to `DistributedDataParallel`. Lets a forward that skips some trainable parameters survive DDP's unused-parameter check. The SSL methods declare it defensively; a stock run leaves nothing unused and would pass without it. |
|
|
3704
|
+
| `static_graph` | `false` | Pass through to `DistributedDataParallel`. Only valid when the autograd graph is identical every step. |
|
|
3705
|
+
| `rollout_shard` | `true` when more than one rank | RL pipelines only. Shard rollout collection (and the minibatch it yields) across ranks. Set `rollout_shard=false` to collect one full rollout per rank (rarely what you want: it multiplies the environment compute with no error). |
|
|
3706
|
+
|
|
3707
|
+
Defaults are chosen so that a multi-GPU run only has to be correct, not
|
|
3708
|
+
tuned: sharding is on whenever it can matter. The validation set is sharded the
|
|
3709
|
+
same way as training, each rank evaluates its disjoint slice, and the per-rank
|
|
3710
|
+
results are merged (`Method.merge_validation`) so the reported metric always
|
|
3711
|
+
describes the whole set; only rank 0 logs it.
|
|
3712
|
+
|
|
3713
|
+
These defaults are *method-aware*: the selected pipeline or method may declare
|
|
3714
|
+
its own default for any flag via `arg_defaults()`, and `--dist_arg` merges on
|
|
3715
|
+
top of it key by key. An explicit command-line flag always wins, so
|
|
3716
|
+
`--dist_arg static_graph` still runs a method's declared
|
|
3717
|
+
`find_unused_parameters` alongside it.
|
|
3718
|
+
|
|
3719
|
+
Three caveats worth knowing before reaching for more GPUs:
|
|
3720
|
+
|
|
3721
|
+
- **Effective batch size scales with `--nproc_per_node`.** Four ranks at
|
|
3722
|
+
`--batch_size 64` see 256 samples per optimizer step; scale `--lr`
|
|
3723
|
+
accordingly.
|
|
3724
|
+
- **`shard` assumes a trainable `DataLoader`.** Pipelines that bypass it are
|
|
3725
|
+
not covered by *that* path: the RL pipelines drive environments and replay
|
|
3726
|
+
buffers directly, so they use their own `rollout_shard` knob (see the table
|
|
3727
|
+
above) instead of `shard`.
|
|
3728
|
+
- **A checkpoint from a multi-GPU run is portable**, but resuming with a
|
|
3729
|
+
*different* `--nproc_per_node` changes the shard boundaries, so the epoch
|
|
3730
|
+
position is approximate if the dataset size is not divisible by the rank
|
|
3731
|
+
count.
|
|
3732
|
+
|
|
3733
|
+
---
|
|
3734
|
+
|
|
3735
|
+
## Hyperparameter Search
|
|
3736
|
+
|
|
3737
|
+
`genml-kit-tune` runs a hyperparameter *study* on top of the unified trainer.
|
|
3738
|
+
It is deliberately decoupled from the training code: **every trial is a fresh
|
|
3739
|
+
`genml-kit-train` subprocess**, and the only contracts between the tuner and a
|
|
3740
|
+
trial are the command line it is handed and the `<log_dir>/metrics.jsonl` file
|
|
3741
|
+
the JSONL sink already writes. No training internals are imported, so the
|
|
3742
|
+
isolation boundary *is* the process boundary.
|
|
3743
|
+
|
|
3744
|
+
A study is data, not code — describe it in YAML (or all on the command line):
|
|
3745
|
+
|
|
3746
|
+
```yaml
|
|
3747
|
+
# study.yaml
|
|
3748
|
+
objective: val/macro_f1 # metric tag to optimise
|
|
3749
|
+
direction: max # max | min
|
|
3750
|
+
sampler: tpe # grid | random | tpe | gpei
|
|
3751
|
+
pruner: asha # none | median | asha
|
|
3752
|
+
max_trials: 50
|
|
3753
|
+
max_epochs_per_trial: 20 # budget caps each trial
|
|
3754
|
+
space:
|
|
3755
|
+
opt.lr: log(1e-4:1e-1) # log-uniform float range
|
|
3756
|
+
batch_size: [32, 64, 128] # categorical choice
|
|
3757
|
+
dropout: 0.0:0.5 # uniform float range
|
|
3758
|
+
model.embed_dim: 64:1024 # integer range
|
|
3759
|
+
base_args: ["--pipeline", "images", "--method", "classification"]
|
|
3760
|
+
```
|
|
3761
|
+
|
|
3762
|
+
### Running a study
|
|
3763
|
+
|
|
3764
|
+
Points after a literal `--` are forwarded verbatim to every trial:
|
|
3765
|
+
|
|
3766
|
+
<!-- genml-cli-test
|
|
3767
|
+
name: tune-run-study
|
|
3768
|
+
override:
|
|
3769
|
+
- --max_trials 2
|
|
3770
|
+
- --workers 1
|
|
3771
|
+
- -.--model
|
|
3772
|
+
- -.--dataset
|
|
3773
|
+
- -- --model convvit --dataset {tmp}/img --image_size 32 --max_items 16
|
|
3774
|
+
-->
|
|
3775
|
+
```bash
|
|
3776
|
+
genml-kit-tune run \
|
|
3777
|
+
--space opt.lr=log(1e-4:1e-1) --space batch_size=32,64 \
|
|
3778
|
+
--objective val/macro_f1 --direction max --sampler tpe --pruner asha \
|
|
3779
|
+
--max_trials 8 --workers 2 \
|
|
3780
|
+
--epochs 1 \
|
|
3781
|
+
-- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
|
|
3782
|
+
```
|
|
3783
|
+
|
|
3784
|
+
The equivalent all-YAML form is `genml-kit-tune run --study study.yaml`; command
|
|
3785
|
+
line flags override the file. Trials land under
|
|
3786
|
+
`runs/study-<timestamp>/trials/<id>/` (each with its own `metrics.jsonl` and
|
|
3787
|
+
checkpoints) and the study ledger is `runs/study-<timestamp>/study.jsonl`.
|
|
3788
|
+
|
|
3789
|
+
### Samplers and pruners
|
|
3790
|
+
|
|
3791
|
+
| Name | Kind | Notes |
|
|
3792
|
+
|---|---|---|
|
|
3793
|
+
| `grid` | sampler | Deterministic Cartesian product; resume-safe. |
|
|
3794
|
+
| `random` | sampler | Uniform draws over the space. |
|
|
3795
|
+
| `tpe` | sampler | Tree-structured Parzen Estimator — sample-efficient Bayesian optimisation (numpy only). The default. |
|
|
3796
|
+
| `gpei` | sampler | Gaussian-Process Expected Improvement (registered only when `scipy` is installed; `pip install "genml_kit[tune]"`). |
|
|
3797
|
+
| `none` | pruner | Run every trial to its full budget. |
|
|
3798
|
+
| `median` | pruner | Stop trials below the running median at a rung. |
|
|
3799
|
+
| `asha` | pruner | Asynchronous Successive Halving; keeps the top `1/reduction_factor` at each rung. |
|
|
3800
|
+
| `hyperband` | pruner | Multiple ASHA brackets over `min_resource * rf^s` budgets; trials are assigned round-robin so every resource level gets explored. |
|
|
3801
|
+
|
|
3802
|
+
Pruning is delivered as a clean `SIGTERM` to the trial subprocess, which the
|
|
3803
|
+
trainer's existing signal guard turns into a consistent checkpoint before exit.
|
|
3804
|
+
|
|
3805
|
+
### Concurrency, GPUs and resume
|
|
3806
|
+
|
|
3807
|
+
By default studies run **one trial at a time**. `--workers N` runs `N` trials
|
|
3808
|
+
concurrently, and `--gpus 0,1,2,3` pins each trial to a slot from that pool via
|
|
3809
|
+
Interrupt a study at any time and continue it later (the second command
|
|
3810
|
+
prints the leaderboard of the same study):
|
|
3811
|
+
|
|
3812
|
+
<!-- genml-cli-test
|
|
3813
|
+
name: tune-resume-report
|
|
3814
|
+
override:
|
|
3815
|
+
- --max_trials 2
|
|
3816
|
+
- -.--model
|
|
3817
|
+
- -.--dataset
|
|
3818
|
+
- -- --model convvit --dataset {tmp}/img --image_size 32 --max_items 16
|
|
3819
|
+
-->
|
|
3820
|
+
```bash
|
|
3821
|
+
# First invocation creates the study; a later one resumes it (the budget
|
|
3822
|
+
# already spent is subtracted, so only the remaining trials run).
|
|
3823
|
+
genml-kit-tune run --study_dir runs/study-20250101-120000 \
|
|
3824
|
+
--space opt.lr=log(1e-4:1e-1) --objective val/macro_f1 --max_trials 50 \
|
|
3825
|
+
-- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
|
|
3826
|
+
genml-kit-tune run --study_dir runs/study-20250101-120000 --resume \
|
|
3827
|
+
--space opt.lr=log(1e-4:1e-1) --objective val/macro_f1 --max_trials 50 \
|
|
3828
|
+
-- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
|
|
3829
|
+
genml-kit-tune report runs/study-20250101-120000 --top 10
|
|
3830
|
+
```
|
|
3831
|
+
|
|
3832
|
+
### Extending
|
|
3833
|
+
|
|
3834
|
+
Both algorithm families are registries, exactly like `METHODS`/`PIPELINES`.
|
|
3835
|
+
A new sampler subclasses `Sampler` (implementing `suggest(observed, n)`) and
|
|
3836
|
+
decorates itself with `@SEARCHERS.register`; a new pruner subclasses `Pruner`
|
|
3837
|
+
(implementing `_decide(rung, value)`) and uses `@PRUNERS.register`. The
|
|
3838
|
+
subsystem stays zero-dependency: `tpe`, `grid` and `random` need only numpy,
|
|
3839
|
+
and the scipy-backed `gpei` registers itself only when scipy is importable
|
|
3840
|
+
(`pip install "genml_kit[tune]"`).
|
|
3841
|
+
|
|
1963
3842
|
## References And Further Reading
|
|
1964
3843
|
|
|
1965
3844
|
- Xie et al., [SimMIM: A Simple Framework for Masked Image Modeling](https://arxiv.org/abs/2111.09886), CVPR 2022.
|
|
@@ -2000,6 +3879,42 @@ instead of scrolling past. Code is formatted with
|
|
|
2000
3879
|
(Google style, 2-space indent) and linted with
|
|
2001
3880
|
[Ruff](https://docs.astral.sh/ruff/) (`ruff check .`).
|
|
2002
3881
|
|
|
3882
|
+
### Verifying README commands
|
|
3883
|
+
|
|
3884
|
+
Command blocks preceded by a `<!-- genml-cli-test ... -->` HTML comment are
|
|
3885
|
+
executed by `tests/test_readme_cli.py`. The comment is invisible on
|
|
3886
|
+
GitHub; its YAML body controls limiting args, network/token markers, and
|
|
3887
|
+
substitutions. Run them with `pytest -m network` (needs internet, no
|
|
3888
|
+
credentials) — the offline suite excludes them by default.
|
|
3889
|
+
|
|
3890
|
+
A fence marked with `launcher: torchrun` is rewritten to a multi-process
|
|
3891
|
+
`torch.distributed.run` invocation and executed that way (run it with
|
|
3892
|
+
`pytest -m distributed`); this is opt-in per fence, so every other example
|
|
3893
|
+
keeps running single-process.
|
|
3894
|
+
|
|
3895
|
+
A fence that talks to an external service names it with `service: <name>`
|
|
3896
|
+
(`hf`, `s3`, `r2`, `gcs`, `github`, `wandb`). Such a fence is run when the
|
|
3897
|
+
service is reachable either through its credentials **or** through an
|
|
3898
|
+
endpoint-override proxy/emulator, because the underlying client libraries
|
|
3899
|
+
honour their own override variables natively:
|
|
3900
|
+
|
|
3901
|
+
| `service:` | credentials | endpoint override |
|
|
3902
|
+
|---|---|---|
|
|
3903
|
+
| `hf` | `HF_TOKEN`, `HUGGING_FACE_HUB_TOKEN` | `HF_ENDPOINT` |
|
|
3904
|
+
| `s3` | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL` |
|
|
3905
|
+
| `r2` | `R2_ACCESS_KEY_ID` + `R2_SECRET_ACCESS_KEY` | `R2_ENDPOINT_URL`, `AWS_ENDPOINT_URL_S3` |
|
|
3906
|
+
| `gcs` | `GOOGLE_APPLICATION_CREDENTIALS` | `STORAGE_EMULATOR_HOST` |
|
|
3907
|
+
| `github` | `GITHUB_TOKEN`, `GH_TOKEN` | `GITHUB_API_URL` |
|
|
3908
|
+
| `wandb` | `WANDB_API_KEY` | `WANDB_BASE_URL` |
|
|
3909
|
+
|
|
3910
|
+
Select them with `pytest -m service` (all services) or a per-service marker
|
|
3911
|
+
such as `pytest -m hf`. `service:` implies `network: true`.
|
|
3912
|
+
|
|
3913
|
+
Every `bash`/`sh` block that contains a `genml-kit-*` command must either be
|
|
3914
|
+
marked for execution as above or explicitly opted out with a
|
|
3915
|
+
`<!-- genml-cli-test-exclude: <reason> -->` comment; a test enforces this so a
|
|
3916
|
+
new example cannot be added untested by accident.
|
|
3917
|
+
|
|
2003
3918
|
### Naming Conventions For Class Members
|
|
2004
3919
|
|
|
2005
3920
|
- Anything that PyTorch registers — child `nn.Module`, `nn.Parameter`,
|
|
@@ -2011,7 +3926,7 @@ instead of scrolling past. Code is formatted with
|
|
|
2011
3926
|
leading underscore. A member called from other classes, overridden as
|
|
2012
3927
|
an extension point, or documented as API stays public.
|
|
2013
3928
|
- Extension-point methods that *callers* invoke (e.g.
|
|
2014
|
-
`
|
|
3929
|
+
`Method.build_transform`) are public; same-named hooks the
|
|
2015
3930
|
base class *itself* dispatches (e.g. `BaseImageProcessor._build_transform`)
|
|
2016
3931
|
are private.
|
|
2017
3932
|
|